BatchSuperSeggerOpti

 BatchSuperSeggerOpti : runs everything from start to finish,
 including alignment, building the directory structure,
single image segmentation, error resolution, cell linking,
 fluorescence analysis, and cell files.

 Processes a raw data set by
 (1) Aligning and cropping time series
 (2) Organizing files into directories
        xy points are put in their own dir's
        phase, fluor files put in their own dir's
        makes seg and cell directories
 (3) Segmenting the frames into cell regions
 (4) Linking the regions between time steps
 (5) finding loci and calculating fluor statistics
 (6) Putting complete cells into the cell dir.

 INPUT :
 dirname_ : dir containing raw tif files
 skip     : The segmentation is performed every skip files
          : this skip is very useful for high frequency timelapse where
          : cells would switch back and forth between one and two
          : segments leading to errors that are difficult to resolve.
          : This segments every skip files, then copies the segments into
          : the intermediate frames.
 clean_flag : Set this to be true to start from scratch and reseg all the
            : files regardless of whether any seg files exist. If this
            : flag is false, use existing segments, if they exist and
            : new segs if they don't yet exist.
 res       : is a string that is passed to loadConstants(Mine).m to load
           : the right constants for processing.
 showWarnings : Set to 0 to mute warnings
 startEnd : array of two values to indicate where to start and where to
 stop the program. 1, alignment, 2, segmentation, 3, stripping, 4 linking,
 5, cell marker, 6 : fluor, 7 : foci, 8, cellA structrues, 9, clist, 10 cell files.

 Copyright (C) 2016 Wiggins Lab
 Written by Paul Wiggins & Stella Stylianidou.
 University of Washington, 2016
 This file is part of SuperSegger.

0001 function BatchSuperSeggerOpti(dirname_,skip,clean_flag,res,startEnd,showWarnings)
0002 % BatchSuperSeggerOpti : runs everything from start to finish,
0003 % including alignment, building the directory structure,
0004 %single image segmentation, error resolution, cell linking,
0005 % fluorescence analysis, and cell files.
0006 %
0007 % Processes a raw data set by
0008 % (1) Aligning and cropping time series
0009 % (2) Organizing files into directories
0010 %        xy points are put in their own dir's
0011 %        phase, fluor files put in their own dir's
0012 %        makes seg and cell directories
0013 % (3) Segmenting the frames into cell regions
0014 % (4) Linking the regions between time steps
0015 % (5) finding loci and calculating fluor statistics
0016 % (6) Putting complete cells into the cell dir.
0017 %
0018 % INPUT :
0019 % dirname_ : dir containing raw tif files
0020 % skip     : The segmentation is performed every skip files
0021 %          : this skip is very useful for high frequency timelapse where
0022 %          : cells would switch back and forth between one and two
0023 %          : segments leading to errors that are difficult to resolve.
0024 %          : This segments every skip files, then copies the segments into
0025 %          : the intermediate frames.
0026 % clean_flag : Set this to be true to start from scratch and reseg all the
0027 %            : files regardless of whether any seg files exist. If this
0028 %            : flag is false, use existing segments, if they exist and
0029 %            : new segs if they don't yet exist.
0030 % res       : is a string that is passed to loadConstants(Mine).m to load
0031 %           : the right constants for processing.
0032 % showWarnings : Set to 0 to mute warnings
0033 % startEnd : array of two values to indicate where to start and where to
0034 % stop the program. 1, alignment, 2, segmentation, 3, stripping, 4 linking,
0035 % 5, cell marker, 6 : fluor, 7 : foci, 8, cellA structrues, 9, clist, 10 cell files.
0036 %
0037 % Copyright (C) 2016 Wiggins Lab
0038 % Written by Paul Wiggins & Stella Stylianidou.
0039 % University of Washington, 2016
0040 % This file is part of SuperSegger.
0041 
0042 % SuperSegger is free software: you can redistribute it and/or modify
0043 % it under the terms of the GNU General Public License as published by
0044 % the Free Software Foundation, either version 3 of the License, or
0045 % (at your option) any later version.
0046 %
0047 % SuperSegger is distributed in the hope that it will be useful,
0048 % but WITHOUT ANY WARRANTY; without even the implied warranty of
0049 % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0050 % GNU General Public License for more details.
0051 %
0052 % You should have received a copy of the GNU General Public License
0053 % along with SuperSegger.  If not, see <http://www.gnu.org/licenses/>.
0054 
0055 % Init
0056 
0057 if (nargin < 1) || isempty( dirname_ ) || strcmp(dirname_ ,'.')
0058     dirname_ = pwd;
0059 end
0060 dirname_ = fixDir(dirname_);
0061 
0062 if nargin < 2 || isempty( skip )
0063     skip = 1; % default : don't skip frames
0064 end
0065 
0066 if nargin < 3 || isempty( clean_flag )
0067     clean_flag = 0; % default : don't resegment frames if segmented.
0068 end
0069 
0070 if nargin < 4 || isempty( res )
0071     res = [];
0072 end
0073 
0074 
0075 if ~exist( 'startEnd', 'var' ) || isempty( startEnd )
0076     startEnd = [1 20];
0077 end
0078 
0079 if ~exist( 'showWarnings', 'var' ) || isempty( showWarnings )
0080     showWarnings = 1;
0081 end
0082 
0083 
0084 if ~checkToolboxes
0085     return;
0086 end
0087 
0088 %if you pass a res value, write over CONST values. If it isn't passed,
0089 % use existing values, if they exist. If not, load the default values.
0090 if isstruct(res)
0091     CONST = res;
0092 else
0093     disp (['BatchSuperSeggerOpti : Loading constants file ', res]);
0094     if exist('loadConstantsMine','file');
0095         CONST = loadConstantsMine(res);
0096     else
0097         CONST = loadConstants(res,0);
0098     end
0099 end
0100 
0101 
0102 if clean_flag && showWarnings
0103     try
0104         disp ('Clean flag is set to true.')
0105         answer=input('Do you want to continue, Y/N [Y]:','s');
0106         if lower(answer) ~='y'
0107             disp ('Exiting BatchSuperSegger. Reset clean flag and rerun');
0108             return
0109         end
0110     catch
0111         % can not use input  - in eval mode
0112     end
0113 end
0114 
0115 if startEnd(1) >1 
0116     CONST.align.ALIGN_FLAG = 0;
0117 end
0118 
0119 % align frames
0120 if exist( dirname_, 'dir' )    
0121     if exist( [dirname_,filesep,'raw_im'] ,'dir') && ...
0122             (numel(dir ([dirname_,filesep,'raw_im',filesep,'*.tif'])) || ...
0123             exist([dirname_,filesep,'raw_im',filesep,'cropbox.mat'],'file'))
0124         disp('BatchSuperSeggerOpti : images already aligned');
0125         if exist([dirname_,filesep,'raw_im',filesep,'cropbox.mat'],'file')
0126             tmp = load( [dirname_,filesep,'raw_im',filesep,'cropbox.mat'] );
0127             crop_box_array = tmp.crop_box_array;
0128         else
0129             crop_box_array = cell(1,10000);
0130         end
0131     elseif numel(dir ([dirname_,filesep,'*.tif']))
0132         % check naming convention
0133         if ~isRightNameFormat(dirname_)
0134             disp('Images in incorrect naming format. Using convertImageNames to convert names.')
0135             convertImageNames(dirname_)
0136         end
0137         
0138         mkdir( [dirname_,filesep,'raw_im'] );
0139         if CONST.align.ALIGN_FLAG
0140             crop_box_array = trackOptiAlignPad( dirname_,...
0141                 CONST.parallel.parallel_pool_num, CONST);
0142             movefile( [dirname_,filesep,'*.tif'], [dirname_,filesep,'raw_im'] ) % moves images to raw_im
0143             movefile( [dirname_,'align',filesep,'*.tif'], [dirname_,filesep]); % moves aligned back to main folder
0144             rmdir( [dirname_,'align'] ); % removes _align directory
0145         else
0146             crop_box_array = cell(1,10000);
0147         end
0148     else
0149         error('No images found');
0150     end
0151     
0152 else
0153     error(['BatchSuperSeggerOpti : Can''t find directory ''',dirname_,'''. Exiting.'] );
0154 end
0155 
0156 
0157 % setups the dir structure for analysis.
0158 trackOptiPD(dirname_, CONST);
0159 save( [dirname_,'CONST.mat'],'-STRUCT', 'CONST' ); % Saves CONST set you used.
0160 save( [dirname_,'raw_im',filesep,'cropbox.mat'], 'crop_box_array' );
0161 
0162 % Loop through xy directories
0163 % Reset n values in case directories have already been made.
0164 % setup nxy values
0165 contents = dir([dirname_,'xy*']);
0166 
0167 if isempty(contents)
0168     disp('BSSO: No xy directories were found.');
0169 else
0170     num_dir_tmp = numel(contents);
0171     nxy = [];
0172     num_xy = 0;
0173     
0174     for i = 1:num_dir_tmp
0175         if (contents(i).isdir) && (numel(contents(i).name) > 2)
0176             num_xy = num_xy+1;
0177             nxy = [nxy, str2num(contents(i).name(3:end))];
0178             dirname_list{i} = [dirname_,contents(i).name,filesep];
0179         end
0180     end
0181     
0182     % set values for nc (array of channels (phase and fluorescent))
0183     contents = dir([dirname_list{1},'fluor*']);
0184     num_dir_tmp = numel(contents);
0185     nc = 1;
0186     num_c = 1;
0187     
0188     for i = 1:num_dir_tmp
0189         if (contents(i).isdir) && (numel(contents(i).name) > numel('fluor'))
0190             num_c = num_c+1;
0191             nc = [nc, str2num(contents(i).name(numel('fluor')+1:end))+1];
0192         end
0193     end
0194     
0195     
0196     % Set up parallel loop for each xy point if more than one xy position
0197     % exists. If not more than one xy, we will parallelize inner loops
0198     if (num_xy>1) && (CONST.parallel.parallel_pool_num>0)
0199         workers = CONST.parallel.parallel_pool_num;
0200         CONST.parallel.parallel_pool_num = 0;
0201     else
0202         workers=0;
0203     end
0204     
0205     if workers || ~CONST.parallel.show_status
0206         h = [];
0207     else
0208         h = waitbar( 0, ['Data segmentation xy: 0/',num2str(num_xy)] );
0209         cleanup = onCleanup( @()( delete( h ) ) );
0210     end
0211     
0212     parfor(j = 1:num_xy,workers)
0213         %for j = 1:num_xy
0214         
0215         dirname_xy = dirname_list{j};
0216         intProcessXY( dirname_xy, skip, nc, num_c, clean_flag, ...
0217             CONST, startEnd, crop_box_array{j})
0218         
0219         if workers || ~CONST.parallel.show_status
0220             disp( ['BatchSuperSeggerOpti: No status bar. xy ',num2str(j), ...
0221                 ' of ', num2str(num_xy),'.']);
0222         else
0223             if isvalid(h)
0224                 waitbar( j/num_xy,h,...
0225                     ['Data segmentation xy: ',num2str(j),...
0226                     '/',num2str(num_xy)]);
0227             end
0228         end
0229     end
0230     
0231     if workers % shutting down parallel pool
0232         poolobj = gcp('nocreate');
0233         delete(poolobj);
0234     end
0235     
0236     
0237     if ~workers
0238         close(h);
0239     end
0240     
0241 end
0242 
0243 % done!
0244 end
0245 
0246 function intProcessXY( dirname_xy, skip, nc, num_c, clean_flag, ...
0247     CONST, startEnd, crop_box)
0248 % intProcessXY : the details of running the code in parallel.
0249 % Essentially for parallel processing to work, you have to hand each
0250 % processor all the information it needs to process the images.
0251 
0252 % Initialization
0253 file_filter = '*.tif';
0254 verbose = CONST.parallel.verbose;
0255 
0256 % get header to show xy position
0257 tmp1 = strfind( dirname_xy, 'xy');
0258 tmp2 = strfind( dirname_xy,[filesep]);
0259 
0260 if ~isempty(tmp1) && ~isempty(tmp2)
0261     header = [dirname_xy(tmp1(end):(tmp2(end)-1)),': '];
0262 else
0263     header = [dirname_xy];
0264 end
0265 
0266 % reset nz values
0267 contents=dir([dirname_xy,'phase',filesep,file_filter]);
0268 num_im = numel(contents);
0269 
0270 nz = []; % array of numbers of z frames
0271 nt = []; % array of frame numbers
0272 
0273 for i = 1:num_im;
0274     nameInfo = ReadFileName( contents(i).name );
0275     nt = [nt, nameInfo.npos(1,1)];
0276     nz = [nz, nameInfo.npos(4,1)];
0277 end
0278 
0279 nt = sort(unique(nt));
0280 nz = sort(unique(nz));
0281 
0282 num_t = numel(nt);
0283 num_z = numel(nz);
0284 
0285 if  isempty(nz) || nz(1)==-1 % no z frames
0286     nz = 1;
0287 end
0288 
0289 
0290 disp([header 'BatchSuperSeggerOpti : Segmenting Cells']);
0291 
0292 if (CONST.parallel.parallel_pool_num>0)
0293     workers = CONST.parallel.parallel_pool_num; % number of workers
0294 else
0295     workers=0;
0296 end
0297 
0298 if ~CONST.parallel.show_status
0299     h = [];
0300 else
0301     h = waitbar( 0, ['BatchSuperSeggerOpti : Frame 0/',num2str(num_t)] );
0302     cleanup = onCleanup( @()( delete( h ) ) );
0303 end
0304 
0305 stamp_name = [dirname_xy,'seg',filesep,'.doSegFull'];
0306 
0307 if clean_flag
0308     cleanSuperSegger (dirname_xy, startEnd, skip)
0309 end
0310 
0311 
0312 % does the segmentations for all the frames in parallel
0313 if startEnd(1) <= 2 && startEnd(2) >=2 && ~exist( stamp_name, 'file' )
0314     parfor(i=1:num_t,workers) % through all frames
0315         %for i = 1:num_t
0316         
0317         if isempty( crop_box )
0318             crop_box_tmp = [];
0319         else
0320             crop_box_tmp = crop_box(i,:);
0321         end
0322         
0323         doSeg(i, nameInfo, nc, nz, nt, num_z, num_c, dirname_xy, ...
0324             skip, CONST, [header,'t',num2str(i),': '], crop_box_tmp);
0325         
0326         if ~CONST.parallel.show_status
0327             if verbose
0328                 disp( [header, 'BatchSuperSeggerOpti : Segment. Frame ',num2str(i), ...
0329                     ' of ', num2str(num_t),'.']);
0330             end
0331         else
0332             waitbar( i/num_t, h,...
0333                 ['Data segmentation t: ',num2str(i),'/',num2str(num_t)]);
0334         end
0335     end
0336     time_stamp = clock; %#ok saved below
0337     save( stamp_name, 'time_stamp'); % saves that xydir was full segmented
0338 end
0339 if CONST.parallel.show_status
0340     if isvalid(h)
0341         close(h);
0342     end
0343 end
0344 
0345 % trackOpti has all the rest of things : Linking, Cell files, Fluorescence calculation etc
0346 if startEnd(2) >2
0347     trackOpti(dirname_xy,skip,CONST, header, startEnd);
0348 end
0349 end
0350