day_reduce_gs

 Example usage 

 This will reduce a day, calculate the ground template, then apply the ground templates, using load and save


 The redscripts will be chosen appropriately within this script.

0001 function day_reduce_gs(date,output_stub,sched_to_reduce)
0002 % Example usage
0003 %
0004 % This will reduce a day, calculate the ground template, then apply the ground templates, using load and save
0005 %
0006 %
0007 % The redscripts will be chosen appropriately within this script.
0008 %
0009 
0010 
0011 % day_reduce('02-Nov-2010','_jamie')
0012 % Will find all obs for that day and reduce them one by one.
0013 % will save to the concat of date and output_stub e.g. '02-Nov-2010_jamie'
0014 %
0015 % Use the optional path_to_redscript to specify which redscript you want
0016 % Use the optional sched_to_reduce to specify only specific schedule names
0017 % to be reduced.
0018 %
0019 
0020 
0021 
0022 % JL 24 Oct 2013 - I changed the behaviour of optional parameter sched_to_reduce. It will now
0023 % reduce all observations with have a schedule name which has sched_to_reduce as a substring.
0024 % E.g. to reduce all the observations which have schedules with the substring "cal" in them
0025 % (i.e the calibrator observations), you can do
0026 %
0027 % day_reduce('02-Nov-2010','_jamie','/reduc/redScript.red','cal')
0028 %
0029 % this would reduce shedules like scan_calibrators_2013.sch, but not
0030 % e.g. cbass_survey_5speed.sch.
0031 %
0032 % You can also invert this behaviour by prepending a ~. E.g. to reduce all the schedules which
0033 % do *not* have "cal" as a subsring you can do
0034 %
0035 % day_reduce('02-Nov-2010','_jamie','/reduc/redScript.red','~cal')
0036 %
0037 %
0038 % ACT 4th Jan 2012 - fixed a bug whereby if the observation needed to be
0039 % split into chunks, anything starting/ending at 00:00:00 hours crashed.
0040 % ACT/JL 3rd Jan - new paramater max_doable_obs_len added to allow whatever
0041 % length obs you fancy through before chopping up.
0042 %
0043 % MAS 23-Feb-2012 - added a parameter "runhealth" to allow antenna health
0044 % to be run optionally.  Run antenna health by default.
0045 %
0046 % SJCM 09-aug-2012:  bug was present when splitting into chunks.  was
0047 % interpreting the number of hours as teh number of chunks needed to reduce.
0048 % also added a final_dir for transferring all the reduced files to a
0049 % specific location
0050 
0051 if(nargin < 6)
0052   final_dir = [];
0053 end
0054 
0055 
0056 max_doable_obs_len=360;
0057 
0058 % This redscript using teh setension NoGalNo42
0059 % reduceData traps this case on mapping and cuts the galaxy out before
0060 % making the maps
0061 % This routine also saves out the d structures for each structure
0062 % such that they can be read in later.
0063 
0064 % THIS IS WHERE YOU HAVE TO SET YOUR REDSCRIPTS
0065 path_to_redscript_prep='reduc/redScript_prep_and_save_gs.red';
0066 % THE NO LOAD STEP VERSION
0067 %path_to_redscript_prep='reduc/redScript_prep_and_save_gs_no_load.red';
0068 path_to_redscript_apply='reduc/redScript_load_and_apply_gs.red';
0069 
0070 if(~exist('runhealth','var'))
0071   runhealth=1;
0072 end
0073 
0074 [home,installeddir]=where_am_i();
0075 location = [home,'/',installeddir];
0076 
0077 date_number=datenum(date);
0078 num_date = datestr(date_number, 'yyyymmdd');
0079 
0080 antenna_health_dir = ['reduc/',num_date,output_stub,'_gs'];
0081 
0082 outputdir = [num_date,output_stub];
0083 errorfile = [num_date,output_stub,'_errors.txt'];
0084 errorfilepath = [location,'/','reduc/',outputdir,'/',errorfile];
0085 mkdir([location,'/','reduc/',outputdir]);
0086 disp(['day_reduce_gs:: Writing errors to ',errorfilepath]);
0087 errfile = fopen(errorfilepath,'w');
0088 
0089 file = [location,'/','log/obs_log.html'];
0090 
0091 [start_dates,end_dates,schedule] =get_days_obs(file,date);
0092 
0093 disp(['day_reduce_gs:: FOUND ',int2str(length(start_dates)),' observations...']);
0094 
0095 for i=1:length(start_dates)
0096 % just do first five as a test
0097 % for i=1:10
0098    
0099     sched_bits = regexp(char(schedule(i)), '/', 'split');
0100     sched_name = char(sched_bits(length(sched_bits)));
0101     % strip out possible open bracket if there is one.
0102     sched_name = strrep(sched_name, '(','');
0103     disp(['day_reduce_gs:: Schedule is ',sched_name]);
0104     
0105     % Check for error state.
0106     if strcmp(char(start_dates(i)),'Error')
0107         continue;
0108     end; %endif
0109  
0110 
0111     
0112     if (exist('sched_to_reduce','var'))
0113       negate = 0;
0114       stripped_sched_to_reduce=sched_to_reduce;
0115       if (~isempty(regexp(sched_to_reduce,'^~'))) 
0116       % if we have a tilde up front we should prepare to negate and remove
0117       % the tildee in prep. for future match
0118       negate = 1;
0119       stripped_sched_to_reduce = regexprep(sched_to_reduce,'^~',''); 
0120       end
0121       does_sched_match = ~isempty(regexp(sched_name,stripped_sched_to_reduce,'match'));       
0122       xor_exp = ~xor(does_sched_match,negate);
0123 
0124       if (xor_exp) % This is a case sensitive regex style match
0125     disp(['day_reduce_gs:: SKIPPING this observation ', sched_name,' - does not match the requested schedule substring, namely ''',sched_to_reduce,'''']);   
0126             continue;
0127         end;
0128     end;
0129     
0130     start_string =  remove_colon_datestring(char(start_dates(i)));
0131     end_string =  remove_colon_datestring(char(end_dates(i)));
0132     start_num = datenum(start_string);
0133     end_num = datenum(end_string);
0134     
0135     obs_length_mins = (end_num-start_num)*24*60;
0136     %disp(['Date num start= ',int2str(start_num), '  date num end= ',int2str(end_num),' Time in mins = ',int2str(obs_length_mins)]);
0137     
0138     if (obs_length_mins<max_doable_obs_len)
0139         
0140         
0141         try
0142             disp(['day_reduce_gs:: READING Obs No. ', int2str(i),' of ',int2str(length(start_dates)),' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))]);
0143             d=read_arc(char(start_dates(i)),char(end_dates(i)));
0144             
0145         catch exception
0146             error_string= ['day_reduce_gs:: FAILED TO READ ', int2str(i),' of ',int2str(length(start_dates)), ' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))];
0147             disp(error_string);
0148             rep = getReport(exception, 'basic');
0149             fprintf(errfile,'%s\n',error_string);
0150             fprintf(errfile,'%s\n',rep);
0151             continue;
0152         end; %endtry
0153         
0154         try
0155             disp(['day_reduce_gs:: REDUCING Obs No. ', int2str(i),' of ',int2str(length(start_dates)),' -> ',char(start_dates(i)),' --- ',char(end_dates(i))]);
0156             dcal2 = reduceData(d, path_to_redscript_prep, 0, outputdir,sched_name);
0157         catch exception
0158             error_string = ['day_reduce_gs:: FAILED TO REDUCE ',int2str(i),' of ',int2str(length(start_dates)), ' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))];
0159             disp(error_string);
0160             rep = getReport(exception, 'basic');
0161             fprintf(errfile,'%s\n',error_string);
0162             fprintf(errfile,'%s\n',rep);
0163             continue;
0164         end;  %endtry
0165         
0166     else
0167         %disp(['day_reduce_gs:: Detected observation longer than 3 hours...']);
0168         % observation greater than 3hrs must split into chunks
0169         
0170         start_num_mins =  start_num*24*60;
0171         end_num_mins =  end_num*24*60;
0172         
0173         no_chunks_reqd =  ceil( obs_length_mins/max_doable_obs_len);
0174 
0175     
0176         disp(['day_reduce_gs:: Obs longer than 360 mins will split obs from ',datestr(start_num),' to ', datestr(end_num), ' into ',int2str(no_chunks_reqd),' sub-observations']);
0177         for j=1:no_chunks_reqd
0178       if (j== no_chunks_reqd)
0179                 new_start_mins= start_num_mins + (j-1)*max_doable_obs_len;
0180                 new_start_days= new_start_mins/(24*60);
0181                 new_start_string = datestr(new_start_days,0);
0182                 new_start_string = insert_colon_datestring(new_start_string);
0183                 new_end_string = char(end_dates(i));
0184             else
0185                 new_start_mins= start_num_mins + (j-1)*max_doable_obs_len;
0186                 new_start_days= new_start_mins/(24*60);
0187                 new_start_string = datestr(new_start_days,0);
0188                 new_end_mins= start_num_mins + (j)*max_doable_obs_len;
0189                 new_end_days= new_end_mins/(24*60);
0190                 %keyboard;
0191                 new_end_string = datestr(new_end_days);
0192                 whos
0193                 % Re-insert semicolon into date string.
0194                 
0195                 new_start_string = insert_colon_datestring(new_start_string) ;
0196                 new_end_string = insert_colon_datestring(new_end_string) ;
0197                 
0198             end
0199             
0200             
0201             
0202             try
0203                 disp(['day_reduce_gs:: READING Obs No. ', int2str(i),', chunk ',int2str(j),' of ',int2str(no_chunks_reqd),'   ->  ',...
0204                     new_start_string, ' --- ',new_end_string]);
0205                 d=read_arc(new_start_string,new_end_string);
0206                 
0207             catch exception
0208                 error_string= ['day_reduce_gs:: FAILED TO READ ',int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0209                     new_start_string, ' --- ',new_end_string];
0210                 
0211                 disp(error_string);
0212                 rep = getReport(exception, 'basic');
0213                 fprintf(errfile,'%s\n',error_string);
0214                 fprintf(errfile,'%s\n',rep);
0215                 continue;
0216             end; %endtry
0217             
0218             try
0219                 disp(['day_reduce_gs:: REDUCING Obs No. ', int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0220                     new_start_string, ' --- ',new_end_string]);
0221                  dcal2 = reduceData(d, path_to_redscript_prep, 0, outputdir,sched_name);
0222             catch exception
0223                 error_string = ['day_reduce_gs:: FAILED TO REDUCE ',int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0224                     new_start_string, ' --- ',new_end_string];
0225                 
0226                 disp(error_string);
0227                 rep = getReport(exception, 'basic');
0228                 fprintf(errfile,'%s\n',error_string);
0229                 fprintf(errfile,'%s\n',rep);
0230                 continue;
0231             end; %endtry
0232             
0233         end; %endfor no_chunk_reqd
0234         
0235     end; % end if > 3 hrs
0236 end; % end main for
0237 
0238 disp( ['day_reduce_gs: FINISHED ',date,' output dir is ',outputdir] );
0239 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
0240 % At this point we have reduced a whole day
0241 % and made fits files with the galaxy cut out
0242 % so we need to run the function to calculate the ground templates
0243 
0244 % First we need to make a files.txt file which contains all
0245 % of the non-map fits files created in the step above.
0246 
0247 % MATLAB EQUIVALENT OF
0248 % my  @matched_daily_fits  = grep { !/map/ } `find . -name "*.fits"`;
0249 % and then write out to files.txt
0250 % or we could infer these names from the start_dates array.
0251 
0252 disp('day_reduce_gs: !!!!!! Creating files.txt' );
0253 files_file_filename=[location,'/','reduc/',outputdir,'/','files.txt'];
0254 files_file = fopen(files_file_filename,'w');
0255 
0256 top_level_dir = [location,'/','reduc/',outputdir];
0257 current_dir=pwd;
0258 cd(top_level_dir);
0259 [status,cmdout] = unix(['find . -name "*.fits" |grep -v map']);
0260 cd(current_dir);
0261 fprintf(files_file,'%s\n',cmdout);
0262 fclose(files_file);
0263 
0264 %for i=1:length(start_dates)
0265 %for i=1:10
0266 %char(start_dates(i))
0267 
0268 % This prints out filenames in the style of
0269 %./18-Oct-2013:00:10:21/fits/18-Oct-2013:00:10:21.fits
0270 % into a files.txt file.
0271 
0272 %fprintf(files_file,'%s\n',['./',char(start_dates(i)),'/fits/',char(start_dates(i)),'.fits']);
0273 %end
0274 
0275 %fclose(files_file);
0276 
0277 disp('day_reduce_gs: !!!!! Making ground templates...' );
0278 
0279 cd([location,'/']);
0280 % Get a numerical list of the observed elevations for this particular day.
0281 el_list = get_days_observed_els('log/obs_log.html',date)
0282 % TODO INSERT a trap to see if this el_list is returned empty, in
0283 % which case no point in attempting to do the ground removals.
0284 
0285 %el_list = [37,47]
0286 
0287 
0288 % name is set tpo the date here so will get
0289 cd(top_level_dir);
0290 
0291 MakeGroundTemplate(files_file_filename, 1, date, 32,el_list);
0292 
0293 cd(current_dir);
0294 
0295 disp('day_reduce_gs: ... !!!!! done making ground templates.' );
0296 
0297 %% TO DO - Need to transmit the date string which acts as the
0298 %% template removal ground stub into
0299 
0300 % Now need to make a fresh directory for the rerun
0301 outputdir = [num_date,output_stub,'_gs'];
0302 errorfilepath = [location,'/','reduc/',outputdir,'/',errorfile];
0303 mkdir([location,'/','reduc/',outputdir]);
0304 
0305 disp(['day_reduce_gs: ... !!!!! Preparing to rereduce in directory',outputdir]);
0306 
0307 
0308 
0309 
0310 for i=1:length(start_dates)
0311 % for i=1:10
0312     sched_bits = regexp(char(schedule(i)), '/', 'split');
0313     sched_name = char(sched_bits(length(sched_bits)));
0314     % strip out possible open bracket if there is one.
0315     sched_name = strrep(sched_name, '(','');
0316     disp(['day_reduce_gs: Schedule is ',sched_name]);
0317     
0318     % Check for error state.
0319     if strcmp(char(start_dates(i)),'Error')
0320         continue;
0321     end; %endif
0322      
0323     if (exist('sched_to_reduce','var'))
0324       negate = 0;
0325       stripped_sched_to_reduce=sched_to_reduce;
0326       if (~isempty(regexp(sched_to_reduce,'^~'))) 
0327       % if we have a tilde up front we should prepare to negate and remove
0328       % the tildee in prep. for future match
0329       negate = 1;
0330       stripped_sched_to_reduce = regexprep(sched_to_reduce,'^~',''); 
0331       end
0332       does_sched_match = ~isempty(regexp(sched_name,stripped_sched_to_reduce,'match'));       
0333       xor_exp = ~xor(does_sched_match,negate);
0334 
0335       if (xor_exp) % This is a case sensitive regex style match
0336     disp(['day_reduce_gs:: SKIPPING this observation ', sched_name,' - does not match the requested schedule substring, namely ''',sched_to_reduce,'''']);   
0337             continue;
0338         end;
0339     end;
0340     
0341     start_string =  remove_colon_datestring(char(start_dates(i)));
0342     end_string =  remove_colon_datestring(char(end_dates(i)));
0343     start_num = datenum(start_string);
0344     end_num = datenum(end_string);
0345     
0346     obs_length_mins = (end_num-start_num)*24*60;
0347     %disp(['Date num start= ',int2str(start_num), '  date num end= ',int2str(end_num),' Time in mins = ',int2str(obs_length_mins)]);
0348     
0349     if (obs_length_mins<max_doable_obs_len)
0350         
0351         
0352         try
0353             disp(['day_reduce_gs:: READING Obs No. ', int2str(i),' of ',int2str(length(start_dates)),' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))]);
0354             d=read_arc(char(start_dates(i)),char(end_dates(i)));
0355             
0356         catch exception
0357             error_string= ['day_reduce_gs:: FAILED TO READ ', int2str(i),' of ',int2str(length(start_dates)), ' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))];
0358             disp(error_string);
0359             rep = getReport(exception, 'basic');
0360             fprintf(errfile,'%s\n',error_string);
0361             fprintf(errfile,'%s\n',rep);
0362             continue;
0363         end; %endtry
0364         
0365         try
0366             disp(['day_reduce_gs:: REDUCING Obs No. ', int2str(i),' of ',int2str(length(start_dates)),' -> ',char(start_dates(i)),' --- ',char(end_dates(i))]);
0367             dcal2 = reduceData(d, path_to_redscript_apply, 0, outputdir,sched_name);
0368         catch exception
0369             error_string = ['day_reduce_gs:: FAILED TO REDUCE ',int2str(i),' of ',int2str(length(start_dates)), ' ->  ',char(start_dates(i)),' --- ',char(end_dates(i))];
0370             disp(error_string);
0371             rep = getReport(exception, 'basic');
0372             fprintf(errfile,'%s\n',error_string);
0373             fprintf(errfile,'%s\n',rep);
0374             continue;
0375         end;  %endtry
0376         
0377     else
0378         %disp(['day_reduce_gs:: Detected observation longer than 3 hours...']);
0379         % observation greater than 3hrs must split into chunks
0380         
0381         start_num_mins =  start_num*24*60;
0382         end_num_mins =  end_num*24*60;
0383         
0384         no_chunks_reqd =  ceil( obs_length_mins/max_doable_obs_len);
0385 
0386     
0387         disp(['day_reduce_gs:: Obs longer than 360 mins will split obs from ',datestr(start_num),' to ', datestr(end_num), ' into ',int2str(no_chunks_reqd),' sub-observations']);
0388         for j=1:no_chunks_reqd
0389       if (j== no_chunks_reqd)
0390                 new_start_mins= start_num_mins + (j-1)*max_doable_obs_len;
0391                 new_start_days= new_start_mins/(24*60);
0392                 new_start_string = datestr(new_start_days,0);
0393                 new_start_string = insert_colon_datestring(new_start_string);
0394                 new_end_string = char(end_dates(i));
0395             else
0396                 new_start_mins= start_num_mins + (j-1)*max_doable_obs_len;
0397                 new_start_days= new_start_mins/(24*60);
0398                 new_start_string = datestr(new_start_days,0);
0399                 new_end_mins= start_num_mins + (j)*max_doable_obs_len;
0400                 new_end_days= new_end_mins/(24*60);
0401                 %keyboard;
0402                 new_end_string = datestr(new_end_days);
0403                 whos
0404                 % Re-insert semicolon into date string.
0405                 
0406                 new_start_string = insert_colon_datestring(new_start_string) ;
0407                 new_end_string = insert_colon_datestring(new_end_string) ;
0408                 
0409             end
0410             
0411             
0412             
0413             try
0414                 disp(['day_reduce_gs:: READING Obs No. ', int2str(i),', chunk ',int2str(j),' of ',int2str(no_chunks_reqd),'   ->  ',...
0415                     new_start_string, ' --- ',new_end_string]);
0416                 d=read_arc(new_start_string,new_end_string);
0417                 
0418             catch exception
0419                 error_string= ['day_reduce_gs:: FAILED TO READ ',int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0420                     new_start_string, ' --- ',new_end_string];
0421                 
0422                 disp(error_string);
0423                 rep = getReport(exception, 'basic');
0424                 fprintf(errfile,'%s\n',error_string);
0425                 fprintf(errfile,'%s\n',rep);
0426                 continue;
0427             end; %endtry
0428             
0429             try
0430                 disp(['day_reduce_gs:: REDUCING Obs No. ', int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0431                     new_start_string, ' --- ',new_end_string]);
0432                 dcal2 = reduceData(d, path_to_redscript_apply, 0, outputdir,sched_name);
0433             catch exception
0434                 error_string = ['day_reduce_gs:: FAILED TO REDUCE ',int2str(i),':',int2str(j), '  of ',int2str(length(start_dates)),':',int2str(no_chunks_reqd),'   ->  ',...
0435                     new_start_string, ' --- ',new_end_string];
0436                 
0437                 disp(error_string);
0438                 rep = getReport(exception, 'basic');
0439                 fprintf(errfile,'%s\n',error_string);
0440                 fprintf(errfile,'%s\n',rep);
0441                 continue;
0442             end; %endtry
0443             
0444         end; %endfor no_chunk_reqd
0445         
0446     end; % end if > 3 hrs
0447 end; % end main for
0448 
0449 
0450 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
0451 % Now try running the antenna health script
0452 
0453 %runhealth=0;
0454 
0455 if runhealth ~= 0
0456     try
0457         disp( ['day_reduce_gs:: Attempting to run antenna health script : location',antenna_health_dir]);
0458         antHealth(date,antenna_health_dir);
0459     catch exception
0460         error_string = ['day_reduce_gs:: FAILED to run antenna health script : location',antenna_health_dir];
0461         disp(error_string);
0462         rep = getReport(exception, 'basic');
0463         fprintf(errfile,'%s\n',error_string);
0464         fprintf(errfile,'%s\n',rep);
0465 
0466     end; %endtry
0467 else
0468     disp('day_reduce_gs:: NOT attempting to run antenna health script.');
0469 end
0470 
0471 
0472 fclose(errfile);
0473 
0474 if(~isempty(final_dir))
0475   txt = sprintf('mv reduc/%s/ %s/', outputdir, final_dir);
0476   unix(txt);
0477 end
0478 
0479 disp('day_reduce_gs:: FINISHED rereduction with ground removal');
0480 
0481 return;
0482 
0483 
0484 
0485 function processed_string = remove_colon_datestring(date_string)
0486 splitline = regexp(char(date_string), ':', 'split');
0487 processed_string=[char(splitline(1)),' ',char(splitline(2)),':',char(splitline(3)),':',char(splitline(4))];
0488 
0489 function processed_string = insert_colon_datestring(date_string)
0490 splitline = regexp(char(date_string), ' ', 'split');
0491 processed_string=[char(splitline(1)),':',char(splitline(2))];