massageData

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  function d = massageData(d)
  
  this function should be used to massage any eccentricities we find with
  previous data.  one example is the weird azimuth offset in the encoder
  values in january 2010, or the timing error that was found.  please
  document whatever it is you are fixing.

  sjcm

%%%%%%%%%%
 Edit 1:
 Fixed it to not crash when only temperature sensor data is read in
 ogk
%%%%%%%%%%
 Edit 2:
 Fixed it to not crash when only frame data is read in
 ogk
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%
 Edit 3:
 Fixed it so that if Tracker Thread Timing Issue is really bad (i.e
 21-Jul-2010:13:02:25 Time End: 21-Jul-2010:15:18:08) then the process
 errors out
 CJC 23/8/2010
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

0001 function d = massageData(d)
0002 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
0003 %  function d = massageData(d)
0004 %
0005 %  this function should be used to massage any eccentricities we find with
0006 %  previous data.  one example is the weird azimuth offset in the encoder
0007 %  values in january 2010, or the timing error that was found.  please
0008 %  document whatever it is you are fixing.
0009 %
0010 %  sjcm
0011 %
0012 %%%%%%%%%%%
0013 % Edit 1:
0014 % Fixed it to not crash when only temperature sensor data is read in
0015 % ogk
0016 %%%%%%%%%%%
0017 % Edit 2:
0018 % Fixed it to not crash when only frame data is read in
0019 % ogk
0020 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
0021 %%%%%%%%%%%
0022 % Edit 3:
0023 % Fixed it so that if Tracker Thread Timing Issue is really bad (i.e
0024 % 21-Jul-2010:13:02:25 Time End: 21-Jul-2010:15:18:08) then the process
0025 % errors out
0026 % CJC 23/8/2010
0027 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
0028 
0029 if(~isfield(d, 'array'))
0030   return;
0031 end
0032 
0033 %%%%%%%%%%%%%%%%
0034 % Added by ogk in edit 1
0035 if(~isfield(d.array, 'frame'))
0036   return;
0037 end
0038 %%%%%%%%%%%%%%%%
0039 
0040 
0041 % timing fix for times before feb , 2010
0042 if(d.array.frame.utc(1)<55230)
0043   if(isfield(d, 'antenna0'));
0044     if(isfield(d.antenna0, 'receiver'))
0045       d.antenna0.receiver.utc = ...
0046       d.antenna0.receiver.utc-0.3/24/60/60;
0047     end
0048     if(isfield(d.antenna0, 'servo'))
0049       d.antenna0.servo.utc = ...
0050       d.antenna0.servo.utc+0.5/24/60/60;
0051     end
0052   end
0053 elseif(d.array.frame.utc(1)>55230 & d.array.frame.utc(1)<55244)
0054   if(isfield(d, 'antenna0'));  
0055     if(isfield(d.antenna0, 'servo'))
0056       d.antenna0.servo.utc = d.antenna0.servo.utc+0.5/24/60/60;
0057     end
0058   end
0059 end
0060 
0061 
0062 % fix for az position offset from Jan 22 to Jan 27, 2010
0063 if(d.array.frame.utc(1)>55218 & d.array.frame.utc(1)<55223)
0064   if(isfield(d, 'antenna0'))
0065     if(isfield(d.antenna0, 'servo'))
0066       if(isfield(d.antenna0.servo, 'fast_az_pos'))
0067     d.antenna0.servo.fast_az_pos = ...
0068         d.antenna0.servo.fast_az_pos-22.42;
0069     d.antenna0.servo.slow_az_pos = ...
0070         d.antenna0.servo.slow_az_pos-22.42;
0071       end
0072     end
0073   end
0074 end
0075 
0076 % OGK edit 2: Added conditional nature of the call to fixTimingOrder
0077 % let's fix the order of the timing.
0078 if(isfield(d, 'antenna0'))
0079   if(isfield(d.antenna0, 'servo') && isfield(d.antenna0, 'receiver') && ...
0080     isfield(d.antenna0, 'thermal'))
0081         d = fixTimingOrder(d);
0082   end
0083 end
0084 % next we check that no times are doubled.
0085 
0086 % this one's weird:  the clock for the tracker thread sometimes doesn't
0087 % update -- it not only causes the problems with the control system that
0088 % matthew saw at the beginning of july, but it also makes the data have
0089 % issues being processed.  This fix only takes care of the data being
0090 % processed.
0091 
0092 if(isfield(d, 'antenna0'))
0093   if(isfield(d.antenna0, 'tracker'))
0094     dt = deriv(d.antenna0.tracker.utc)*24*60*60;
0095     ind = dt>1.1 | dt<0.9;
0096     if(~isempty(find(ind)))
0097       l = length(find(ind));
0098       display('Tracker Thread Timing Issue');
0099       display(sprintf('Issue affected %d out of %d data samples', l, length(ind)));
0100       display('Fixing data for postprocessing');
0101       d.antenna0.tracker.utc(ind) = d.array.frame.utc(ind) + 1/24/60/60;
0102       display('Fixed data for postprocessing');
0103       %STUPID THING TO ADD
0104 %      if(l/length(ind)>0.01)%added by CJC for really bad data 23/8/2010
0105 %          error('massageData:Chk','Error massaging the data')%added by CJC for really bad data 23/8/2010
0106 %      end
0107     end
0108   end
0109 end
0110 
0111 if(d.array.frame.utc > date2mjd(2012, 12, 06) & d.array.frame.utc < ...
0112       date2mjd(2013, 01, 22))
0113   % problem where the survey sched 0.3 didn't set the flag properly.
0114   speed = abs(deriv(d.antenna0.servo.slow_az_pos));
0115   ind = speed > 3.75 & speed < 4.25;
0116   if(~isempty(ind))
0117     % those are the ones during the scan, if we want to maintain the stuff in
0118     % between the scans, we do the following:
0119     [si ei] = findStartStop(~ind);
0120     aa  = find(ei - si < 10);
0121     for m=1:length(aa)
0122       ind(si(aa(m)):ei(aa(m))) = 1;
0123     end
0124     d.array.frame.features(ind) = d.array.frame.features(ind) + 2^0;
0125   end
0126 end
0127 
0128 
0129 return;
0130 
0131 
0132 function dfinal = fixTimingOrder(d)
0133 % basically the servo and the receiver data are from two seconds ago,
0134 % cryocon is from 1 second ago, and the data frame is current time
0135 
0136 if(isfield(d, 'antenna0'))
0137   if(~isfield(d.antenna0, 'servo') | ~isfield(d.antenna0, 'receiver') | ...
0138     ~isfield(d.antenna0, 'thermal'))
0139     dfinal = d;
0140     return;
0141   end
0142 end
0143 
0144 if(~isfield(d.antenna0.receiver, 'data'))
0145   dfinal = d;
0146   return;
0147 end
0148 
0149 
0150 dservo = d.antenna0.servo;
0151 dreceiver = d.antenna0.receiver;
0152 dthermal = d.antenna0.thermal;
0153 lengthOrig = size(d.array.frame.utc);
0154 
0155 % now from dfinal we cut out the last frame and from everything else we cut
0156 % out the first frame.
0157 indFinal = ones(lengthOrig);
0158 indFinal(length(indFinal)-1:length(indFinal)) = 0;
0159 
0160 indFinal = ones(lengthOrig);
0161 indFinal(1:2) = 0;
0162 dservo = framecutSub(dservo, logical(indFinal));
0163 dreceiver = framecutSub(dreceiver, logical(indFinal));
0164 
0165 indFinal = ones(lengthOrig);
0166 indFinal([1 length(indFinal)]) = 0;
0167 dthermal = framecutSub(dthermal, logical(indFinal));
0168 
0169 dfinal = framecutSub(d, logical(indFinal));
0170 clear d;
0171 dfinal.antenna0.receiver = dreceiver;
0172 dfinal.antenna0.servo.utc = dservo.utc;
0173 dfinal.antenna0.servo.fast_az_pos = dservo.fast_az_pos;
0174 dfinal.antenna0.servo.fast_el_pos = dservo.fast_el_pos;
0175 dfinal.antenna0.servo.fast_az_err = dservo.fast_az_err;
0176 dfinal.antenna0.servo.fast_el_err = dservo.fast_el_err;
0177 dfinal.antenna0.thermal.utc       = dthermal.utc;
0178 dfinal.antenna0.thermal.ccTemperatureLoad = dthermal.ccTemperatureLoad;
0179 
0180 % now all the data should be on the same timescale, which should make things
0181 % easier to cut/etc.
0182 
0183 return;
0184               
0185               
0186 
0187 function d = framecutSub(d, ind)
0188 
0189 names=fieldnames(d);
0190 for i=1:length(names)
0191   if(eval(sprintf('isstruct(d.%s)',names{i})))
0192     eval(sprintf('d.%s=framecutSub(d.%s,ind);',names{i},names{i}));
0193   else
0194     eval(sprintf('d.%s=regcut(d.%s,ind);',names{i},names{i}));            
0195   end
0196 end
0197 
0198 return
0199 
0200 %%%%%%%%%%%%%%%%%%%%%%%%
0201 function r=regcut(r,ind)
0202 
0203 switch(ndims(r))
0204   case 1
0205     r=r(ind);
0206   case 2
0207     r=r(ind,:);
0208   case 3
0209     r=r(ind,:,:);
0210   case 4
0211     r=r(ind,:,:,:);
0212 end
0213 
0214 return