function PROC_StreetScenesDBToNewFeature(FunctionName, NewLabel, options);
%function PROC_StreetScenesDBToNewFeature
%
% takes the first 4 groups of data from the street scenes database, and the name of a function
% which takes a grayscale image (matrix), the image size, and returns a 
% struct array of feature vectors (matricies).
%  Built in Parallelsim
%
% cmd = sprintf('saX2 = %s(X1,options)',FuncitonName)
% saX2 is a struct array of all matricies to be recorded (if multiple different features are recorded)
%
 
 d.HelpTextInfo = GetHelpTextInfo;
 d.ResetParallel = 0;
 d.bOverwrite = 1;
 d.bMandateSourceIndex = 0;
 d.SourceTemplate = 'AppearanceData_128x128_%s_f%.3d.mat';
 d.SourceRoot = '/cbcl/scratch01/bileschi/PrecomputedFeatures/StreetScenes/AppearanceDetector';
 d.Sourcevariablename = 'grayX';
 d.bSplitTo50s = 0;
 if(nargin < 3), options = [];, end;
 options = ResolveMissingOptions(options,d);
 
 here = pwd;
 Root = options.SourceRoot;
 cd(Root);
 Template = options.SourceTemplate;
 NewTemplate = [NewLabel,'_%s_f%.3d.mat'];
 variablename = options.Sourcevariablename;
 Parallelizor = sprintf('/cbcl/scratch01/bileschi/Parallel_PROC_SSDBTNF_%s_%s.mat',FunctionName,NewLabel);
 if(not(options.bMandateSourceIndex))
   if((not(exist(Parallelizor))) || (options.ResetParallel))
     PROCEDURE_ResetMyIdx(Parallelizor);
   end
   myIdx = PROCEDURE_IncrementMyIdx(Parallelizor);
   while(isempty(intersect(myIdx,[1,2,3,4,11,12,13,14,19,20])))
     if(myIdx > 20), break, end
     myIdx = PROCEDURE_IncrementMyIdx(Parallelizor);
   end
   sourceIdx = myIdx;
 else
   sourceIdx = options.bMandateSourceIndex;
   myIdx = sourceIdx;
 end
 objidx = 1;
 if(myIdx > 10)
  objidx = 2;
  sourceIdx = sourceIdx - 10;
 end
 if(myIdx > 18)
  objidx = 3;
  sourceIdx = sourceIdx - 8;
 end
 
 objname{1} = 'car';
 objname{2} = 'pedestrian';
 objname{3} = 'bicycle';
 on = objname{objidx};
 if(myIdx > 20)
  fprintf('job completely allocated \n');
 else
  savename = fullfile(Root,sprintf(NewTemplate, on,sourceIdx));
  fprintf('FunctionName %s ProcessNumber %d Object %d sourceIdx %d starting...\n', FunctionName, myIdx, objidx, sourceIdx);
  if(not(options.bOverwrite))
    if(exist(savename))
      fprintf('%s already exists.  No overwrite.\n',savename);
      return;
    end
  end
  [y, NewDataX] = GetNewDataX(Template, variablename,sourceIdx,on,Root);
  if(options.bSplitTo50s)
     ally = y;
     bAllAreComplete = 1;
     maxnparts = ceil(size(NewDataX,2)/50);
     % cut the data into parts and save the processed parts
     seedrand
     for i = randperm(maxnparts)
       PartSavename = [fullfile(Root,sprintf(NewTemplate, on,sourceIdx)),sprintf('part_%.2dof.%2d.mat',i,maxnparts)];
       if(exist(savename))
         %% This is in case the program has been finished by another cpu already.
	 try
	   load(savename);
	   fprintf('%s has been completed by another cpu.  No overwrite.\n',savename);
           return;
	 catch
	   pause(60);
	 end
	 try
	   fprintf('Second try... \n');
	   load(savename);
           return;
	 catch
	   delete(savename)
	   fprintf('\n');
	   fprintf('%s has been completed by another cpu but WAS CORRUPT.\n',savename);
	   fprintf('file deleted... Im continuing...\n');
	 end
       end
       fprintf('Building Random Fraction %d of %d...\n',i,maxnparts);
       if(exist(PartSavename))
         fprintf('ALREADY EXISTS\n');
	 continue;
       else
         Which50 = intersect(1:size(NewDataX,2),(1:50) + (i-1)*50);
         NewDataXPart = NewDataX(:,Which50);
	 y = ally(Which50);
         cmd = sprintf('saNewMats = %s(NewDataXPart,options);',FunctionName);
         eval(cmd);
	 fns = fieldnames(saNewMats);
         nfns = length(fns);
         savecmd = 'save(PartSavename,''y''';
         for j = 1:nfns
           cmd = sprintf('%s = saNewMats.%s;',fns{j},fns{j});
           eval(cmd);
           savecmd = [savecmd,sprintf(',''%s''',fns{j})];
	  end
          savecmd = [savecmd,');'];
	  fprintf('saving %s... ',PartSavename);
          ntriesleft = 5;
	  while(ntriesleft > 0)
	    ntriesleft = ntriesleft -1;
  	    try
   	       eval(savecmd);
	       fprintf('success\n');
	       break;
            catch
	       pause(ceil(rand*30))
	       fprintf('ERROR, trying again\n');
	       continue;
	    end
            fprintf('Out of save attempts.  Perhaps Disk is full?\n');
	  end
       end
     end
     % rebuild the data and save the whole
     for i = 1:maxnparts
       PartSavename = [fullfile(Root,sprintf(NewTemplate, on,sourceIdx)),sprintf('part_%.2dof.%2d.mat',i,maxnparts)];
       try
          N = load(PartSavename);
       catch
          fprintf('THere has been an error loading the part file\n');
	  fprintf('%s\n',PartSavename);
	  fprintf('for consolidation\n');
	  fprintf('it is possible that there was a disk error\n')
	  fprintf('alternatively, two consolidation processes were working in parallel and this one\n')
	  fprintf('was left behind\n\n');
	  if(exist(savename))
  	     fprintf('Returning without error, the full savefile seems to exist\n');
	     return;
	  else
	     fprintf('Pausing 180 seconds...');
	     pause(180);        
	     fprintf('After 180 seconds, the file still does not exist, there has likly been an error\n');
	     continue;
	  end
       end
       fns = fieldnames(N);
       nfns = length(fns);
       for j = 1:nfns
         if(strcmpi(fns{j},'options'))
	   continue;
	 end
	 if i==1
	   cmd = sprintf('%s = [];',fns{j});;
	   eval(cmd);
	 end
         cmd = sprintf('%s = [%s,N.%s];',fns{j},fns{j},fns{j});
         eval(cmd);
       end
     end
     savecmd = 'save(savename,''y'',''options''';
     for j = 1:nfns
       savecmd = [savecmd,sprintf(',''%s''',fns{j})];
     end  
     savecmd = [savecmd,');'];
     fprintf('saving %s\n',savename);
     ntriesleft = 5;
     while(ntriesleft > 0)
       ntriesleft = ntriesleft -1;
       try
         eval(savecmd);
         load(savename);
	 fprintf('success\n');
	 break;
       catch
	 pause(ceil(rand*60))
	 fprintf('ERROR, trying again\n');
	 continue;
       end
       fprintf('Out of save attempts.  Perhaps Disk is full?\n');
     end
  else  
     % THIS CODE IS WAHT THIS FUNCTION WAS BEFORE ADDING THE bSplitTo50s part.
     cmd = sprintf('saNewMats = %s(NewDataX,options);',FunctionName);
     eval(cmd);
     fns = fieldnames(saNewMats);
     nfns = length(fns);
     savecmd = 'save(savename,''y'',''options''';
     for i = 1:nfns
       cmd = sprintf('%s = saNewMats.%s;',fns{i},fns{i});
       eval(cmd);
       savecmd = [savecmd,sprintf(',''%s''',fns{i})];
     end  
     savecmd = [savecmd,');'];
     eval(savecmd);
     fprintf('saving %s\n',savename);
   end
 end
 cd(here);

function [yfull, NewDataX] = GetNewDataX(Template, variablename,sourceIdx,on,Root)
 NewDataX = [];
 yfull = [];
 iFl = sourceIdx;
 D = load(fullfile(Root,sprintf(Template,on,sourceIdx)));%-->grayX,toyotaX,y;
 NewDataX = [NewDataX,getfield(D,variablename)];
 yfull = [yfull,D.y]; 
