JobUtils.java

Go to the documentation of this file.

package cedar.jetweb.job;

import java.io.*;

import java.util.*;
import java.util.Map.Entry;

import cedar.jetweb.*;
import cedar.jetweb.html.*;
import cedar.jetweb.xml.*;
import cedar.jetweb.model.*;
import cedar.jetweb.model.plots.*;
import cedar.jetweb.model.paper.*;
import cedar.jetweb.model.fit.Fit;
import cedar.jetweb.db.DBJobManager;
import cedar.jetweb.batch.SubmitScriptHandler;
import cedar.jetweb.generator.Generator;
import cedar.jetweb.util.JetWebFile;

import cedar.jetweb.*;

import org.apache.commons.io.FileUtils;

public abstract class JobUtils {

    public static void updateJobs() throws JetWebException {

    // Add all new LogFiles and return a vector of them:


    System.out.println("=== checking new files ===");

    Vector<LogFile> logFiles = getLogFiles
        (new File(JetWebConfig.jobInDirName));

    System.out.println("=== found vector of new files ===");

    // for each logfile find the models that are affected.

    for(LogFile log: logFiles){

        System.out.println("===          ");

        System.out.println("=== getting runseries from file ===");

        RunSeries rs = log.getRunSeries();
       
        System.out.println("=== found runseries ===");

        //Set the Cuts applied to this runseries

        System.out.println("=== setting cuts on runseries ===");

        rs.getGenerator().setCutCollection
        (log.getGenerator().getCutCollection());
        
        

        //set the smallest cut collection that fits all the plots 
        //matched by this runseries

        rs.setCuts();
        

        System.out.println("=== cuts set ===");

        System.out.println("=== getting consistent models ===");

        Vector<Model> changedModels = 
        rs.getConsistentModels();
        //make a new model if there isn't one

        System.out.println("=== got consistent models ===");

        if(changedModels.size()==0){

        System.out.println("JobUtils: Found no consistent Models");

        System.out.println("=== making and storing new model ===");

        Model model  = new Model(rs);
        model.store();
        changedModels.add(model);

        System.out.println("=== made and stored new model ===");

        }

        for(Model model: changedModels){
        
        System.out.println
            ("=== at start of loop over changed models ===");

        System.out.println("=== adding all runseries to model ===");

        model.addAllRunSeries();

        System.out.println("=== all runseries added ===");

        System.out.println("=== storing model ===");

        model.store();

        System.out.println("=== model stored ===");

        System.out.println("=== making new plotselection ===");

        PlotSelection sel = new PlotSelection();

        System.out.println("=== made new plotselection ===");
        System.out.println
            ("=== setting default plots in selection ===");

        sel.setFitDefault();
        sel.setSumDefault();

        System.out.println("=== default plots set ===");

        System.out.println("=== getting cuts from logfile ===");

        CutCollection phaseSpace = 
            log.getGenerator().getCutCollection();

        System.out.println("=== got cuts ===");

        System.out.println("=== restricting plotselection cuts ===");

        sel.restrictCutCollection(rs.getMCProcessType(),phaseSpace);

        System.out.println("=== done restricting cuts ===");

        Fit fit = null;
        boolean goodFit = false;

        try {

            System.out.println("=== making new fit ===");

            fit = new Fit(model,sel);
            goodFit = true;

            System.out.println("=== made new fit ===");

        } catch (JetWebException jwe) {         
            // Failed to build the default fit, so do the best we can.


            System.out.println("=== setting best plots ===");

            sel.setFitToBest(model);
            sel.setSumToBest(model);        
            sel.restrictCutCollection(rs.getMCProcessType(),phaseSpace);

            System.out.println("=== set best plots ===");

            if (sel.getFitPlots().size()==0){
            
            // None of the default fit plots have been simulated,
            // so just generate a fit
            // using all the plots which the Chi2 is used for.

            System.out.println
                ("=== setting fit plots ===");

            sel.setFitPlots(sel.getSumPlots());

            System.out.println("=== fit plots set ===");
            }
            if (sel.getFitPlots().size()>0){
            try{

                System.out.println("=== making new fit ===");
                
                fit = new Fit(model,sel);
                goodFit = true;

                System.out.println("=== new fit made ===");

            }catch (Throwable err){
                System.out.println(err.getMessage());
            }
            }
        }

        if(goodFit){
            System.out.println("Now to update the fit and save it");

            System.out.println("=== updating fit ===");
            fit.update(false);

            System.out.println("=== fit updated ===");

            System.out.println("back from update");
        }
        }
    }
    
    return;
    }       


    public static void doJobRequest(ResultSearchPattern pattern, 
                    Vector processList,
                    PrintWriter out,
                    String servletPath,
                    String requestingHost
                    ) throws JetWebException {

    JobRequest jreq = null;
    Model model = null;

    if (processList.size()==0){
        // Must specify at least one process.
        HTMLWriter op = new HTMLWriter(out,servletPath);
        HTMLUtils.writeJobRequestForm("Please specify a process to generate.",
                pattern,new PlotSelection(),
                out,servletPath);
        return;         
    }
    HashMap<MCProcessType, RunSeries> submittedRunSeries = 
        new HashMap<MCProcessType, RunSeries>();
    // Build a new job request.
    try {

        // Build a Model from the search pattern.
        model = new Model(pattern);
        model.addAllRunSeries();

        // New logfile row not written at this point

        //LogFile log = new LogFile();

        //Get a list of the already existing runseries that match
        //the model to be submitted.
        //Vector<RunSeries> existingRunSeries = new Vector<RunSeries>();
        
        HashMap<MCProcessType, RunSeries> existingRunSeries = 
        new HashMap<MCProcessType, RunSeries>();
       
        for(Enumeration proc = processList.elements(); 
        proc.hasMoreElements();){
        
        for(RunSeries rs: 
            model.getRunSeries((MCProcessType)proc.nextElement())){
            if(pattern.matches(rs)){

            //need to check cut collection against 
            //existing runseries

            CutCollection existingCuts = 
                rs.getCutCollection();

            if(existingCuts!=null &&
               (existingCuts.getCuts().size()==0 || 
                existingCuts.isWithin(
                pattern.getGenerator().getCutCollection()))
               ){

                RunSeries exists = existingRunSeries.get
                (rs.getMCProcessType());
                if(exists==null || 
                   exists.getCutCollection().
                   isWithin(rs.getCutCollection())
                   ){
                
                existingRunSeries.put(rs.getMCProcessType(), rs);
                }
            }
            }
        }
        }
        
        // Create a run series if necessary 
        // for this submission
        
        for(Enumeration proc = processList.elements(); 
        proc.hasMoreElements();){
        
        RunSeries tmpRS = new RunSeries(pattern);
        MCProcessType tmpProc = (MCProcessType)proc.nextElement();
        tmpRS.setMCProcessType(tmpProc);
        tmpRS.setCuts();
        submittedRunSeries.put(tmpProc,tmpRS);      

        }
    
        for(Map.Entry entry: existingRunSeries.entrySet()){

        RunSeries tmpRS = (RunSeries) entry.getValue();
        MCProcessType tmpProc = (MCProcessType) entry.getKey();
        
        RunSeries submitted = submittedRunSeries.get(tmpProc);

        if(submitted==null ||
           submitted.getCutCollection().isWithin
           (tmpRS.getCutCollection()) || 
           submitted.getCutCollection().equals
            (tmpRS.getCutCollection())){
            submittedRunSeries.put
            (tmpProc, tmpRS); 

            }
        }

        //for(RunSeries rs: existingRunSeries){
        //submittedRunSeries.put(rs.getMCProcessType(), rs);
        //}
            
    } catch (JetWebException e) {
        // Keep bouncing back if there is not enough information to
        // specify the model.
        HTMLWriter op = new HTMLWriter(out,servletPath);
        HTMLUtils.writeJobRequestForm("Please specify all fields.",
                pattern,new PlotSelection(),
                out,servletPath);
        return;         
    }

    //No logfile written to DB by this point
    StringBuffer jobsDone = new StringBuffer();
    Enumeration list = processList.elements();
    int status = -1;
    while (list.hasMoreElements()){
        MCProcessType proc = (MCProcessType)list.nextElement();

        jreq = new JobRequest(model,proc,requestingHost);

        if (jreq==null) {
        throw new JetWebException("JobUtils:","No process types were selected!");
        }

        // Set the job name here.
        jreq.setJobName();

        // Make the files for this job.
        makeCards(jreq);
        String newFilePath = JetWebConfig.jobOutDirName+"/"+jreq.getJobName();
        makeScript(newFilePath,jreq.getModel().getGenerator());

        // Submit it.
        status = submit(jreq);

        if (status == 0) {
        jobsDone.append("Successfully submitted: "
                +HTMLJobWriter.getDescription(jreq));
        RunSeries rs = submittedRunSeries.get(proc);
        
        LogFile log = new LogFile(rs);

        log.setFile(new JetWebFile(JetWebConfig.jobOutDirName + 
                     "/" + 
                     jreq.getJobName()));
        //retrieve the already reserved logfile name
        //doing this sets the log status to reserved
        log.retrieve();
        log.getGenerator().setCutCollection
            (pattern.getGenerator().getCutCollection());
        log.setStatus(LogFile.SUBMITTED);

        log.store();
        //model.store();
        } else {
        jobsDone.append("Failed job submission code "+status+"<br/>");
        }
    }

    if(status == 0 ){
        model.addAllRunSeries();
        model.store();
    }

    HTMLUtils.writeQueryScreen(jobsDone.toString(),
                   pattern,new PlotSelection(),
                   out,servletPath);
    return;         
    }           

    public static void makeCards(JobRequest jreq) 
    throws JetWebException {

    Model model = jreq.getModel();
    MCProcessType proc = jreq.getMCProcessType();
    Collision collision = proc.getCollision();
    Generator gen = model.getGenerator();
    
    System.out.println("JobUtils.makecards:"+jreq.getCardName()+" , "+gen);

    File test = new File(JetWebConfig.rootDirectory+jreq.getCardName());
    
    if (test.exists()) {
        throw new JobRequestException("cannot create new card because filename already exists");
    }

    try {
        BufferedWriter newCards = new BufferedWriter(new FileWriter(jreq.getCardName()) );

        gen.writeSteering(newCards,model,proc);
        newCards.newLine();
        newCards.close();

    } catch ( IOException e ) {
        System.out.println("problem creating new card file:"+e);
        throw new JetWebException(e);
    } catch ( NullPointerException n ) {
        throw new JetWebException(n);
    }

    }

    public static void makeScript(String scriptname, Generator gen)
    throws JetWebException {

    String line;
    File test = new File(scriptname);
    
    if (test.exists()) {
        throw new JobRequestException("cannot create new script because filename already exists");
    }
    
    try {
        BufferedReader templateScript = 
        new BufferedReader( new InputStreamReader( new FileInputStream(SubmitScriptHandler.getTemplate())));
        BufferedWriter newScript = new BufferedWriter(new FileWriter(scriptname) );
        while((line=templateScript.readLine())!=null){
        // Choose executable
        if (line.startsWith("PROGRAM=")) {
            line = "PROGRAM=" + SubmitScriptHandler.getExecutable(gen);
        }
        newScript.write(line);
        newScript.newLine();
        }

        newScript.close();
        templateScript.close();

    } catch ( IOException e ) {
        System.out.println("problem creating new script file:"+e);
        return;
    }

    }

    public static void moveLogs(Enumeration logs, String newDir){
    
    String editDir = JetWebConfig.rootDirectory+"/"+newDir+"/";

    while (logs.hasMoreElements()){
        
        LogFile log = (LogFile)logs.nextElement();

        String xmlName = log.getStem()+".xml";
        File xml   = new File(xmlName); 

        String aidaName = log.getStem()+".aida";
        File aida   = new File(aidaName); 

        File file  = log.getFile();

        File newFile = new File(editDir+file.getName());
        file.renameTo(newFile);

        newFile = new File(editDir+xml.getName());
        xml.renameTo(newFile);

        newFile = new File(editDir+aida.getName());
        aida.renameTo(newFile);

        // Remove this logfile from the logfile table.
        DBJobManager.removeLogFile(log);
    }
    }

    
    public static boolean isValidLogName(String name){

    if (name.equals("CVS")){
        return false;
    }

    if ( ((name.indexOf(".o")>0) || (name.indexOf(".log")>0)  &&
         !(name.indexOf("~")>0) && 
          !(name.indexOf("#")>0)) &&
         !name.contains(".gz")){
        return true;
    }
    return false;

    }

    public static String zipIt(String name) throws JetWebException {

    JetWebFile file = new JetWebFile(name);
    
    file = file.Zip();

    return file.directory() + file + file.extension() + ".gz";
    }

    public static void removeBadPoints(PredictedPlot plot, LogFile log) 
    throws JetWebException {
    
    String logName = log.getFile().getName();
    String runLabel = logName.substring(0,1);
    int runNumber;
    int csnId = plot.getId();
    try {
        runNumber = Integer.parseInt(logName.substring(1,12));
    } catch (Exception e) {
        runNumber = 0;
        runLabel = "old";
    }

    //System.out.println(runSeries+","+runNumber);

    return;
    }

    public static int countJobs(String batchNum) throws JetWebException {
    return DBJobManager.countJobs(batchNum);
    }
    public static int countCompletedJobs(String batchNum) throws JetWebException {
    return DBJobManager.countCompletedJobs(batchNum);
    }

    public static Enumeration selectPendingJobs() throws JetWebException {
    return DBJobManager.selectPendingJobs();
    }



    public static int submit(JobRequest jreq) { 

    String line;
    try {
        jreq.setSubmitScript(File.createTempFile("job",".ready."+jreq.getRequestingHost(),
                       new File(JetWebConfig.spoolDirName)));
    } catch (IOException e){
        return SubmitScriptHandler.CANTWRITESCRIPT;
    }
    
    //  pj.setScriptName(submitScript.getName()); // BMW: pj not created

    // make the submit script.
    return SubmitScriptHandler.make(jreq);

    }

    public static boolean moveAndStore(LogFile log) throws JetWebException {
    
    JetWebFile logFile = log.getFile();

    String stem = log.getStem();

    JetWebFile file  = new JetWebFile(stem, JetWebFile.AIDA);
    file = file.Archive();
    file = file.Zip();

    file = new JetWebFile(stem, JetWebFile.XML);
    file = file.Archive();
    file = file.Zip();

    file = new JetWebFile(stem, JetWebFile.OUT);
    file = file.Archive();
    file = file.Zip();

    if (log.store()) {
        log.setStatus(LogFile.COMPLETED);
    }
    return true;
    }

    public static Vector<Model> getNewData(File root) throws JetWebException {

    Vector<Model> modelList   = new Vector<Model>();
    Vector<File> fileList   = new Vector<File>();

    if (root.isDirectory()) {

        // Add the root directory to the list of files to be checked.
        fileList.add(root);

        // Find files ending with .xml in this directory
        // and any subdirectories.
        while (fileList.size()>0){
        
        Enumeration<File> list = fileList.elements();

        while (list.hasMoreElements()){
            
            File tmpfile = list.nextElement();

            // First look for directories.
            if (tmpfile.isDirectory()){
            System.out.println
                ("JobUtils: Searching in "+tmpfile.getPath());

            File[] array = tmpfile.listFiles();
            int n = array.length;
            for (int i=0; i<n; i++){
                fileList.addElement(array[i]);
            }

            } else {

            // Now look for aida files.
            String tmpname = tmpfile.getPath(); 
            
            if ( JobUtils.isValidLogName(tmpname)) {
                // Now search through to see if it terminated successfully.
                
                // first, gzip it if necessary.
                tmpname = JobUtils.zipIt(tmpname);
                
                // Instantiate, but don't add to the DB yet.
                LogFile log = new LogFile(tmpname);
                System.out.println
                ("JobUtils: Created log: about to fill.");
                System.out.println("JobUtils: Using xml file name "
                           + log.getXmlFileName());
                JetWebLogReader logReader = 
                new JetWebLogReader(log.getXmlFileName());

                logReader.fillLog(log);
                log.checkStatus();
                System.out.println
                ("JobUtils: log status = " + log.getStatus());
                if (log.getStatus()==LogFile.RUN){
                System.out.println("JobUtils:"+log.toString());
                // Now see if an aida file really exists.
                if (log.hasAidaFile()){
                    // Move the logfile & add it to the DB.
                    if (moveAndStore(log)){
                    
                    // Get all models which can be constructed from 
                    // this runSeries in combination with others. 
                    System.out.println
                        ("JobUtils: adding models");
                    RunSeries rs = log.getRunSeries();
                    Vector<Model> mList = 
                        rs.getConsistentModels();

                    if (mList.size()==0) {
                        // Make a new model with
                        // this run series plus
                        // default parameters for
                        // any not set by the
                        // runseries.
                        System.out.println
                 ("JobUtils: Model: Making a new one for " + 
                         log.getXmlFileName());
                        Model model = new Model(rs);
                        model.store();
                        mList.add(model);
                    } else {
                        System.out.println("JobUtils: Model:Not making a new one for "+log.getXmlFileName());
                    }
                    modelList.addAll(mList);
                    System.out.println("JobUtils: added models");
                    
                    }
                }
                }
            }
            }
            // Now remove this file from the vector.
            fileList.remove(tmpfile);
        }

        }
        
    } else {
        System.out.println("Invalid root directory: "+root.getPath());
    } 
    System.out.println("JobUtils: "+modelList.size()+" Models affected");
    return modelList;

    }

    public static Vector<LogFile> getLogFiles(int rsid) throws JetWebException {
    return DBJobManager.getLogFiles(rsid); 
    }

    private static Vector<LogFile> getLogFiles(File dir)
    throws JetWebException{
    
    Vector<File> fileList = new Vector<File>();
    Vector<LogFile> logFiles = new Vector<LogFile>();
    if(dir.isDirectory()){
        fileList.add(dir);

        while(fileList.size()>0){
        Enumeration<File> list = fileList.elements();
        while (list.hasMoreElements()){

            File tmpfile = list.nextElement();

            // First look for directories.
            if (tmpfile.isDirectory()){
            
            File[] array = tmpfile.listFiles();
            int n = array.length;
            for (int i=0; i<n; i++){
                fileList.addElement(array[i]);
            }
            
            }else{

            // Now look for aida files.
            String tmpname = tmpfile.getPath(); 
            
            if ( JobUtils.isValidLogName(tmpname)) {
                // Now search through to see if it 
                // terminated successfully.
                
                // first, gzip it if necessary.
                tmpname = JobUtils.zipIt(tmpname);
                
                // Instantiate, but don't add to the DB yet.
                LogFile log = new LogFile(tmpname);
                System.out.println
                ("JobUtils: Created log: about to fill.");
                System.out.println("JobUtils: Using xml file name "
                           + log.getXmlFileName());
                JetWebLogReader logReader = 
                new JetWebLogReader(log.getXmlFileName());

                logReader.fillLog(log);
                log.checkStatus();
                if (log.getStatus()==LogFile.RUN){

                // Now see if an aida file really exists.
                if (log.hasAidaFile()&& 
                    !log.isArchived()){
                    if(moveAndStore(log)){
                    logFiles.add(log);
                    }
                }
                }
            }
            }
            
            // Now remove this file from the vector.
            fileList.remove(tmpfile);
        }
        }
    }
    return logFiles;
    }

}

Generated Wed Jan 17 09:14:27 GMT 2007