X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=webservices%2Fcompbio%2Fstat%2Fcollector%2FExecutionStatCollector.java;h=c50ab70b38bf3f1c9eee429f421176ebf9d9f164;hb=4825c5456a465ac3e13d0131f8e87f584b9871fa;hp=62f39c9ef8015a496f37c935e1e15a465eeb4eaa;hpb=1771320b59430a5f09d5fbae60df3aecce7312e7;p=jabaws.git diff --git a/webservices/compbio/stat/collector/ExecutionStatCollector.java b/webservices/compbio/stat/collector/ExecutionStatCollector.java index 62f39c9..c50ab70 100644 --- a/webservices/compbio/stat/collector/ExecutionStatCollector.java +++ b/webservices/compbio/stat/collector/ExecutionStatCollector.java @@ -1,4 +1,5 @@ -/* Copyright (c) 2011 Peter Troshin +/* Copyright (c) 2013 Alexander Sherstnev + * Copyright (c) 2011 Peter Troshin * * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 2.0 * @@ -24,32 +25,23 @@ import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.log4j.Logger; -import compbio.engine.client.Executable; import compbio.engine.client.PathValidator; import compbio.engine.client.SkeletalExecutable; -import compbio.metadata.JobStatus; import compbio.util.FileUtil; -import compbio.ws.client.Services; -import compbio.ws.client.ServicesUtil; /** - * Number of runs of each WS = number of folders with name - * - * Number of successful runs = all runs with no result file - * - * Per period of time = limit per file creating time Runtime (avg/max) = - * - * started time - finished time - * - * Task & result size = result.size + * Class assumptions: + * 1. Number of runs of each WS = number of folders with name + * 2. Number of successful runs = all runs with no result file + * 3. Per period of time = limit per file creating time + * 4. Runtime (avg/max) = finish time - start time + * 5. Task & result size = result.size * * Abandoned runs - not collected runs * @@ -65,18 +57,16 @@ import compbio.ws.client.ServicesUtil; * work directory for local and cluster tasks = from Helper or cmd parameter. WS * names - enumeration. Status file names and content. * - * @author pvtroshin + * @author Peter Troshin + * @author Alexander Sherstnev * */ public class ExecutionStatCollector implements Runnable { - static final int UNDEFINED = -1; - - private static final Logger log = Logger.getLogger(ExecutionStatCollector.class); - static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); + private static final Logger log = Logger.getLogger(ExecutionStatCollector.class); - final private File workDirectory; + final private File workingDirectory; final private List stats; /** * Consider the job that has been working for longer than timeOutInHours @@ -87,16 +77,16 @@ public class ExecutionStatCollector implements Runnable { /** * List subdirectories in the job directory * - * @param workDirectory + * @param workingDirectory * @param timeOutInHours */ - public ExecutionStatCollector(String workDirectory, int timeOutInHours) { - log.info("Starting stat collector for directory: " + workDirectory); + public ExecutionStatCollector(String workingDirectory, int timeOutInHours) { + log.info("Starting stat collector for directory: " + workingDirectory); log.info("Maximum allowed runtime(h): " + timeOutInHours); - if (!PathValidator.isValidDirectory(workDirectory)) { - throw new IllegalArgumentException("workDirectory '" + workDirectory + "' does not exist!"); + if (!PathValidator.isValidDirectory(workingDirectory)) { + throw new IllegalArgumentException("workingDirectory '" + workingDirectory + "' does not exist!"); } - this.workDirectory = new File(workDirectory); + this.workingDirectory = new File(workingDirectory); stats = new ArrayList(); if (timeOutInHours <= 0) { throw new IllegalArgumentException( @@ -139,12 +129,6 @@ public class ExecutionStatCollector implements Runnable { statdb.insertData(rjobs); } - /* - * static void updateTime(File statFile) throws IOException { long lastMod = - * statFile.lastModified(); FileWriter fw = new FileWriter(statFile); - * fw.write(new Long(lastMod).toString()); fw.close(); } - */ - /** * Not in use */ @@ -176,201 +160,37 @@ public class ExecutionStatCollector implements Runnable { static FileFilter directories = new FileFilter() { @Override public boolean accept(File pathname) { - return pathname.isDirectory() - && !pathname.getName().startsWith("."); + return pathname.isDirectory() && !pathname.getName().startsWith("."); } }; - static class JobDirectory { - - File jobdir; - Map files = new HashMap(); - - JobDirectory(File directory) { - this.jobdir = directory; - for (File f : jobdir.listFiles()) { - files.put(f.getName(), f); - } - } - - boolean hasStatus(JobStatus status) { - return files.containsKey(status.toString()); - } - - boolean isCollected() { - return hasStatus(JobStatus.COLLECTED); - } - - boolean isCancelled() { - return hasStatus(JobStatus.CANCELLED); - } - - long getStartTime() { - long starttime = UNDEFINED; - File startfile = files.get(JobStatus.STARTED.toString()); - if (startfile == null) { - startfile = files.get(JobStatus.SUBMITTED.toString()); - } - try { - if (startfile != null) { - String start = FileUtil.readFileToString(startfile); - starttime = Long.parseLong(start.trim()); - } - } catch (IOException ignore) { - log.warn( - "IOException while reading STARTED status file! Ignoring...", - ignore); - // fall back - starttime = startfile.lastModified(); - } catch (NumberFormatException ignore) { - log.warn( - "NumberFormatException while reading STARTED status file! Ignoring...", - ignore); - // fall back - starttime = startfile.lastModified(); - } - - return starttime; - } - - String getClusterJobID() { - String clustjobId = ""; - File jobid = files.get("JOBID"); - try { - if (jobid != null) { - clustjobId = FileUtil.readFileToString(jobid); - } - } catch (IOException ioe) { - log.error( - "IO Exception while reading the content of JOBID file for job " - + jobid, ioe); - } - return clustjobId.trim(); - } - - long getFinishedTime() { - long ftime = UNDEFINED; - File finished = files.get(JobStatus.FINISHED.toString()); - if (finished != null) { - try { - if (finished != null) { - String start = FileUtil.readFileToString(finished); - ftime = Long.parseLong(start.trim()); - } - } catch (IOException ignore) { - log.warn( - "IOException while reading FINISHED status file! Ignoring...", - ignore); - // fall back - ftime = finished.lastModified(); - } catch (NumberFormatException ignore) { - log.warn( - "NumberFormatException while reading FINISHED status file! Ignoring...", - ignore); - // fall back - ftime = finished.lastModified(); - } - } - return ftime; - } - - private Services getService() { - return ServicesUtil.getServiceByJobDirectory(jobdir); - } - - long getResultSize() { - Class> name = ServicesUtil - .getRunnerByJobDirectory(jobdir); - - File f = null; - if (name.getSimpleName().equalsIgnoreCase("IUPred")) { - f = files.get("out.glob"); - if (f == null) - f = files.get("out.short"); - if (f == null) - f = files.get("out.long"); - } else { - f = files.get(SkeletalExecutable.OUTPUT); - } - if (f != null) { - return f.length(); - } - return UNDEFINED; - } - - long getInputSize() { - Class> name = ServicesUtil - .getRunnerByJobDirectory(jobdir); - - File input = files.get(SkeletalExecutable.INPUT); - if (input != null) { - return input.length(); - } - return UNDEFINED; - } - - JobStat getJobStat() { - return JobStat.newInstance(getService(), getClusterJobID(), - jobdir.getName(), getStartTime(), getFinishedTime(), - getInputSize(), getResultSize(), isCancelled(), - isCollected()); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((jobdir == null) ? 0 : jobdir.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - JobDirectory other = (JobDirectory) obj; - if (jobdir == null) { - if (other.jobdir != null) - return false; - } else if (!jobdir.equals(other.jobdir)) - return false; - return true; - } - } - // TODO test! void collectStatistics() { // clear stats array; stats.clear(); - File[] files = workDirectory.listFiles(directories); - for (File file : files) { - // skip work directory with test input - if (InputFilter.accept(new File(file.getPath() + File.separator + SkeletalExecutable.INPUT))) { - JobDirectory jd = new JobDirectory(file); + File[] dirs = workingDirectory.listFiles(directories); + for (File dir : dirs) { + // skip work directory with test inputas + log.debug("check directory: " + dir.getName() + "..."); + if (InputFilter.accept(new File(dir.getPath() + File.separator + SkeletalExecutable.INPUT))) { + JobDirectory jd = new JobDirectory(dir); JobStat jstat = jd.getJobStat(); // Do not record stats on the job that has not completed yet if (hasCompleted(jd)) { stats.add(jstat); - System.out.println("added: id = " + jd); } else { - log.debug("Skipping the job: " + jstat); - log.debug("As it has not completed yet"); + log.debug("Skipping the job: " + jstat + " as it has not completed yet"); } - // System.out.println(jd.getJobStat().getJobReportTabulated()); } else { - log.trace("training input: " + file.getPath() + File.separator + SkeletalExecutable.INPUT); + log.trace("training input: " + dir.getName() + File.separator + SkeletalExecutable.INPUT); } } } + @Override public void run() { log.info("Started updating statistics at " + new Date()); - log.info("For directory: " + workDirectory.getAbsolutePath()); + log.info("For directory: " + workingDirectory.getAbsolutePath()); collectStatistics();