X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=webservices%2Fcompbio%2Fstat%2Fcollector%2FExecutionStatCollector.java;h=cd97f4a2a8a60b4e49505ef7069c7c347f2f898d;hb=1e1c3681ba25ee1797a46f871b8c80f259afe2ca;hp=bf3130aa2ff1c4599866d460f2452943f59487d6;hpb=5289bedee673d95739fdf622dba7be64f8c2df13;p=jabaws.git diff --git a/webservices/compbio/stat/collector/ExecutionStatCollector.java b/webservices/compbio/stat/collector/ExecutionStatCollector.java index bf3130a..cd97f4a 100644 --- a/webservices/compbio/stat/collector/ExecutionStatCollector.java +++ b/webservices/compbio/stat/collector/ExecutionStatCollector.java @@ -1,3 +1,20 @@ +/* Copyright (c) 2011 Peter Troshin + * + * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 2.0 + * + * This library is free software; you can redistribute it and/or modify it under the terms of the + * Apache License version 2 as published by the Apache Software Foundation + * + * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache + * License for more details. + * + * A copy of the license is in apache_license.txt. It is also available here: + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt + * + * Any republication or derived work distributed in source code form + * must include this copyright and license notice. + */ package compbio.stat.collector; import java.io.File; @@ -16,6 +33,8 @@ import java.util.Set; import org.apache.log4j.Logger; import compbio.engine.client.Executable; +import compbio.engine.client.PathValidator; +import compbio.engine.client.SkeletalExecutable; import compbio.metadata.JobStatus; import compbio.util.FileUtil; import compbio.ws.client.Services; @@ -57,6 +76,7 @@ public class ExecutionStatCollector implements Runnable { static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); + final private File workDirectory; final private List stats; /** * Consider the job that has been working for longer than timeOutInHours @@ -73,22 +93,18 @@ public class ExecutionStatCollector implements Runnable { public ExecutionStatCollector(String workDirectory, int timeOutInHours) { log.info("Starting stat collector for directory: " + workDirectory); log.info("Maximum allowed runtime(h): " + timeOutInHours); - File[] files = FileUtil.getFiles(workDirectory, directories); + if (!PathValidator.isValidDirectory(workDirectory)) { + throw new IllegalArgumentException("workDirectory '" + + workDirectory + "' does not exist!"); + } + this.workDirectory = new File(workDirectory); stats = new ArrayList(); - assert timeOutInHours > 0; - this.timeOutInHours = timeOutInHours; - for (File file : files) { - JobDirectory jd = new JobDirectory(file); - JobStat jstat = jd.getJobStat(); - // Do not record stats on the job that has not completed yet - if (hasCompleted(jd)) { - stats.add(jstat); - } else { - log.debug("Skipping the job: " + jstat); - log.debug("As it has not completed yet"); - } - // System.out.println(jd.getJobStat().getJobReportTabulated()); + if (timeOutInHours <= 0) { + throw new IllegalArgumentException( + "Timeout value must be greater than 0! Given value: " + + timeOutInHours); } + this.timeOutInHours = timeOutInHours; } boolean hasCompleted(JobDirectory jd) { @@ -104,11 +120,18 @@ public class ExecutionStatCollector implements Runnable { return ((System.currentTimeMillis() - jd.jobdir.lastModified()) / (1000 * 60 * 60)) > timeOutInHours; } - public StatProcessor getStats() { + /* + * Make sure that collectStatistics methods was called prior to calling + * this! TODO consider running collectStatistics from here on the first call + */ + StatProcessor getStats() { + if (stats.isEmpty()) { + log.info("Please make sure collectStatistics method was called prior to calling getStats()!"); + } return new StatProcessor(stats); } - public void writeStatToDB() throws SQLException { + void writeStatToDB() throws SQLException { Set rjobs = new HashSet(stats); StatDB statdb = new StatDB(); log.debug("Removing records that has already been recorded"); @@ -125,10 +148,7 @@ public class ExecutionStatCollector implements Runnable { */ /** - * - * @param args - * @throws IOException - * @throws SQLException + * Not in use */ public static void main(String[] args) throws IOException, SQLException { @@ -168,14 +188,14 @@ public class ExecutionStatCollector implements Runnable { File jobdir; Map files = new HashMap(); - public JobDirectory(File directory) { + JobDirectory(File directory) { this.jobdir = directory; for (File f : jobdir.listFiles()) { files.put(f.getName(), f); } } - public boolean hasStatus(JobStatus status) { + boolean hasStatus(JobStatus status) { return files.containsKey(status.toString()); } @@ -193,13 +213,25 @@ public class ExecutionStatCollector implements Runnable { if (startfile == null) { startfile = files.get(JobStatus.SUBMITTED.toString()); } - if (startfile != null) { + try { + if (startfile != null) { + String start = FileUtil.readFileToString(startfile); + starttime = Long.parseLong(start.trim()); + } + } catch (IOException ignore) { + log.warn( + "IOException while reading STARTED status file! Ignoring...", + ignore); + // fall back + starttime = startfile.lastModified(); + } catch (NumberFormatException ignore) { + log.warn( + "NumberFormatException while reading STARTED status file! Ignoring...", + ignore); + // fall back starttime = startfile.lastModified(); - /* - * String start = FileUtil.readFileToString(startfile); - * starttime = Long.parseLong(start.trim()); - */ } + return starttime; } @@ -211,8 +243,9 @@ public class ExecutionStatCollector implements Runnable { clustjobId = FileUtil.readFileToString(jobid); } } catch (IOException ioe) { - ioe.printStackTrace(); - // TODO LOG + log.error( + "IO Exception while reading the content of JOBID file for job " + + jobid, ioe); } return clustjobId.trim(); } @@ -221,19 +254,25 @@ public class ExecutionStatCollector implements Runnable { long ftime = UNDEFINED; File finished = files.get(JobStatus.FINISHED.toString()); if (finished != null) { - ftime = finished.lastModified(); - /* - * String start = FileUtil.readFileToString(finished); ftime = - * Long.parseLong(start.trim()); - */ - // System.out.println("f " + ftime); + try { + if (finished != null) { + String start = FileUtil.readFileToString(finished); + ftime = Long.parseLong(start.trim()); + } + } catch (IOException ignore) { + log.warn( + "IOException while reading FINISHED status file! Ignoring...", + ignore); + // fall back + ftime = finished.lastModified(); + } catch (NumberFormatException ignore) { + log.warn( + "NumberFormatException while reading FINISHED status file! Ignoring...", + ignore); + // fall back + ftime = finished.lastModified(); + } } - /* - * } catch (IOException e) { log.log(Level.WARN, - * "Cannot parse finished time: " + e.getMessage(), e); } catch - * (NumberFormatException e) { log.log(Level.WARN, - * "Cannot parse finished time: " + e.getMessage(), e); } - */ return ftime; } @@ -241,22 +280,19 @@ public class ExecutionStatCollector implements Runnable { return Services.getServiceByJobDirectory(jobdir); } - // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out - // Probcons task:fasta.in result:alignment.out - /* - * TODO replace with Universal names for WS! - */ long getResultSize() { Class> name = Services .getRunnerByJobDirectory(jobdir); File f = null; - if (name.getSimpleName().equalsIgnoreCase("Probcons")) { - f = files.get("alignment.out"); - } else if (name.getSimpleName().equalsIgnoreCase("ClustalW")) { - f = files.get("output.txt"); + if (name.getSimpleName().equalsIgnoreCase("IUPred")) { + f = files.get("out.glob"); + if (f == null) + f = files.get("out.short"); + if (f == null) + f = files.get("out.long"); } else { - f = files.get("fasta.out"); + f = files.get(SkeletalExecutable.OUTPUT); } if (f != null) { return f.length(); @@ -265,7 +301,10 @@ public class ExecutionStatCollector implements Runnable { } long getInputSize() { - File input = files.get("fasta.in"); + Class> name = Services + .getRunnerByJobDirectory(jobdir); + + File input = files.get(SkeletalExecutable.INPUT); if (input != null) { return input.length(); } @@ -304,12 +343,30 @@ public class ExecutionStatCollector implements Runnable { return false; return true; } + } + void collectStatistics() { + File[] files = workDirectory.listFiles(directories); + for (File file : files) { + JobDirectory jd = new JobDirectory(file); + JobStat jstat = jd.getJobStat(); + // Do not record stats on the job that has not completed yet + if (hasCompleted(jd)) { + stats.add(jstat); + } else { + log.debug("Skipping the job: " + jstat); + log.debug("As it has not completed yet"); + } + // System.out.println(jd.getJobStat().getJobReportTabulated()); + } } @Override public void run() { log.info("Started updating statistics at " + new Date()); + log.info("For directory: " + workDirectory.getAbsolutePath()); + + collectStatistics(); StatProcessor local_stats = getStats(); log.info("Found " + local_stats.getJobNumber() + " jobs!"); @@ -321,5 +378,4 @@ public class ExecutionStatCollector implements Runnable { } log.info("Finished updating statistics at " + new Date()); } - }