From f2ed5ce799ceb51f32daa98c9d6c8fe936bf2195 Mon Sep 17 00:00:00 2001 From: pvtroshin Date: Thu, 3 Mar 2011 16:40:30 +0000 Subject: [PATCH] Centralized statistic collector. All WS are converted to use the same input and output names nano time changed to milliseconds time. Is not finished yet git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@3789 e3abac25-378b-4346-85de-24260fe3988d --- TODO.txt | 8 + engine/compbio/engine/ExecutionStatCollector.java | 203 +++++++++++++++++++++ engine/compbio/engine/client/Util.java | 2 +- webservices/compbio/ws/server/AAConWS.java | 1 - webservices/compbio/ws/server/DisemblWS.java | 1 - webservices/compbio/ws/server/GlobPlotWS.java | 1 - webservices/compbio/ws/server/JronnWS.java | 1 - webservices/compbio/ws/server/MafftWS.java | 1 - webservices/compbio/ws/server/MuscleWS.java | 1 - webservices/compbio/ws/server/ProbconsWS.java | 1 - webservices/compbio/ws/server/TcoffeeWS.java | 1 - 11 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 engine/compbio/engine/ExecutionStatCollector.java diff --git a/TODO.txt b/TODO.txt index d7ae747..5eec1b6 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,11 @@ TODO: +Test all WS as names from Executables were removed + +Add default names for input and output in every executable and use them consistently +throughts (e.g. in all WS). Best of all use the same name from SceletalExectuable +For statistics. + FIXME: Conecting to JABAWS version 2 service 09-Feb-2011 15:27:53 compbio.ws.client.Jws2Client connect @@ -11,6 +17,8 @@ Exception in thread "main" java.lang.NullPointerException ScoreManager should output scores properly +Allow empty Parameters and Preset files! + Check the WS input and reject it on submission rather then of access with error message Globprot need a proper reference to bio python and sav_gol binaries -> they should be diff --git a/engine/compbio/engine/ExecutionStatCollector.java b/engine/compbio/engine/ExecutionStatCollector.java new file mode 100644 index 0000000..2088f52 --- /dev/null +++ b/engine/compbio/engine/ExecutionStatCollector.java @@ -0,0 +1,203 @@ +package compbio.engine; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import compbio.engine.client.ConfExecutable; +import compbio.engine.conf.PropertyHelperManager; +import compbio.metadata.JobStatus; +import compbio.util.FileUtil; +import compbio.util.PropertyHelper; + +/** + * Number of runs of each WS = number of folders with name + * + * Number of successful runs = all runs with no result file + * + * Per period of time = limit per file creating time Runtime (avg/max) = + * + * started time - finished time Task & result size = result.size + * + * Abandoned runs - not collected runs + * + * Cancelled runs - cancelled + * + * Cluster vs local runs + * + * Reasons for failure = look in the err out? + * + * + * Metadata required: + * + * work directory for local and cluster tasks = from Helper or cmd parameter. WS + * names - enumeration. Status file names and content. + * + * @author pvtroshin + * + */ +public class ExecutionStatCollector { + + private static final Logger log = Logger + .getLogger(ExecutionStatCollector.class); + + static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); + + static String getClusterJobDir() { + String clusterdir = ph.getProperty("cluster.tmp.directory"); + if (clusterdir != null) { + clusterdir.trim(); + } + return clusterdir; + } + + static String getLocalJobDir() { + String locdir = ph.getProperty("local.tmp.directory"); + if (locdir != null) { + locdir.trim(); + } + return locdir; + } + + /** + * + * @param args + */ + public static void main(String[] args) { + + String workDir = PropertyHelperManager.getLocalPath() + + getLocalJobDir().trim(); + System.out.println(workDir); + File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout", + directories); + for (File file : files) { + JobDirectory jd = new JobDirectory(file); + System.out.println(jd.getJobReport()); + } + } + + static FileFilter directories = new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.isDirectory(); + } + }; + + static class JobDirectory { + static final int UNDEFINED = -1; + + File jobdir; + Map files = new HashMap(); + + public JobDirectory(File directory) { + this.jobdir = directory; + for (File f : jobdir.listFiles()) { + files.put(f.getName(), f); + } + } + + public boolean hasStatus(JobStatus status) { + return files.containsKey(status.toString()); + } + + long getStartTime() { + long starttime = UNDEFINED; + try { + File startfile = files.get(JobStatus.STARTED.toString()); + if (startfile == null) { + startfile = files.get(JobStatus.SUBMITTED.toString()); + } + if (startfile != null) { + String start = FileUtil.readFileToString(startfile); + starttime = Long.parseLong(start.trim()); + } + } catch (IOException e) { + log.log(Level.WARN, + "Cannot parse start time: " + e.getMessage(), e); + } catch (NumberFormatException e) { + log.log(Level.WARN, + "Cannot parse start time: " + e.getMessage(), e); + } + return starttime; + } + long getFinishedTime() { + long ftime = UNDEFINED; + try { + File finished = files.get(JobStatus.FINISHED.toString()); + if (finished != null) { + String start = FileUtil.readFileToString(finished); + ftime = Long.parseLong(start.trim()); + // System.out.println("f " + ftime); + } + } catch (IOException e) { + log.log(Level.WARN, + "Cannot parse finished time: " + e.getMessage(), e); + } catch (NumberFormatException e) { + log.log(Level.WARN, + "Cannot parse finished time: " + e.getMessage(), e); + } + return ftime; + } + + public int getRuntime() { + return (int) (getFinishedTime() - getStartTime()); + } + + String getWSName() { + String name = jobdir.getName().split("#")[0]; + if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { + assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; + name = name.substring(1); + } + return name; + } + + // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out + // Probcons task:fasta.in result:alignment.out + /* + * TODO replace with Universal names for WS! + */ + long getResultSize() { + String name = getWSName(); + File f = null; + if (name.equalsIgnoreCase("Probcons")) { + f = files.get("alignment.out"); + } + f = files.get("fasta.out"); + if (f != null) { + return f.length(); + } + return UNDEFINED; + } + + long getInputSize() { + File input = files.get("fasta.in"); + if (input != null) { + return input.length(); + } + return UNDEFINED; + } + + String getJobReport() { + String report = "JOB: " + jobdir.getName() + "\n"; + if (getStartTime() != UNDEFINED) { + report += "Started " + new Date(getStartTime()) + "\n"; + } + if (getFinishedTime() != UNDEFINED) { + report += "Finished " + new Date(getFinishedTime()) + "\n"; + } + if (getStartTime() != UNDEFINED && getFinishedTime() != UNDEFINED) { + report += "Runtime " + getRuntime() + "\n"; + } + report += "Input size " + getInputSize() + "\n"; + report += "Result size " + getResultSize() + "\n"; + return report; + } + } +} diff --git a/engine/compbio/engine/client/Util.java b/engine/compbio/engine/client/Util.java index 4d0b8a1..afe2b6c 100644 --- a/engine/compbio/engine/client/Util.java +++ b/engine/compbio/engine/client/Util.java @@ -69,7 +69,7 @@ public final class Util { // ignore if (!compbio.util.Util.isEmpty(workDirectory)) { writeFile(workDirectory, fileAndEventName, new Long(System - .nanoTime()).toString(), false); + .currentTimeMillis()).toString(), false); } } diff --git a/webservices/compbio/ws/server/AAConWS.java b/webservices/compbio/ws/server/AAConWS.java index 8637af1..e6f8122 100644 --- a/webservices/compbio/ws/server/AAConWS.java +++ b/webservices/compbio/ws/server/AAConWS.java @@ -57,7 +57,6 @@ public class AAConWS implements SequenceAnnotation { ConfiguredExecutable init(List sequences) throws JobSubmissionException { AACon aacon = new AACon(); - aacon.setInput("fasta.in").setOutput("aacon.out"); return Configurator.configureExecutable(aacon, sequences); } diff --git a/webservices/compbio/ws/server/DisemblWS.java b/webservices/compbio/ws/server/DisemblWS.java index 5c6bac5..62f70d3 100644 --- a/webservices/compbio/ws/server/DisemblWS.java +++ b/webservices/compbio/ws/server/DisemblWS.java @@ -53,7 +53,6 @@ public class DisemblWS implements SequenceAnnotation { ConfiguredExecutable init(List sequences) throws JobSubmissionException { Disembl disembl = new Disembl(); - disembl.setInput("fasta.in").setOutput("disembl.out"); return Configurator.configureExecutable(disembl, sequences); } diff --git a/webservices/compbio/ws/server/GlobPlotWS.java b/webservices/compbio/ws/server/GlobPlotWS.java index b058882..603c7dd 100644 --- a/webservices/compbio/ws/server/GlobPlotWS.java +++ b/webservices/compbio/ws/server/GlobPlotWS.java @@ -53,7 +53,6 @@ public class GlobPlotWS implements SequenceAnnotation { ConfiguredExecutable init(List sequences) throws JobSubmissionException { GlobPlot globPlot = new GlobPlot(); - globPlot.setInput("fasta.in").setOutput("globPlot.out"); return Configurator.configureExecutable(globPlot, sequences); } diff --git a/webservices/compbio/ws/server/JronnWS.java b/webservices/compbio/ws/server/JronnWS.java index b7a8f43..202cad1 100644 --- a/webservices/compbio/ws/server/JronnWS.java +++ b/webservices/compbio/ws/server/JronnWS.java @@ -55,7 +55,6 @@ public class JronnWS implements SequenceAnnotation { ConfiguredExecutable init(List sequences) throws JobSubmissionException { Jronn jronn = new Jronn(); - jronn.setInput("fasta.in").setOutput("jronn.out"); return Configurator.configureExecutable(jronn, sequences); } diff --git a/webservices/compbio/ws/server/MafftWS.java b/webservices/compbio/ws/server/MafftWS.java index 83c5758..d2de592 100644 --- a/webservices/compbio/ws/server/MafftWS.java +++ b/webservices/compbio/ws/server/MafftWS.java @@ -78,7 +78,6 @@ public class MafftWS implements MsaWS { ConfiguredExecutable init(List dataSet) throws JobSubmissionException { Mafft mafft = new Mafft(); - mafft.setInput("fasta.in").setOutput("fasta.out"); return Configurator.configureExecutable(mafft, dataSet); } diff --git a/webservices/compbio/ws/server/MuscleWS.java b/webservices/compbio/ws/server/MuscleWS.java index 5b694dc..d6a9ba1 100644 --- a/webservices/compbio/ws/server/MuscleWS.java +++ b/webservices/compbio/ws/server/MuscleWS.java @@ -78,7 +78,6 @@ public class MuscleWS implements MsaWS { ConfiguredExecutable init(List sequences) throws JobSubmissionException { Muscle muscle = new Muscle(); - muscle.setInput("fasta.in").setOutput("fasta.out"); return Configurator.configureExecutable(muscle, sequences); } diff --git a/webservices/compbio/ws/server/ProbconsWS.java b/webservices/compbio/ws/server/ProbconsWS.java index bfad95a..3600c46 100644 --- a/webservices/compbio/ws/server/ProbconsWS.java +++ b/webservices/compbio/ws/server/ProbconsWS.java @@ -76,7 +76,6 @@ public class ProbconsWS implements MsaWS { ConfiguredExecutable init(List dataSet) throws JobSubmissionException { Probcons probcons = new Probcons(); - probcons.setInput("fasta.in").setOutput("alignment.out"); return Configurator.configureExecutable(probcons, dataSet); } diff --git a/webservices/compbio/ws/server/TcoffeeWS.java b/webservices/compbio/ws/server/TcoffeeWS.java index 74c1723..5608a53 100644 --- a/webservices/compbio/ws/server/TcoffeeWS.java +++ b/webservices/compbio/ws/server/TcoffeeWS.java @@ -81,7 +81,6 @@ public class TcoffeeWS implements MsaWS { ConfiguredExecutable init(List sequences) throws JobSubmissionException { Tcoffee tcoffee = new Tcoffee(); - tcoffee.setInput("fasta.in").setOutput("fasta.out"); ConfiguredExecutable confCoffee = Configurator .configureExecutable(tcoffee, sequences); if (confCoffee.getExecProvider() == Executable.ExecProvider.Cluster) { -- 1.7.10.2