From: pvtroshin Date: Wed, 9 Mar 2011 18:55:05 +0000 (+0000) Subject: Add exec statistics processor & derby database X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=d5491e105fa2b74fed3e0b652278acc1894a15f8;p=jabaws.git Add exec statistics processor & derby database git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@3806 e3abac25-378b-4346-85de-24260fe3988d --- diff --git a/.classpath b/.classpath index 4820b06..59011e3 100644 --- a/.classpath +++ b/.classpath @@ -12,5 +12,6 @@ + diff --git a/ExecutionStatistic/db.lck b/ExecutionStatistic/db.lck new file mode 100644 index 0000000..f130523 Binary files /dev/null and b/ExecutionStatistic/db.lck differ diff --git a/ExecutionStatistic/log/log.ctrl b/ExecutionStatistic/log/log.ctrl new file mode 100644 index 0000000..8fcdcc8 Binary files /dev/null and b/ExecutionStatistic/log/log.ctrl differ diff --git a/ExecutionStatistic/log/log1.dat b/ExecutionStatistic/log/log1.dat new file mode 100644 index 0000000..596c3ec Binary files /dev/null and b/ExecutionStatistic/log/log1.dat differ diff --git a/ExecutionStatistic/log/logmirror.ctrl b/ExecutionStatistic/log/logmirror.ctrl new file mode 100644 index 0000000..8fcdcc8 Binary files /dev/null and b/ExecutionStatistic/log/logmirror.ctrl differ diff --git a/ExecutionStatistic/seg0/c10.dat b/ExecutionStatistic/seg0/c10.dat new file mode 100644 index 0000000..0666696 Binary files /dev/null and b/ExecutionStatistic/seg0/c10.dat differ diff --git a/ExecutionStatistic/seg0/c101.dat b/ExecutionStatistic/seg0/c101.dat new file mode 100644 index 0000000..d8e6a8b Binary files /dev/null and b/ExecutionStatistic/seg0/c101.dat differ diff --git a/ExecutionStatistic/seg0/c111.dat b/ExecutionStatistic/seg0/c111.dat new file mode 100644 index 0000000..b046864 Binary files /dev/null and b/ExecutionStatistic/seg0/c111.dat differ diff --git a/ExecutionStatistic/seg0/c121.dat b/ExecutionStatistic/seg0/c121.dat new file mode 100644 index 0000000..dbfded6 Binary files /dev/null and b/ExecutionStatistic/seg0/c121.dat differ diff --git a/ExecutionStatistic/seg0/c130.dat b/ExecutionStatistic/seg0/c130.dat new file mode 100644 index 0000000..e06e9ec Binary files /dev/null and b/ExecutionStatistic/seg0/c130.dat differ diff --git a/ExecutionStatistic/seg0/c141.dat b/ExecutionStatistic/seg0/c141.dat new file mode 100644 index 0000000..428b101 Binary files /dev/null and b/ExecutionStatistic/seg0/c141.dat differ diff --git a/ExecutionStatistic/seg0/c150.dat b/ExecutionStatistic/seg0/c150.dat new file mode 100644 index 0000000..7110cae Binary files /dev/null and b/ExecutionStatistic/seg0/c150.dat differ diff --git a/ExecutionStatistic/seg0/c161.dat b/ExecutionStatistic/seg0/c161.dat new file mode 100644 index 0000000..318bc86 Binary files /dev/null and b/ExecutionStatistic/seg0/c161.dat differ diff --git a/ExecutionStatistic/seg0/c171.dat b/ExecutionStatistic/seg0/c171.dat new file mode 100644 index 0000000..f967175 Binary files /dev/null and b/ExecutionStatistic/seg0/c171.dat differ diff --git a/ExecutionStatistic/seg0/c180.dat b/ExecutionStatistic/seg0/c180.dat new file mode 100644 index 0000000..8f768e5 Binary files /dev/null and b/ExecutionStatistic/seg0/c180.dat differ diff --git a/ExecutionStatistic/seg0/c191.dat b/ExecutionStatistic/seg0/c191.dat new file mode 100644 index 0000000..6ab9ebc Binary files /dev/null and b/ExecutionStatistic/seg0/c191.dat differ diff --git a/ExecutionStatistic/seg0/c1a1.dat b/ExecutionStatistic/seg0/c1a1.dat new file mode 100644 index 0000000..cf7a015 Binary files /dev/null and b/ExecutionStatistic/seg0/c1a1.dat differ diff --git a/ExecutionStatistic/seg0/c1b1.dat b/ExecutionStatistic/seg0/c1b1.dat new file mode 100644 index 0000000..284bda0 Binary files /dev/null and b/ExecutionStatistic/seg0/c1b1.dat differ diff --git a/ExecutionStatistic/seg0/c1c0.dat b/ExecutionStatistic/seg0/c1c0.dat new file mode 100644 index 0000000..c5b91e2 Binary files /dev/null and b/ExecutionStatistic/seg0/c1c0.dat differ diff --git a/ExecutionStatistic/seg0/c1d1.dat b/ExecutionStatistic/seg0/c1d1.dat new file mode 100644 index 0000000..451f02f Binary files /dev/null and b/ExecutionStatistic/seg0/c1d1.dat differ diff --git a/ExecutionStatistic/seg0/c1e0.dat b/ExecutionStatistic/seg0/c1e0.dat new file mode 100644 index 0000000..761408d Binary files /dev/null and b/ExecutionStatistic/seg0/c1e0.dat differ diff --git a/ExecutionStatistic/seg0/c1f1.dat b/ExecutionStatistic/seg0/c1f1.dat new file mode 100644 index 0000000..78d701f Binary files /dev/null and b/ExecutionStatistic/seg0/c1f1.dat differ diff --git a/ExecutionStatistic/seg0/c20.dat b/ExecutionStatistic/seg0/c20.dat new file mode 100644 index 0000000..57f18e8 Binary files /dev/null and b/ExecutionStatistic/seg0/c20.dat differ diff --git a/ExecutionStatistic/seg0/c200.dat b/ExecutionStatistic/seg0/c200.dat new file mode 100644 index 0000000..c3a7808 Binary files /dev/null and b/ExecutionStatistic/seg0/c200.dat differ diff --git a/ExecutionStatistic/seg0/c211.dat b/ExecutionStatistic/seg0/c211.dat new file mode 100644 index 0000000..54e1586 Binary files /dev/null and b/ExecutionStatistic/seg0/c211.dat differ diff --git a/ExecutionStatistic/seg0/c221.dat b/ExecutionStatistic/seg0/c221.dat new file mode 100644 index 0000000..59900bc Binary files /dev/null and b/ExecutionStatistic/seg0/c221.dat differ diff --git a/ExecutionStatistic/seg0/c230.dat b/ExecutionStatistic/seg0/c230.dat new file mode 100644 index 0000000..5fee972 Binary files /dev/null and b/ExecutionStatistic/seg0/c230.dat differ diff --git a/ExecutionStatistic/seg0/c241.dat b/ExecutionStatistic/seg0/c241.dat new file mode 100644 index 0000000..e11dd66 Binary files /dev/null and b/ExecutionStatistic/seg0/c241.dat differ diff --git a/ExecutionStatistic/seg0/c251.dat b/ExecutionStatistic/seg0/c251.dat new file mode 100644 index 0000000..ec65ace Binary files /dev/null and b/ExecutionStatistic/seg0/c251.dat differ diff --git a/ExecutionStatistic/seg0/c260.dat b/ExecutionStatistic/seg0/c260.dat new file mode 100644 index 0000000..25f81fd Binary files /dev/null and b/ExecutionStatistic/seg0/c260.dat differ diff --git a/ExecutionStatistic/seg0/c271.dat b/ExecutionStatistic/seg0/c271.dat new file mode 100644 index 0000000..51cde57 Binary files /dev/null and b/ExecutionStatistic/seg0/c271.dat differ diff --git a/ExecutionStatistic/seg0/c281.dat b/ExecutionStatistic/seg0/c281.dat new file mode 100644 index 0000000..cfed875 Binary files /dev/null and b/ExecutionStatistic/seg0/c281.dat differ diff --git a/ExecutionStatistic/seg0/c290.dat b/ExecutionStatistic/seg0/c290.dat new file mode 100644 index 0000000..4d10db4 Binary files /dev/null and b/ExecutionStatistic/seg0/c290.dat differ diff --git a/ExecutionStatistic/seg0/c2a1.dat b/ExecutionStatistic/seg0/c2a1.dat new file mode 100644 index 0000000..8e2ed6a Binary files /dev/null and b/ExecutionStatistic/seg0/c2a1.dat differ diff --git a/ExecutionStatistic/seg0/c2b1.dat b/ExecutionStatistic/seg0/c2b1.dat new file mode 100644 index 0000000..2a29692 Binary files /dev/null and b/ExecutionStatistic/seg0/c2b1.dat differ diff --git a/ExecutionStatistic/seg0/c2c1.dat b/ExecutionStatistic/seg0/c2c1.dat new file mode 100644 index 0000000..5511575 Binary files /dev/null and b/ExecutionStatistic/seg0/c2c1.dat differ diff --git a/ExecutionStatistic/seg0/c2d0.dat b/ExecutionStatistic/seg0/c2d0.dat new file mode 100644 index 0000000..4adc6e4 Binary files /dev/null and b/ExecutionStatistic/seg0/c2d0.dat differ diff --git a/ExecutionStatistic/seg0/c2e1.dat b/ExecutionStatistic/seg0/c2e1.dat new file mode 100644 index 0000000..b37b9b2 Binary files /dev/null and b/ExecutionStatistic/seg0/c2e1.dat differ diff --git a/ExecutionStatistic/seg0/c2f0.dat b/ExecutionStatistic/seg0/c2f0.dat new file mode 100644 index 0000000..d854b4b Binary files /dev/null and b/ExecutionStatistic/seg0/c2f0.dat differ diff --git a/ExecutionStatistic/seg0/c300.dat b/ExecutionStatistic/seg0/c300.dat new file mode 100644 index 0000000..2053e01 Binary files /dev/null and b/ExecutionStatistic/seg0/c300.dat differ diff --git a/ExecutionStatistic/seg0/c31.dat b/ExecutionStatistic/seg0/c31.dat new file mode 100644 index 0000000..3e7eab8 Binary files /dev/null and b/ExecutionStatistic/seg0/c31.dat differ diff --git a/ExecutionStatistic/seg0/c311.dat b/ExecutionStatistic/seg0/c311.dat new file mode 100644 index 0000000..f60c260 Binary files /dev/null and b/ExecutionStatistic/seg0/c311.dat differ diff --git a/ExecutionStatistic/seg0/c321.dat b/ExecutionStatistic/seg0/c321.dat new file mode 100644 index 0000000..a9d7453 Binary files /dev/null and b/ExecutionStatistic/seg0/c321.dat differ diff --git a/ExecutionStatistic/seg0/c331.dat b/ExecutionStatistic/seg0/c331.dat new file mode 100644 index 0000000..85ee72b Binary files /dev/null and b/ExecutionStatistic/seg0/c331.dat differ diff --git a/ExecutionStatistic/seg0/c340.dat b/ExecutionStatistic/seg0/c340.dat new file mode 100644 index 0000000..d99b11a Binary files /dev/null and b/ExecutionStatistic/seg0/c340.dat differ diff --git a/ExecutionStatistic/seg0/c351.dat b/ExecutionStatistic/seg0/c351.dat new file mode 100644 index 0000000..f822f4c Binary files /dev/null and b/ExecutionStatistic/seg0/c351.dat differ diff --git a/ExecutionStatistic/seg0/c361.dat b/ExecutionStatistic/seg0/c361.dat new file mode 100644 index 0000000..b5c8f25 Binary files /dev/null and b/ExecutionStatistic/seg0/c361.dat differ diff --git a/ExecutionStatistic/seg0/c371.dat b/ExecutionStatistic/seg0/c371.dat new file mode 100644 index 0000000..ad11f01 Binary files /dev/null and b/ExecutionStatistic/seg0/c371.dat differ diff --git a/ExecutionStatistic/seg0/c380.dat b/ExecutionStatistic/seg0/c380.dat new file mode 100644 index 0000000..35379f4 Binary files /dev/null and b/ExecutionStatistic/seg0/c380.dat differ diff --git a/ExecutionStatistic/seg0/c391.dat b/ExecutionStatistic/seg0/c391.dat new file mode 100644 index 0000000..598eafc Binary files /dev/null and b/ExecutionStatistic/seg0/c391.dat differ diff --git a/ExecutionStatistic/seg0/c3a1.dat b/ExecutionStatistic/seg0/c3a1.dat new file mode 100644 index 0000000..1edb998 Binary files /dev/null and b/ExecutionStatistic/seg0/c3a1.dat differ diff --git a/ExecutionStatistic/seg0/c3b1.dat b/ExecutionStatistic/seg0/c3b1.dat new file mode 100644 index 0000000..b6f1651 Binary files /dev/null and b/ExecutionStatistic/seg0/c3b1.dat differ diff --git a/ExecutionStatistic/seg0/c3c0.dat b/ExecutionStatistic/seg0/c3c0.dat new file mode 100644 index 0000000..4d061cf Binary files /dev/null and b/ExecutionStatistic/seg0/c3c0.dat differ diff --git a/ExecutionStatistic/seg0/c3d1.dat b/ExecutionStatistic/seg0/c3d1.dat new file mode 100644 index 0000000..45c9fa2 Binary files /dev/null and b/ExecutionStatistic/seg0/c3d1.dat differ diff --git a/ExecutionStatistic/seg0/c3e1.dat b/ExecutionStatistic/seg0/c3e1.dat new file mode 100644 index 0000000..48f53e6 Binary files /dev/null and b/ExecutionStatistic/seg0/c3e1.dat differ diff --git a/ExecutionStatistic/seg0/c3f1.dat b/ExecutionStatistic/seg0/c3f1.dat new file mode 100644 index 0000000..08acdce Binary files /dev/null and b/ExecutionStatistic/seg0/c3f1.dat differ diff --git a/ExecutionStatistic/seg0/c400.dat b/ExecutionStatistic/seg0/c400.dat new file mode 100644 index 0000000..1e8976f Binary files /dev/null and b/ExecutionStatistic/seg0/c400.dat differ diff --git a/ExecutionStatistic/seg0/c41.dat b/ExecutionStatistic/seg0/c41.dat new file mode 100644 index 0000000..78127c2 Binary files /dev/null and b/ExecutionStatistic/seg0/c41.dat differ diff --git a/ExecutionStatistic/seg0/c411.dat b/ExecutionStatistic/seg0/c411.dat new file mode 100644 index 0000000..8aba2fb Binary files /dev/null and b/ExecutionStatistic/seg0/c411.dat differ diff --git a/ExecutionStatistic/seg0/c421.dat b/ExecutionStatistic/seg0/c421.dat new file mode 100644 index 0000000..65775ee Binary files /dev/null and b/ExecutionStatistic/seg0/c421.dat differ diff --git a/ExecutionStatistic/seg0/c430.dat b/ExecutionStatistic/seg0/c430.dat new file mode 100644 index 0000000..55c948d Binary files /dev/null and b/ExecutionStatistic/seg0/c430.dat differ diff --git a/ExecutionStatistic/seg0/c441.dat b/ExecutionStatistic/seg0/c441.dat new file mode 100644 index 0000000..3948b2a Binary files /dev/null and b/ExecutionStatistic/seg0/c441.dat differ diff --git a/ExecutionStatistic/seg0/c451.dat b/ExecutionStatistic/seg0/c451.dat new file mode 100644 index 0000000..fe1ab73 Binary files /dev/null and b/ExecutionStatistic/seg0/c451.dat differ diff --git a/ExecutionStatistic/seg0/c461.dat b/ExecutionStatistic/seg0/c461.dat new file mode 100644 index 0000000..e6d9854 Binary files /dev/null and b/ExecutionStatistic/seg0/c461.dat differ diff --git a/ExecutionStatistic/seg0/c51.dat b/ExecutionStatistic/seg0/c51.dat new file mode 100644 index 0000000..db67c98 Binary files /dev/null and b/ExecutionStatistic/seg0/c51.dat differ diff --git a/ExecutionStatistic/seg0/c60.dat b/ExecutionStatistic/seg0/c60.dat new file mode 100644 index 0000000..542e21c Binary files /dev/null and b/ExecutionStatistic/seg0/c60.dat differ diff --git a/ExecutionStatistic/seg0/c71.dat b/ExecutionStatistic/seg0/c71.dat new file mode 100644 index 0000000..5e7d85c Binary files /dev/null and b/ExecutionStatistic/seg0/c71.dat differ diff --git a/ExecutionStatistic/seg0/c81.dat b/ExecutionStatistic/seg0/c81.dat new file mode 100644 index 0000000..ed09765 Binary files /dev/null and b/ExecutionStatistic/seg0/c81.dat differ diff --git a/ExecutionStatistic/seg0/c90.dat b/ExecutionStatistic/seg0/c90.dat new file mode 100644 index 0000000..80251eb Binary files /dev/null and b/ExecutionStatistic/seg0/c90.dat differ diff --git a/ExecutionStatistic/seg0/ca1.dat b/ExecutionStatistic/seg0/ca1.dat new file mode 100644 index 0000000..318f909 Binary files /dev/null and b/ExecutionStatistic/seg0/ca1.dat differ diff --git a/ExecutionStatistic/seg0/cb1.dat b/ExecutionStatistic/seg0/cb1.dat new file mode 100644 index 0000000..d1b3dbb Binary files /dev/null and b/ExecutionStatistic/seg0/cb1.dat differ diff --git a/ExecutionStatistic/seg0/cc0.dat b/ExecutionStatistic/seg0/cc0.dat new file mode 100644 index 0000000..2268720 Binary files /dev/null and b/ExecutionStatistic/seg0/cc0.dat differ diff --git a/ExecutionStatistic/seg0/cd1.dat b/ExecutionStatistic/seg0/cd1.dat new file mode 100644 index 0000000..d919a1b Binary files /dev/null and b/ExecutionStatistic/seg0/cd1.dat differ diff --git a/ExecutionStatistic/seg0/ce1.dat b/ExecutionStatistic/seg0/ce1.dat new file mode 100644 index 0000000..299e0c4 Binary files /dev/null and b/ExecutionStatistic/seg0/ce1.dat differ diff --git a/ExecutionStatistic/seg0/cf0.dat b/ExecutionStatistic/seg0/cf0.dat new file mode 100644 index 0000000..1c00d1d Binary files /dev/null and b/ExecutionStatistic/seg0/cf0.dat differ diff --git a/TODO.txt b/TODO.txt index 5eec1b6..e08270c 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,7 @@ TODO: +Current stat collector has to rely on file dates! + Test all WS as names from Executables were removed Add default names for input and output in every executable and use them consistently diff --git a/WEB-INF/lib/derby.jar b/WEB-INF/lib/derby.jar new file mode 100644 index 0000000..631cdd7 Binary files /dev/null and b/WEB-INF/lib/derby.jar differ diff --git a/engine/compbio/engine/ExecutionStatCollector.java b/engine/compbio/engine/ExecutionStatCollector.java deleted file mode 100644 index 2088f52..0000000 --- a/engine/compbio/engine/ExecutionStatCollector.java +++ /dev/null @@ -1,203 +0,0 @@ -package compbio.engine; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import compbio.engine.client.ConfExecutable; -import compbio.engine.conf.PropertyHelperManager; -import compbio.metadata.JobStatus; -import compbio.util.FileUtil; -import compbio.util.PropertyHelper; - -/** - * Number of runs of each WS = number of folders with name - * - * Number of successful runs = all runs with no result file - * - * Per period of time = limit per file creating time Runtime (avg/max) = - * - * started time - finished time Task & result size = result.size - * - * Abandoned runs - not collected runs - * - * Cancelled runs - cancelled - * - * Cluster vs local runs - * - * Reasons for failure = look in the err out? - * - * - * Metadata required: - * - * work directory for local and cluster tasks = from Helper or cmd parameter. WS - * names - enumeration. Status file names and content. - * - * @author pvtroshin - * - */ -public class ExecutionStatCollector { - - private static final Logger log = Logger - .getLogger(ExecutionStatCollector.class); - - static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); - - static String getClusterJobDir() { - String clusterdir = ph.getProperty("cluster.tmp.directory"); - if (clusterdir != null) { - clusterdir.trim(); - } - return clusterdir; - } - - static String getLocalJobDir() { - String locdir = ph.getProperty("local.tmp.directory"); - if (locdir != null) { - locdir.trim(); - } - return locdir; - } - - /** - * - * @param args - */ - public static void main(String[] args) { - - String workDir = PropertyHelperManager.getLocalPath() - + getLocalJobDir().trim(); - System.out.println(workDir); - File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout", - directories); - for (File file : files) { - JobDirectory jd = new JobDirectory(file); - System.out.println(jd.getJobReport()); - } - } - - static FileFilter directories = new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.isDirectory(); - } - }; - - static class JobDirectory { - static final int UNDEFINED = -1; - - File jobdir; - Map files = new HashMap(); - - public JobDirectory(File directory) { - this.jobdir = directory; - for (File f : jobdir.listFiles()) { - files.put(f.getName(), f); - } - } - - public boolean hasStatus(JobStatus status) { - return files.containsKey(status.toString()); - } - - long getStartTime() { - long starttime = UNDEFINED; - try { - File startfile = files.get(JobStatus.STARTED.toString()); - if (startfile == null) { - startfile = files.get(JobStatus.SUBMITTED.toString()); - } - if (startfile != null) { - String start = FileUtil.readFileToString(startfile); - starttime = Long.parseLong(start.trim()); - } - } catch (IOException e) { - log.log(Level.WARN, - "Cannot parse start time: " + e.getMessage(), e); - } catch (NumberFormatException e) { - log.log(Level.WARN, - "Cannot parse start time: " + e.getMessage(), e); - } - return starttime; - } - long getFinishedTime() { - long ftime = UNDEFINED; - try { - File finished = files.get(JobStatus.FINISHED.toString()); - if (finished != null) { - String start = FileUtil.readFileToString(finished); - ftime = Long.parseLong(start.trim()); - // System.out.println("f " + ftime); - } - } catch (IOException e) { - log.log(Level.WARN, - "Cannot parse finished time: " + e.getMessage(), e); - } catch (NumberFormatException e) { - log.log(Level.WARN, - "Cannot parse finished time: " + e.getMessage(), e); - } - return ftime; - } - - public int getRuntime() { - return (int) (getFinishedTime() - getStartTime()); - } - - String getWSName() { - String name = jobdir.getName().split("#")[0]; - if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { - assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; - name = name.substring(1); - } - return name; - } - - // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out - // Probcons task:fasta.in result:alignment.out - /* - * TODO replace with Universal names for WS! - */ - long getResultSize() { - String name = getWSName(); - File f = null; - if (name.equalsIgnoreCase("Probcons")) { - f = files.get("alignment.out"); - } - f = files.get("fasta.out"); - if (f != null) { - return f.length(); - } - return UNDEFINED; - } - - long getInputSize() { - File input = files.get("fasta.in"); - if (input != null) { - return input.length(); - } - return UNDEFINED; - } - - String getJobReport() { - String report = "JOB: " + jobdir.getName() + "\n"; - if (getStartTime() != UNDEFINED) { - report += "Started " + new Date(getStartTime()) + "\n"; - } - if (getFinishedTime() != UNDEFINED) { - report += "Finished " + new Date(getFinishedTime()) + "\n"; - } - if (getStartTime() != UNDEFINED && getFinishedTime() != UNDEFINED) { - report += "Runtime " + getRuntime() + "\n"; - } - report += "Input size " + getInputSize() + "\n"; - report += "Result size " + getResultSize() + "\n"; - return report; - } - } -} diff --git a/webservices/compbio/ws/execstat/ExecutionStatCollector.java b/webservices/compbio/ws/execstat/ExecutionStatCollector.java new file mode 100644 index 0000000..74218ce --- /dev/null +++ b/webservices/compbio/ws/execstat/ExecutionStatCollector.java @@ -0,0 +1,254 @@ +package compbio.ws.execstat; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; + +import compbio.engine.client.ConfExecutable; +import compbio.engine.conf.PropertyHelperManager; +import compbio.metadata.JobStatus; +import compbio.util.FileUtil; +import compbio.util.PropertyHelper; +import compbio.ws.client.Services; + +/** + * Number of runs of each WS = number of folders with name + * + * Number of successful runs = all runs with no result file + * + * Per period of time = limit per file creating time Runtime (avg/max) = + * + * started time - finished time + * + * Task & result size = result.size + * + * Abandoned runs - not collected runs + * + * Cancelled runs - cancelled + * + * Cluster vs local runs + * + * Reasons for failure = look in the err out? + * + * + * Metadata required: + * + * work directory for local and cluster tasks = from Helper or cmd parameter. WS + * names - enumeration. Status file names and content. + * + * @author pvtroshin + * + */ +public class ExecutionStatCollector { + + static final int UNDEFINED = -1; + + private static final Logger log = Logger + .getLogger(ExecutionStatCollector.class); + + static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); + + static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); + + static String getClusterJobDir() { + String clusterdir = ph.getProperty("cluster.tmp.directory"); + if (clusterdir != null) { + clusterdir.trim(); + } + return clusterdir; + } + + static void updateTime(File statFile) throws IOException { + long lastMod = statFile.lastModified(); + FileWriter fw = new FileWriter(statFile); + fw.write(new Long(lastMod).toString()); + fw.close(); + } + + static String getLocalJobDir() { + String locdir = ph.getProperty("local.tmp.directory"); + if (locdir != null) { + locdir.trim(); + } + return locdir; + } + + /** + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + // updateTime(new File( + // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED")); + + String workDir = PropertyHelperManager.getLocalPath() + + getLocalJobDir().trim(); + System.out.println(workDir); + File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout", + directories); + List stats = new ArrayList(); + for (File file : files) { + JobDirectory jd = new JobDirectory(file); + stats.add(jd.getJobStat()); + // System.out.println(jd.getJobStat().getJobReportTabulated()); + } + StatProcessor sp = new StatProcessor(stats); + System.out.println(sp.reportStat()); + System.out.println(); + System.out.println("!!!!!!!!!!!!!!!!!!"); + System.out.println(); + System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat()); + } + + static FileFilter directories = new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.isDirectory(); + } + }; + + static class JobDirectory { + + File jobdir; + Map files = new HashMap(); + + public JobDirectory(File directory) { + this.jobdir = directory; + for (File f : jobdir.listFiles()) { + files.put(f.getName(), f); + } + } + + public boolean hasStatus(JobStatus status) { + return files.containsKey(status.toString()); + } + + boolean isCollected() { + return hasStatus(JobStatus.COLLECTED); + } + + boolean isCancelled() { + return hasStatus(JobStatus.CANCELLED); + } + + long getStartTime() { + long starttime = UNDEFINED; + File startfile = files.get(JobStatus.STARTED.toString()); + if (startfile == null) { + startfile = files.get(JobStatus.SUBMITTED.toString()); + } + if (startfile != null) { + starttime = startfile.lastModified(); + /* + * String start = FileUtil.readFileToString(startfile); + * starttime = Long.parseLong(start.trim()); + */ + } + return starttime; + } + + long getFinishedTime() { + long ftime = UNDEFINED; + File finished = files.get(JobStatus.FINISHED.toString()); + if (finished != null) { + ftime = finished.lastModified(); + /* + * String start = FileUtil.readFileToString(finished); ftime = + * Long.parseLong(start.trim()); + */ + // System.out.println("f " + ftime); + } + /* + * } catch (IOException e) { log.log(Level.WARN, + * "Cannot parse finished time: " + e.getMessage(), e); } catch + * (NumberFormatException e) { log.log(Level.WARN, + * "Cannot parse finished time: " + e.getMessage(), e); } + */ + return ftime; + } + + String getWSName() { + String name = jobdir.getName().split("#")[0]; + if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { + assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; + name = name.substring(1); + } + if (name.startsWith("ClustalW")) { + name = name.trim().substring(name.length() - 1); + } + return name; + } + + Services getService() { + return Services.getService(getWSName() + "WS"); + } + // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out + // Probcons task:fasta.in result:alignment.out + /* + * TODO replace with Universal names for WS! + */ + long getResultSize() { + String name = getWSName(); + File f = null; + if (name.equalsIgnoreCase("Probcons")) { + f = files.get("alignment.out"); + } + f = files.get("fasta.out"); + if (f != null) { + return f.length(); + } + return UNDEFINED; + } + + long getInputSize() { + File input = files.get("fasta.in"); + if (input != null) { + return input.length(); + } + return UNDEFINED; + } + + StatProcessor.JobStat getJobStat() { + return new StatProcessor.JobStat(getService(), jobdir.getName(), + getStartTime(), getFinishedTime(), getInputSize(), + getResultSize(), isCollected(), isCancelled()); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((jobdir == null) ? 0 : jobdir.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + JobDirectory other = (JobDirectory) obj; + if (jobdir == null) { + if (other.jobdir != null) + return false; + } else if (!jobdir.equals(other.jobdir)) + return false; + return true; + } + + } +} diff --git a/webservices/compbio/ws/execstat/StatProcessor.java b/webservices/compbio/ws/execstat/StatProcessor.java new file mode 100644 index 0000000..0697eb8 --- /dev/null +++ b/webservices/compbio/ws/execstat/StatProcessor.java @@ -0,0 +1,260 @@ +package compbio.ws.execstat; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; + +import compbio.ws.client.Services; + +public class StatProcessor { + + List stats; + + StatProcessor(List stats) { + this.stats = stats; + } + + List getAbandonedJobs() { + List abJobs = new ArrayList(); + for (JobStat js : stats) { + if (!js.isCollected) { + abJobs.add(js); + } + } + return abJobs; + } + + List getCancelledJobs() { + List abJobs = new ArrayList(); + for (JobStat js : stats) { + if (js.isCancelled) { + abJobs.add(js); + } + } + return abJobs; + } + + List sortByRuntime() { + List abJobs = new ArrayList(stats); + Collections.sort(abJobs, JobStat.RUNTIME); + return abJobs; + } + + List sortByStartTime() { + List abJobs = new ArrayList(stats); + Collections.sort(abJobs, JobStat.STARTTIME); + return abJobs; + } + + List sortByResultSize() { + List abJobs = new ArrayList(stats); + Collections.sort(abJobs, JobStat.RESULTSIZE); + return abJobs; + } + + int getJobNumber() { + return stats.size(); + } + + public StatProcessor getSingleWSStat(Services webService) { + List wsStat = new ArrayList(); + for (JobStat js : stats) { + if (js.webService == webService) { + wsStat.add(js); + } + } + return new StatProcessor(wsStat); + } + + long getTotalRuntime() { + long counter = 0; + for (JobStat js : stats) { + int jobtime = js.getRuntime(); + if (jobtime != ExecutionStatCollector.UNDEFINED) { + counter += jobtime; + } + } + return counter; + } + + List getUnsuccessfulJobs() { + List aJobs = new ArrayList(); + for (JobStat js : stats) { + int jobtime = js.getRuntime(); + if (js.resultSize == ExecutionStatCollector.UNDEFINED) { + aJobs.add(js); + } + } + return aJobs; + } + + public String reportStat() { + String report = "Total Jobs: " + getJobNumber() + "\n"; + report += "Abandoned Jobs: " + getAbandonedJobs().size() + "\n"; + report += "Cancelled Jobs: " + getCancelledJobs().size() + "\n"; + report += "Total Runtime (s): " + getTotalRuntime() + "\n"; + report += "Unsuccessful Jobs: " + getUnsuccessfulJobs().size() + "\n"; + report += "10 longest jobs: \n\n" + sortByRuntime().subList(0, 9) + + "\n"; + report += "10 biggest jobs: \n\n" + sortByResultSize().subList(0, 9) + + "\n"; + return report; + } + + static class JobStat { + + static final Comparator RUNTIME = new Comparator() { + @Override + public int compare(JobStat o1, JobStat o2) { + return new Integer(o2.getRuntime()).compareTo(o1.getRuntime()); + } + }; + + static final Comparator STARTTIME = new Comparator() { + @Override + public int compare(JobStat o1, JobStat o2) { + return new Long(o1.start).compareTo(o2.start); + } + }; + + static final Comparator RESULTSIZE = new Comparator() { + @Override + public int compare(JobStat o1, JobStat o2) { + return new Long(o2.resultSize).compareTo(o1.resultSize); + } + }; + + Services webService; + String jobname; + long start; + long finish; + long inputSize; + long resultSize; + boolean isCollected; + boolean isCancelled; + + JobStat(Services webService, String jobname, long start, long finish, + long inputSize, long resultSize, boolean isCollected, + boolean isCancelled) { + super(); + this.webService = webService; + this.jobname = jobname; + this.start = start; + this.finish = finish; + this.inputSize = inputSize; + this.resultSize = resultSize; + this.isCollected = isCollected; + this.isCancelled = isCancelled; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (finish ^ (finish >>> 32)); + result = prime * result + (int) (inputSize ^ (inputSize >>> 32)); + result = prime * result + (isCancelled ? 1231 : 1237); + result = prime * result + (isCollected ? 1231 : 1237); + result = prime * result + + ((jobname == null) ? 0 : jobname.hashCode()); + result = prime * result + (int) (resultSize ^ (resultSize >>> 32)); + result = prime * result + (int) (start ^ (start >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + JobStat other = (JobStat) obj; + if (finish != other.finish) + return false; + if (inputSize != other.inputSize) + return false; + if (isCancelled != other.isCancelled) + return false; + if (isCollected != other.isCollected) + return false; + if (jobname == null) { + if (other.jobname != null) + return false; + } else if (!jobname.equals(other.jobname)) + return false; + if (resultSize != other.resultSize) + return false; + if (start != other.start) + return false; + return true; + } + + public int getRuntime() { + if (start != ExecutionStatCollector.UNDEFINED + && finish != ExecutionStatCollector.UNDEFINED) { + return (int) (finish - start) / 1000; + } + return ExecutionStatCollector.UNDEFINED; + } + + @Override + public String toString() { + return getJobReport(); + } + + String getJobReport() { + String report = "WS: " + webService + "\n"; + report += "JOB: " + jobname + "\n"; + if (start != ExecutionStatCollector.UNDEFINED) { + report += "Started " + new Date(start) + "\n"; + } + if (finish != ExecutionStatCollector.UNDEFINED) { + report += "Finished " + new Date(finish) + "\n"; + } + if (start != ExecutionStatCollector.UNDEFINED + && finish != ExecutionStatCollector.UNDEFINED) { + report += "Runtime " + getRuntime() + "\n"; + } + report += "Input size " + inputSize + "\n"; + report += "Result size " + resultSize + "\n"; + report += "Collected? " + isCollected + "\n"; + return report; + } + + /** + * Header Job Started Finished Runtime Input Result + */ + String getJobReportTabulated() { + String report = webService + "\t"; + report += jobname + "\t"; + if (start != ExecutionStatCollector.UNDEFINED) { + report += ExecutionStatCollector.DF.format(new Date(start)) + + "\t"; + } else { + report += ExecutionStatCollector.UNDEFINED + "\t"; + } + if (finish != ExecutionStatCollector.UNDEFINED) { + report += ExecutionStatCollector.DF.format(new Date(finish)) + + "\t"; + } else { + report += ExecutionStatCollector.UNDEFINED + "\t"; + } + if (start != ExecutionStatCollector.UNDEFINED + && finish != ExecutionStatCollector.UNDEFINED) { + report += getRuntime() + "\t"; + } else { + report += ExecutionStatCollector.UNDEFINED + "\t"; + } + report += inputSize + "\t"; + report += resultSize + "\t"; + report += isCollected + "\t"; + return report; + } + + } + +} diff --git a/webservices/compbio/ws/execstat/StatWriter.java b/webservices/compbio/ws/execstat/StatWriter.java new file mode 100644 index 0000000..d1b461e --- /dev/null +++ b/webservices/compbio/ws/execstat/StatWriter.java @@ -0,0 +1,100 @@ +package compbio.ws.execstat; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; + +import compbio.ws.execstat.StatProcessor.JobStat; + +public class StatWriter { + + /* the default framework is embedded */ + // private final String framework = "embedded"; + private static final String driver = "org.apache.derby.jdbc.EmbeddedDriver"; + private static final String protocol = "jdbc:derby:"; + private static final String statDBName = "ExecutionStatistic"; + + static Connection getDBConnection() throws SQLException { + // TODO + System.setProperty("derby.system.home", "."); + + Connection conn = DriverManager.getConnection(protocol + statDBName + + ";create=true"); + + // We want to control transactions manually. Autocommit is on by + // default in JDBC. + conn.setAutoCommit(true); + return conn; + } + + // ServiceName,jobname,start,finish,inputSize,resultSize,isCancelled,isCollected + static void createStatTable() throws SQLException { + Connection conn = getDBConnection(); + /* + * Creating a statement object that we can use for running various SQL + * statements commands against the database. + */ + Statement s = conn.createStatement(); + String create = "create table exec_stat(" + + "number INT GENERATED ALWAYS AS IDENTITY," + + "service_name VARCHAR(15) NOT NULL, " + + "job_id VARCHAR(35) NOT NULL PRIMARY KEY, " + + "start TIMESTAMP," + "finish TIMESTAMP," + + "inputsize BIGINT," + "resultsize BIGINT," + + "isCancelled SMALLINT NOT NULL," + + "isCollected SMALLINT NOT NULL)"; + // We create a table... + System.out.println(create); + s.execute(create); + s.close(); + conn.close(); + } + + static void insertData(JobStat jobstatus) throws SQLException { + Connection conn = getDBConnection(); + String insert = "insert into exec_stat (service_name, job_id, start, finish, " + + "inputsize, resultsize, isCancelled, isCollected) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ? )"; + PreparedStatement pstm = conn.prepareStatement(insert); + + pstm.setString(1, "webservice"); + pstm.setString(2, "@Clustal#980273495452357"); + pstm.setTimestamp(3, new Timestamp(190385934834l)); + pstm.setTimestamp(4, new Timestamp(190332423423l)); + pstm.setLong(5, 1232); + pstm.setLong(6, 1432422); + pstm.setShort(7, (short) 1); + pstm.setShort(8, (short) 0); + pstm.executeUpdate(); + pstm.close(); + conn.close(); + } + + void shutdownDBServer() { + // ## DATABASE SHUTDOWN SECTION ## + /*** + * In embedded mode, an application should shut down Derby. Shutdown + * throws the XJ015 exception to confirm success. + ***/ + boolean gotSQLExc = false; + try { + DriverManager.getConnection("jdbc:derby:;shutdown=true"); + } catch (SQLException se) { + if (se.getSQLState().equals("XJ015")) { + gotSQLExc = true; + } + } + if (!gotSQLExc) { + System.out.println("Database did not shut down normally"); + } else { + System.out.println("Database shut down normally"); + } + } + public static void main(String[] args) throws SQLException { + // createStatTable(); + insertData(null); + } +}