From 5289bedee673d95739fdf622dba7be64f8c2df13 Mon Sep 17 00:00:00 2001 From: pvtroshin Date: Mon, 30 May 2011 17:17:34 +0000 Subject: [PATCH] Further work to enable stat collection and display git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@4193 e3abac25-378b-4346-85de-24260fe3988d --- WEB-INF/web.xml | 27 +++++- .../collector/ExecutionStatCollectorTester.java | 90 +++++++++++++++++++- .../stat/collector/ExecutionStatCollector.java | 26 +----- webservices/compbio/stat/collector/StatDB.java | 15 ++-- webservices/compbio/stat/servlet/Joblist.java | 2 + .../compbio/stat/servlet/StatisticCollector.java | 34 +++++--- webservices/compbio/ws/client/Services.java | 72 +++++++++++++++- 7 files changed, 220 insertions(+), 46 deletions(-) diff --git a/WEB-INF/web.xml b/WEB-INF/web.xml index b03b1fe..4f75bd5 100644 --- a/WEB-INF/web.xml +++ b/WEB-INF/web.xml @@ -21,6 +21,25 @@ 20 + + This is a standard tomcat 'default' servlet for making listings + listings + org.apache.catalina.servlets.DefaultServlet + + debug + 0 + + + readonly + true + + + listings + true + + 1 + + Display pre-calculated accounting info DisplayStat @@ -92,7 +111,12 @@ com.sun.xml.ws.transport.http.servlet.WSServlet 1 - + + + listings + / + + DisplayStat /DisplayStat @@ -157,6 +181,7 @@ Administrator pages + / /DisplayStat /Joblist /AnnualStat diff --git a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java index d209e13..9773935 100644 --- a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java +++ b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java @@ -1,21 +1,35 @@ package compbio.stat.collector; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import java.io.File; +import java.util.Date; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.testng.Assert; import org.testng.annotations.Test; import compbio.metadata.AllTestSuit; +import compbio.stat.collector.ExecutionStatCollector.JobDirectory; public class ExecutionStatCollectorTester { + final static String LOCAL_JOBS = AllTestSuit.TEST_DATA_PATH_ABSOLUTE + + File.separator + "ljobs"; + final static String CLUSTER_JOBS = AllTestSuit.TEST_DATA_PATH_ABSOLUTE + + File.separator + "jobs"; @Test public void testCollectStat() { ExecutionStatCollector local_jobs = new ExecutionStatCollector( - AllTestSuit.TEST_DATA_PATH_ABSOLUTE + File.separator + "ljobs"); + LOCAL_JOBS, 1); ExecutionStatCollector cl_jobs = new ExecutionStatCollector( - AllTestSuit.TEST_DATA_PATH_ABSOLUTE + File.separator + "jobs"); + CLUSTER_JOBS, 24); + StatProcessor local_stats = local_jobs.getStats(); StatProcessor cl_stats = cl_jobs.getStats(); // System.out.println("L: " + local_stats.stats); @@ -29,7 +43,77 @@ public class ExecutionStatCollectorTester { assertEquals(2, local_stats.getAbandonedJobs().size()); } - public void testUpdateStat() { + + @Test + public void testUpdateStatTester() { + + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + ScheduledFuture sf = executor.scheduleAtFixedRate( + new ExecutionStatCollector(LOCAL_JOBS, 1), 1, 10, + TimeUnit.SECONDS); + + try { + Thread.sleep(1000 * 35); + executor.shutdown(); + executor.awaitTermination(300, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + /** + * This test will fail in time as it relies on last access time for file in + * the filesystem! This OK as it has been tested by then. + * + */ + @Test + public void testHasTimedOut() { + ExecutionStatCollector ecol = new ExecutionStatCollector(LOCAL_JOBS, 1); + File f = new File(LOCAL_JOBS + File.separator + + "ClustalW#1015373448154965"); + File timedOut = new File(LOCAL_JOBS + File.separator + + "Mafft#7868649707286965"); + + JobDirectory jd = new JobDirectory(f); + JobDirectory timedOutDir = new JobDirectory(timedOut); + System.out.println("! " + new Date(f.lastModified())); + + assertTrue((System.currentTimeMillis() - f.lastModified()) + / (1000 * 60 * 60) < 1); + assertFalse((System.currentTimeMillis() - timedOut.lastModified()) + / (1000 * 60 * 60) < 1); + assertFalse(ecol.hasTimedOut(jd)); + assertTrue(ecol.hasTimedOut(timedOutDir)); + } + + @Test + public void testHasCompleted() { + ExecutionStatCollector ecol_no_timeout = new ExecutionStatCollector( + LOCAL_JOBS, 10000000); + ExecutionStatCollector ecol = new ExecutionStatCollector(LOCAL_JOBS, 1); + + File normal = new File(LOCAL_JOBS + File.separator + + "ClustalW#100368900075204"); + File finished = new File(LOCAL_JOBS + File.separator + + "ClustalW#1015373448154965"); + File cancelled = new File(LOCAL_JOBS + File.separator + + "Mafft#7918237850044965"); + File noresult = new File(LOCAL_JOBS + File.separator + + "ClustalW#1015343425414965"); + + JobDirectory dnormal = new JobDirectory(normal); + JobDirectory dfinished = new JobDirectory(finished); + JobDirectory dcancelled = new JobDirectory(cancelled); + + JobDirectory dnoresult = new JobDirectory(noresult); + + assertTrue(ecol.hasCompleted(dnormal)); + assertTrue(ecol.hasCompleted(dfinished)); + assertTrue(ecol.hasCompleted(dcancelled)); + + assertTrue(ecol.hasCompleted(dnoresult)); + assertFalse(ecol_no_timeout.hasCompleted(dnoresult)); } } diff --git a/webservices/compbio/stat/collector/ExecutionStatCollector.java b/webservices/compbio/stat/collector/ExecutionStatCollector.java index 140de7c..bf3130a 100644 --- a/webservices/compbio/stat/collector/ExecutionStatCollector.java +++ b/webservices/compbio/stat/collector/ExecutionStatCollector.java @@ -15,10 +15,8 @@ import java.util.Set; import org.apache.log4j.Logger; -import compbio.engine.client.ConfExecutable; import compbio.engine.client.Executable; import compbio.metadata.JobStatus; -import compbio.runner.msa.ClustalW; import compbio.util.FileUtil; import compbio.ws.client.Services; @@ -239,26 +237,8 @@ public class ExecutionStatCollector implements Runnable { return ftime; } - @SuppressWarnings("unchecked") - Class> getWSRunnerName() { - String name = jobdir.getName().split("#")[0]; - try { - if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { - assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; - name = name.substring(1); - } - name = ClustalW.class.getPackage().getName() + "." + name; - return (Class>) Class.forName(name); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - throw new RuntimeException( - "Cannot match the directory name to the executable! Executable name is " - + name); - } - } - private Services getService() { - return Services.getService(getWSRunnerName()); + return Services.getServiceByJobDirectory(jobdir); } // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out @@ -267,7 +247,9 @@ public class ExecutionStatCollector implements Runnable { * TODO replace with Universal names for WS! */ long getResultSize() { - Class> name = getWSRunnerName(); + Class> name = Services + .getRunnerByJobDirectory(jobdir); + File f = null; if (name.getSimpleName().equalsIgnoreCase("Probcons")) { f = files.get("alignment.out"); diff --git a/webservices/compbio/stat/collector/StatDB.java b/webservices/compbio/stat/collector/StatDB.java index 0f851ce..56ce9c1 100644 --- a/webservices/compbio/stat/collector/StatDB.java +++ b/webservices/compbio/stat/collector/StatDB.java @@ -49,13 +49,11 @@ public class StatDB { + ";create=false"); conn.setAutoCommit(true); - - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - shutdownDBServer(); - } - }); + /* + * Runtime.getRuntime().addShutdownHook(new Thread() { + * + * @Override public void run() { shutdownDBServer(); } }); + */ } return conn; } @@ -114,7 +112,8 @@ public class StatDB { } void insertData(Set jobstatus) throws SQLException { - System.out.println("Inserting " + jobstatus.size()); + log.info("Inserting " + jobstatus.size() + + " new records into the statistics database"); conn.setAutoCommit(false); String insert = "insert into exec_stat (service_name, cluster_job_id, job_id, start, finish, " diff --git a/webservices/compbio/stat/servlet/Joblist.java b/webservices/compbio/stat/servlet/Joblist.java index 1b73018..4ceb2b8 100644 --- a/webservices/compbio/stat/servlet/Joblist.java +++ b/webservices/compbio/stat/servlet/Joblist.java @@ -1,5 +1,6 @@ package compbio.stat.servlet; +import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.sql.Timestamp; @@ -95,6 +96,7 @@ public class Joblist extends HttpServlet { PropertyHelper helper = PropertyHelperManager.getPropertyHelper(); String clusterTempDir = helper.getProperty("cluster.tmp.directory") .trim(); + clusterTempDir = new File(clusterTempDir).getName(); String localTempDir = helper.getProperty("local.tmp.directory").trim(); // TODO include the time slice Timestamp startDate = new Timestamp(Long.parseLong(fromDate)); diff --git a/webservices/compbio/stat/servlet/StatisticCollector.java b/webservices/compbio/stat/servlet/StatisticCollector.java index 0ffe77d..978d89a 100644 --- a/webservices/compbio/stat/servlet/StatisticCollector.java +++ b/webservices/compbio/stat/servlet/StatisticCollector.java @@ -14,6 +14,7 @@ import compbio.engine.conf.PropertyHelperManager; import compbio.stat.collector.ExecutionStatCollector; import compbio.stat.collector.StatDB; import compbio.util.PropertyHelper; +import compbio.util.Util; public class StatisticCollector implements ServletContextListener { @@ -50,24 +51,37 @@ public class StatisticCollector implements ServletContextListener { int clusterMaxRuntime = getClusterJobTimeOut(); int localMaxRuntime = getLocalJobTimeOut(); - String localWorkDir = getLocalJobDir(); + String localWorkDir = compbio.engine.client.Util + .convertToAbsolute(getLocalJobDir()); log.info("Initializing statistics collector"); - executor = Executors.newScheduledThreadPool(1); + executor = Executors.newScheduledThreadPool(2); if (collectClusterStats()) { + // TODO remove work out of the constructor Tomcat takes ages to + // start! ExecutionStatCollector clusterCollector = new ExecutionStatCollector( clusterWorkDir, clusterMaxRuntime); clustercf = executor.scheduleAtFixedRate(clusterCollector, 60, 24 * 60, TimeUnit.MINUTES); + // clustercf = executor.scheduleAtFixedRate(clusterCollector, 15, + // 30, + // TimeUnit.SECONDS); log.info("Collecting cluster statistics "); + } else { + log.info("Cluster statistics collector is disabled or not configured! "); } if (collectLocalStats()) { ExecutionStatCollector localCollector = new ExecutionStatCollector( localWorkDir, localMaxRuntime); + // localcf = executor.scheduleAtFixedRate(localCollector, 100, 60, + // TimeUnit.SECONDS); + localcf = executor.scheduleAtFixedRate(localCollector, 10, 24 * 60, TimeUnit.MINUTES); log.info("Collecting local statistics "); + } else { + log.info("Local statistics collector is disabled or not configured! "); } } @@ -102,28 +116,26 @@ public class StatisticCollector implements ServletContextListener { } private static String getStringProperty(String propName) { - String locdir = ph.getProperty(propName); - if (locdir != null) { - locdir = locdir.trim(); + if (propName != null) { + propName = propName.trim(); } - return locdir; + return propName; } static boolean collectClusterStats() { return getBooleanProperty(ph .getProperty("cluster.stat.collector.enable")); - } static boolean collectLocalStats() { return getBooleanProperty(ph.getProperty("local.stat.collector.enable")); } - private static boolean getBooleanProperty(String propName) { + private static boolean getBooleanProperty(String propValue) { boolean enabled = false; - if (propName != null) { - propName = propName.trim(); - enabled = Boolean.parseBoolean(propName); + if (!Util.isEmpty(propValue)) { + propValue = propValue.trim(); + enabled = Boolean.parseBoolean(propValue); } return enabled; } diff --git a/webservices/compbio/ws/client/Services.java b/webservices/compbio/ws/client/Services.java index 7f06da7..bff43e3 100644 --- a/webservices/compbio/ws/client/Services.java +++ b/webservices/compbio/ws/client/Services.java @@ -18,6 +18,7 @@ package compbio.ws.client; +import java.io.File; import java.net.URL; import javax.xml.namespace.QName; @@ -26,7 +27,17 @@ import javax.xml.ws.Service; import compbio.data.msa.JABAService; import compbio.data.msa.MsaWS; import compbio.data.msa.SequenceAnnotation; +import compbio.engine.client.ConfExecutable; import compbio.engine.client.Executable; +import compbio.runner.conservation.AACon; +import compbio.runner.disorder.Disembl; +import compbio.runner.disorder.GlobPlot; +import compbio.runner.disorder.Jronn; +import compbio.runner.msa.ClustalW; +import compbio.runner.msa.Mafft; +import compbio.runner.msa.Muscle; +import compbio.runner.msa.Probcons; +import compbio.runner.msa.Tcoffee; /** * List of web services currently supported by JABAWS version 2 @@ -45,7 +56,8 @@ public enum Services { return null; } - public static Services getService(Class> runnerClassName) { + public static Services getServiceByRunner( + Class> runnerClassName) { assert runnerClassName != null; String sname = runnerClassName.getSimpleName().toLowerCase(); for (Services service : Services.values()) { @@ -56,6 +68,64 @@ public enum Services { return null; } + public Class> getServiceImpl() { + switch (this) { + case AAConWS : + return AACon.class; + case ClustalWS : + return ClustalW.class; + case MafftWS : + return Mafft.class; + case MuscleWS : + return Muscle.class; + case TcoffeeWS : + return Tcoffee.class; + case ProbconsWS : + return Probcons.class; + case DisemblWS : + return Disembl.class; + case GlobPlotWS : + return GlobPlot.class; + case JronnWS : + return Jronn.class; + default : + throw new RuntimeException( + "Unknown web service implementation class for service: " + + this); + } + } + + public static Class> getRunnerByJobDirectory(File jobdir) { + Services service = getServiceByRunnerName(getRunnerNameByJobDirectory(jobdir)); + return service.getServiceImpl(); + } + + private static String getRunnerNameByJobDirectory(File jobdir) { + String name = jobdir.getName().split("#")[0]; + + if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { + assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; + name = name.substring(1); + } + return name; + } + + public static Services getServiceByJobDirectory(File jobdir) { + return getServiceByRunnerName(getRunnerNameByJobDirectory(jobdir)); + } + + private static Services getServiceByRunnerName(String name) { + for (Services service : Services.values()) { + String runnerName = service.getServiceImpl().getSimpleName() + .toLowerCase(); + name = name.trim().toLowerCase(); + if (name.startsWith(runnerName)) { + return service; + } + } + return null; + } + Service getService(URL url, String sqname) { QName qname = new QName(sqname, this.toString()); return Service.create(url, qname); -- 1.7.10.2