From: pvtroshin Date: Tue, 31 May 2011 12:27:37 +0000 (+0000) Subject: Stat collection and display done X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=1e8254e6a48302dd3e47e1f68cc51f6a6b9c33db;p=jabaws.git Stat collection and display done git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@4195 e3abac25-378b-4346-85de-24260fe3988d --- diff --git a/.classpath b/.classpath index 88db667..b83e40e 100644 --- a/.classpath +++ b/.classpath @@ -6,12 +6,12 @@ - + diff --git a/TODO.txt b/TODO.txt index a0a2706..86eca1b 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,11 @@ TODO: +Cluster stats: + -Remove hyperlinks from tasks which workdirs were removed + -graph generation + -user documentation + -use the same name for output for different executables + add to help text: To disable a web service remove it from WEB-INF/sun-jaxws.xml descriptor add to help: VirtualBox 4.0.4 works fine with JABAWS update links diff --git a/webservices/compbio/stat/collector/ExecutionStatCollector.java b/webservices/compbio/stat/collector/ExecutionStatCollector.java index bf3130a..31a5747 100644 --- a/webservices/compbio/stat/collector/ExecutionStatCollector.java +++ b/webservices/compbio/stat/collector/ExecutionStatCollector.java @@ -16,6 +16,7 @@ import java.util.Set; import org.apache.log4j.Logger; import compbio.engine.client.Executable; +import compbio.engine.client.PathValidator; import compbio.metadata.JobStatus; import compbio.util.FileUtil; import compbio.ws.client.Services; @@ -57,6 +58,7 @@ public class ExecutionStatCollector implements Runnable { static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); + final private File workDirectory; final private List stats; /** * Consider the job that has been working for longer than timeOutInHours @@ -73,22 +75,18 @@ public class ExecutionStatCollector implements Runnable { public ExecutionStatCollector(String workDirectory, int timeOutInHours) { log.info("Starting stat collector for directory: " + workDirectory); log.info("Maximum allowed runtime(h): " + timeOutInHours); - File[] files = FileUtil.getFiles(workDirectory, directories); + if (!PathValidator.isValidDirectory(workDirectory)) { + throw new IllegalArgumentException("workDirectory '" + + workDirectory + "' does not exist!"); + } + this.workDirectory = new File(workDirectory); stats = new ArrayList(); - assert timeOutInHours > 0; - this.timeOutInHours = timeOutInHours; - for (File file : files) { - JobDirectory jd = new JobDirectory(file); - JobStat jstat = jd.getJobStat(); - // Do not record stats on the job that has not completed yet - if (hasCompleted(jd)) { - stats.add(jstat); - } else { - log.debug("Skipping the job: " + jstat); - log.debug("As it has not completed yet"); - } - // System.out.println(jd.getJobStat().getJobReportTabulated()); + if (timeOutInHours <= 0) { + throw new IllegalArgumentException( + "Timeout value must be greater than 0! Given value: " + + timeOutInHours); } + this.timeOutInHours = timeOutInHours; } boolean hasCompleted(JobDirectory jd) { @@ -104,11 +102,11 @@ public class ExecutionStatCollector implements Runnable { return ((System.currentTimeMillis() - jd.jobdir.lastModified()) / (1000 * 60 * 60)) > timeOutInHours; } - public StatProcessor getStats() { + StatProcessor getStats() { return new StatProcessor(stats); } - public void writeStatToDB() throws SQLException { + void writeStatToDB() throws SQLException { Set rjobs = new HashSet(stats); StatDB statdb = new StatDB(); log.debug("Removing records that has already been recorded"); @@ -125,10 +123,7 @@ public class ExecutionStatCollector implements Runnable { */ /** - * - * @param args - * @throws IOException - * @throws SQLException + * Not in use */ public static void main(String[] args) throws IOException, SQLException { @@ -168,14 +163,14 @@ public class ExecutionStatCollector implements Runnable { File jobdir; Map files = new HashMap(); - public JobDirectory(File directory) { + JobDirectory(File directory) { this.jobdir = directory; for (File f : jobdir.listFiles()) { files.put(f.getName(), f); } } - public boolean hasStatus(JobStatus status) { + boolean hasStatus(JobStatus status) { return files.containsKey(status.toString()); } @@ -211,8 +206,9 @@ public class ExecutionStatCollector implements Runnable { clustjobId = FileUtil.readFileToString(jobid); } } catch (IOException ioe) { - ioe.printStackTrace(); - // TODO LOG + log.error( + "IO Exception while reading the content of JOBID file for job " + + jobid, ioe); } return clustjobId.trim(); } @@ -304,13 +300,30 @@ public class ExecutionStatCollector implements Runnable { return false; return true; } + } + private void collectStatistics() { + File[] files = workDirectory.listFiles(directories); + for (File file : files) { + JobDirectory jd = new JobDirectory(file); + JobStat jstat = jd.getJobStat(); + // Do not record stats on the job that has not completed yet + if (hasCompleted(jd)) { + stats.add(jstat); + } else { + log.debug("Skipping the job: " + jstat); + log.debug("As it has not completed yet"); + } + // System.out.println(jd.getJobStat().getJobReportTabulated()); + } } @Override public void run() { log.info("Started updating statistics at " + new Date()); + collectStatistics(); + StatProcessor local_stats = getStats(); log.info("Found " + local_stats.getJobNumber() + " jobs!"); try { @@ -321,5 +334,4 @@ public class ExecutionStatCollector implements Runnable { } log.info("Finished updating statistics at " + new Date()); } - } diff --git a/webservices/compbio/stat/collector/StatDB.java b/webservices/compbio/stat/collector/StatDB.java index 56ce9c1..c2dd29a 100644 --- a/webservices/compbio/stat/collector/StatDB.java +++ b/webservices/compbio/stat/collector/StatDB.java @@ -105,7 +105,7 @@ public class StatDB { + "isCollected SMALLINT NOT NULL, " + "isClusterJob SMALLINT NOT NULL)"; // We create a table... - System.out.println(create); + log.debug(create); s.execute(create); s.close(); conn.close(); @@ -153,6 +153,7 @@ public class StatDB { pstm.executeUpdate(); } conn.commit(); + conn.setAutoCommit(true); pstm.close(); } @@ -310,8 +311,7 @@ public class StatDB { conn.close(); } } catch (SQLException e) { - System.err.println("Database commit failed with " - + e.getLocalizedMessage()); + log.warn("Database commit failed with " + e.getLocalizedMessage()); } boolean gotSQLExc = false; try { @@ -322,9 +322,9 @@ public class StatDB { } } if (!gotSQLExc) { - System.err.println("Database did not shut down normally"); + log.warn("Database did not shut down normally"); } else { - System.out.println("Database shut down normally"); + log.info("Database shut down normally"); } } public static void main(String[] args) throws SQLException { diff --git a/webservices/compbio/stat/servlet/DisplayStat.java b/webservices/compbio/stat/servlet/DisplayStat.java index 79d984f..95c2d37 100644 --- a/webservices/compbio/stat/servlet/DisplayStat.java +++ b/webservices/compbio/stat/servlet/DisplayStat.java @@ -12,16 +12,20 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.log4j.Logger; + import compbio.stat.servlet.util.StatCollection; import compbio.stat.servlet.util.Totals; public class DisplayStat extends HttpServlet { + private final static Logger log = Logger.getLogger(DisplayStat.class); + @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { String datetime = req.getParameter("datetime"); - System.out.println("? " + datetime); + Date fromDate = new Date(Long.parseLong(datetime)); Calendar toCal = GregorianCalendar.getInstance(); toCal.setTime(fromDate); @@ -31,7 +35,7 @@ public class DisplayStat extends HttpServlet { StatCollection stats = StatCollection.newStatCollecton(fromDate, toCal.getTime()); - System.out.println("stats: " + stats); + log.trace("Stats: " + stats); req.setAttribute("stat", stats); req.setAttribute("statTotal", Totals.sumStats(stats.getAllStat())); req.setAttribute("statTotalCluster", @@ -41,13 +45,13 @@ public class DisplayStat extends HttpServlet { req.setAttribute("startDate", fromDate); req.setAttribute("stopDate", toCal.getTime()); - System.out.println(fromDate + " " + toCal.getTime()); + log.trace("from " + fromDate + " to " + toCal.getTime()); RequestDispatcher dispatcher = req .getRequestDispatcher("statpages/Statistics.jsp"); dispatcher.forward(req, resp); } catch (SQLException e) { - e.printStackTrace(); + log.error(e.getMessage(), e); throw new ServletException(e); } diff --git a/webservices/compbio/stat/servlet/Joblist.java b/webservices/compbio/stat/servlet/Joblist.java index 4ceb2b8..0ffd9ae 100644 --- a/webservices/compbio/stat/servlet/Joblist.java +++ b/webservices/compbio/stat/servlet/Joblist.java @@ -98,7 +98,7 @@ public class Joblist extends HttpServlet { .trim(); clusterTempDir = new File(clusterTempDir).getName(); String localTempDir = helper.getProperty("local.tmp.directory").trim(); - // TODO include the time slice + Timestamp startDate = new Timestamp(Long.parseLong(fromDate)); Timestamp stopDate = new Timestamp(Long.parseLong(toDate)); StatDB statdb = null; diff --git a/webservices/compbio/stat/servlet/StatisticCollector.java b/webservices/compbio/stat/servlet/StatisticCollector.java index 0489790..0fadfb5 100644 --- a/webservices/compbio/stat/servlet/StatisticCollector.java +++ b/webservices/compbio/stat/servlet/StatisticCollector.java @@ -56,10 +56,9 @@ public class StatisticCollector implements ServletContextListener { log.info("Initializing statistics collector"); executor = Executors.newScheduledThreadPool(2); - // FIXME Nullpointer if jobsout is not available? + if (collectClusterStats()) { - // TODO remove work out of the constructor Tomcat takes ages to - // start! + ExecutionStatCollector clusterCollector = new ExecutionStatCollector( clusterWorkDir, clusterMaxRuntime); clustercf = executor.scheduleAtFixedRate(clusterCollector, 60, @@ -83,7 +82,6 @@ public class StatisticCollector implements ServletContextListener { } else { log.info("Local statistics collector is disabled or not configured! "); } - } static String getClusterJobDir() {