From: pvtroshin Date: Thu, 10 Mar 2011 14:43:26 +0000 (+0000) Subject: statistics processor further work X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=76e6c5c7b3a95369c54728ad46d238552c833909;p=jabaws.git statistics processor further work git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@3818 e3abac25-378b-4346-85de-24260fe3988d --- diff --git a/ExecutionStatistic/log/log.ctrl b/ExecutionStatistic/log/log.ctrl index 8fcdcc8..65a47ac 100644 Binary files a/ExecutionStatistic/log/log.ctrl and b/ExecutionStatistic/log/log.ctrl differ diff --git a/ExecutionStatistic/log/log1.dat b/ExecutionStatistic/log/log1.dat index 596c3ec..fcb2616 100644 Binary files a/ExecutionStatistic/log/log1.dat and b/ExecutionStatistic/log/log1.dat differ diff --git a/ExecutionStatistic/log/logmirror.ctrl b/ExecutionStatistic/log/logmirror.ctrl index 8fcdcc8..65a47ac 100644 Binary files a/ExecutionStatistic/log/logmirror.ctrl and b/ExecutionStatistic/log/logmirror.ctrl differ diff --git a/ExecutionStatistic/seg0/c101.dat b/ExecutionStatistic/seg0/c101.dat index d8e6a8b..5c46cc6 100644 Binary files a/ExecutionStatistic/seg0/c101.dat and b/ExecutionStatistic/seg0/c101.dat differ diff --git a/ExecutionStatistic/seg0/c111.dat b/ExecutionStatistic/seg0/c111.dat index b046864..7af9ca7 100644 Binary files a/ExecutionStatistic/seg0/c111.dat and b/ExecutionStatistic/seg0/c111.dat differ diff --git a/ExecutionStatistic/seg0/c121.dat b/ExecutionStatistic/seg0/c121.dat index dbfded6..d93c0aa 100644 Binary files a/ExecutionStatistic/seg0/c121.dat and b/ExecutionStatistic/seg0/c121.dat differ diff --git a/ExecutionStatistic/seg0/c130.dat b/ExecutionStatistic/seg0/c130.dat index e06e9ec..7911c66 100644 Binary files a/ExecutionStatistic/seg0/c130.dat and b/ExecutionStatistic/seg0/c130.dat differ diff --git a/ExecutionStatistic/seg0/c141.dat b/ExecutionStatistic/seg0/c141.dat index 428b101..fb62699 100644 Binary files a/ExecutionStatistic/seg0/c141.dat and b/ExecutionStatistic/seg0/c141.dat differ diff --git a/ExecutionStatistic/seg0/c20.dat b/ExecutionStatistic/seg0/c20.dat index 57f18e8..554c52d 100644 Binary files a/ExecutionStatistic/seg0/c20.dat and b/ExecutionStatistic/seg0/c20.dat differ diff --git a/ExecutionStatistic/seg0/c31.dat b/ExecutionStatistic/seg0/c31.dat index 3e7eab8..7311a9e 100644 Binary files a/ExecutionStatistic/seg0/c31.dat and b/ExecutionStatistic/seg0/c31.dat differ diff --git a/ExecutionStatistic/seg0/c41.dat b/ExecutionStatistic/seg0/c41.dat index 78127c2..9bc7f11 100644 Binary files a/ExecutionStatistic/seg0/c41.dat and b/ExecutionStatistic/seg0/c41.dat differ diff --git a/ExecutionStatistic/seg0/c470.dat b/ExecutionStatistic/seg0/c470.dat new file mode 100644 index 0000000..d3f4f68 Binary files /dev/null and b/ExecutionStatistic/seg0/c470.dat differ diff --git a/ExecutionStatistic/seg0/c481.dat b/ExecutionStatistic/seg0/c481.dat new file mode 100644 index 0000000..805cd49 Binary files /dev/null and b/ExecutionStatistic/seg0/c481.dat differ diff --git a/ExecutionStatistic/seg0/c51.dat b/ExecutionStatistic/seg0/c51.dat index db67c98..e53479c 100644 Binary files a/ExecutionStatistic/seg0/c51.dat and b/ExecutionStatistic/seg0/c51.dat differ diff --git a/ExecutionStatistic/seg0/c60.dat b/ExecutionStatistic/seg0/c60.dat index 542e21c..2a44ef5 100644 Binary files a/ExecutionStatistic/seg0/c60.dat and b/ExecutionStatistic/seg0/c60.dat differ diff --git a/ExecutionStatistic/seg0/c71.dat b/ExecutionStatistic/seg0/c71.dat index 5e7d85c..30e364f 100644 Binary files a/ExecutionStatistic/seg0/c71.dat and b/ExecutionStatistic/seg0/c71.dat differ diff --git a/ExecutionStatistic/seg0/c81.dat b/ExecutionStatistic/seg0/c81.dat index ed09765..0e56d11 100644 Binary files a/ExecutionStatistic/seg0/c81.dat and b/ExecutionStatistic/seg0/c81.dat differ diff --git a/ExecutionStatistic/seg0/c90.dat b/ExecutionStatistic/seg0/c90.dat index 80251eb..a174a91 100644 Binary files a/ExecutionStatistic/seg0/c90.dat and b/ExecutionStatistic/seg0/c90.dat differ diff --git a/ExecutionStatistic/seg0/ca1.dat b/ExecutionStatistic/seg0/ca1.dat index 318f909..5bb75b8 100644 Binary files a/ExecutionStatistic/seg0/ca1.dat and b/ExecutionStatistic/seg0/ca1.dat differ diff --git a/ExecutionStatistic/seg0/cb1.dat b/ExecutionStatistic/seg0/cb1.dat index d1b3dbb..c988f78 100644 Binary files a/ExecutionStatistic/seg0/cb1.dat and b/ExecutionStatistic/seg0/cb1.dat differ diff --git a/ExecutionStatistic/seg0/cf0.dat b/ExecutionStatistic/seg0/cf0.dat index 1c00d1d..0adbb10 100644 Binary files a/ExecutionStatistic/seg0/cf0.dat and b/ExecutionStatistic/seg0/cf0.dat differ diff --git a/ExecutionStatistic/service.properties b/ExecutionStatistic/service.properties new file mode 100644 index 0000000..ecb3e8b --- /dev/null +++ b/ExecutionStatistic/service.properties @@ -0,0 +1,22 @@ +#D:\workspace\JABA2\ExecutionStatistic +# ******************************************************************** +# *** Please do NOT edit this file. *** +# *** CHANGING THE CONTENT OF THIS FILE MAY CAUSE DATA CORRUPTION. *** +# ******************************************************************** +#Wed Mar 09 18:04:00 GMT 2011 +SysschemasIndex2Identifier=225 +SyscolumnsIdentifier=144 +SysconglomeratesIndex1Identifier=49 +SysconglomeratesIdentifier=32 +SyscolumnsIndex2Identifier=177 +SysschemasIndex1Identifier=209 +SysconglomeratesIndex3Identifier=81 +SystablesIndex2Identifier=129 +SyscolumnsIndex1Identifier=161 +derby.serviceProtocol=org.apache.derby.database.Database +SysschemasIdentifier=192 +derby.storage.propertiesId=16 +SysconglomeratesIndex2Identifier=65 +derby.serviceLocale=en_GB +SystablesIdentifier=96 +SystablesIndex1Identifier=113 diff --git a/webservices/compbio/ws/execstat/ExecutionStatCollector.java b/webservices/compbio/ws/execstat/ExecutionStatCollector.java index 74218ce..945076e 100644 --- a/webservices/compbio/ws/execstat/ExecutionStatCollector.java +++ b/webservices/compbio/ws/execstat/ExecutionStatCollector.java @@ -4,9 +4,11 @@ import java.io.File; import java.io.FileFilter; import java.io.FileWriter; import java.io.IOException; +import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -85,8 +87,9 @@ public class ExecutionStatCollector { * * @param args * @throws IOException + * @throws SQLException */ - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, SQLException { // updateTime(new File( // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED")); @@ -94,7 +97,7 @@ public class ExecutionStatCollector { String workDir = PropertyHelperManager.getLocalPath() + getLocalJobDir().trim(); System.out.println(workDir); - File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout", + File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/local_jobsout", directories); List stats = new ArrayList(); for (File file : files) { @@ -108,6 +111,9 @@ public class ExecutionStatCollector { System.out.println("!!!!!!!!!!!!!!!!!!"); System.out.println(); System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat()); + + StatDB.insertData(new HashSet(sp + .getSingleWSStat(Services.TcoffeeWS).stats)); } static FileFilter directories = new FileFilter() { @@ -157,6 +163,20 @@ public class ExecutionStatCollector { return starttime; } + String getClusterJobID() { + String clustjobId = ""; + File jobid = files.get("JOBID"); + try { + if (jobid != null) { + clustjobId = FileUtil.readFileToString(jobid); + } + } catch (IOException ioe) { + ioe.printStackTrace(); + // TODO LOG + } + return clustjobId.trim(); + } + long getFinishedTime() { long ftime = UNDEFINED; File finished = files.get(JobStatus.FINISHED.toString()); @@ -219,9 +239,10 @@ public class ExecutionStatCollector { } StatProcessor.JobStat getJobStat() { - return new StatProcessor.JobStat(getService(), jobdir.getName(), - getStartTime(), getFinishedTime(), getInputSize(), - getResultSize(), isCollected(), isCancelled()); + return new StatProcessor.JobStat(getService(), getClusterJobID(), + jobdir.getName(), getStartTime(), getFinishedTime(), + getInputSize(), getResultSize(), isCollected(), + isCancelled()); } @Override diff --git a/webservices/compbio/ws/execstat/StatDB.java b/webservices/compbio/ws/execstat/StatDB.java new file mode 100644 index 0000000..2c66ea0 --- /dev/null +++ b/webservices/compbio/ws/execstat/StatDB.java @@ -0,0 +1,183 @@ +package compbio.ws.execstat; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Set; + +import compbio.ws.client.Services; +import compbio.ws.execstat.StatProcessor.JobStat; + +public class StatDB { + + /* the default framework is embedded */ + // private final String framework = "embedded"; + private static final String driver = "org.apache.derby.jdbc.EmbeddedDriver"; + private static final String protocol = "jdbc:derby:"; + private static final String statDBName = "ExecutionStatistic"; + + private static Connection getDBConnection() throws SQLException { + // TODO + System.setProperty("derby.system.home", "."); + Connection conn = DriverManager.getConnection(protocol + statDBName + + ";create=true"); + + // We want to control transactions manually. Autocommit is on by + // default in JDBC. + // conn.setAutoCommit(false); + return conn; + } + + // ServiceName,jobname,start,finish,inputSize,resultSize,isCancelled,isCollected + /** + * + * rs.getBoolean(i) will return true for any non-zero value and false for 0 + * on SMALLINT data column. + * + * @throws SQLException + */ + private static void createStatTable() throws SQLException { + Connection conn = getDBConnection(); + /* + * Creating a statement object that we can use for running various SQL + * statements commands against the database. + */ + Statement s = conn.createStatement(); + String create = "create table exec_stat(" + + "number INT GENERATED ALWAYS AS IDENTITY," + + "service_name VARCHAR(15) NOT NULL, " + + "cluster_job_id VARCHAR(30), " + + "job_id VARCHAR(35) NOT NULL PRIMARY KEY, " + + "start TIMESTAMP," + "finish TIMESTAMP," + + "inputsize BIGINT," + "resultsize BIGINT," + + "isCancelled SMALLINT NOT NULL," + + "isCollected SMALLINT NOT NULL, " + + "isClusterJob SMALLINT NOT NULL)"; + // We create a table... + System.out.println(create); + s.execute(create); + s.close(); + conn.close(); + } + + static void insertData(Set jobstatus) throws SQLException { + System.out.println("Inserting " + jobstatus.size()); + Connection conn = getDBConnection(); + conn.setAutoCommit(false); + String insert = "insert into exec_stat (service_name, cluster_job_id, job_id, start, finish, " + + "inputsize, resultsize, isCancelled, isCollected, isClusterJob) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + PreparedStatement pstm = conn.prepareStatement(insert); + for (JobStat js : jobstatus) { + pstm.setString(1, js.webService.toString()); + pstm.setString(2, js.clusterJobId); + pstm.setString(3, js.jobname); + pstm.setTimestamp(4, new Timestamp(js.start)); + pstm.setTimestamp(5, new Timestamp(js.finish)); + pstm.setLong(6, js.inputSize); + pstm.setLong(7, js.resultSize); + pstm.setBoolean(8, js.isCancelled); + pstm.setBoolean(9, js.isCollected); + pstm.setBoolean(10, js.isClusterJob()); + pstm.executeUpdate(); + } + conn.commit(); + pstm.close(); + conn.close(); + } + + static List readData(Timestamp from, Timestamp to, + Services wservice, Boolean clusterOnly) throws SQLException { + Connection conn = getDBConnection(); + String query = "select service_name, cluster_job_id, job_id, start, finish, inputsize, " + + "resultsize, isCancelled, isCollected from exec_stat where start BETWEEN ? and ? "; + + if (wservice != null) { + query += " and service_name=? "; + } + + if (clusterOnly != null) { + if (clusterOnly) { + query += " and isClusterJob!=0 "; + } else { + query += " and isClusterJob=0 "; + } + } + + PreparedStatement pstm = conn.prepareStatement(query); + pstm.setTimestamp(1, from); + pstm.setTimestamp(2, to); + if (wservice != null) { + pstm.setString(3, wservice.toString()); + } + pstm.execute(); + List stats = new ArrayList(); + ResultSet rs = pstm.getResultSet(); + while (rs.next()) { + stats.add(new JobStat(Services.getService(rs.getString(1)), rs + .getString(2), rs.getString(3), rs.getTimestamp(4) + .getTime(), rs.getTimestamp(5).getTime(), rs.getLong(6), rs + .getLong(7), rs.getBoolean(8), rs.getBoolean(9))); + } + rs.close(); + pstm.close(); + conn.close(); + return stats; + } + + static void removeRecordedJobs(Set fsJobs) throws SQLException { + Connection conn = getDBConnection(); + String query = "select job_id from exec_stat"; + + Statement st = conn.createStatement(); + ResultSet result = st.executeQuery(query); + + while (result.next()) { + String recordedJob = result.getString(1); + if (fsJobs.contains(recordedJob)) { + fsJobs.remove(recordedJob); + } + } + result.close(); + conn.close(); + } + + void shutdownDBServer() { + // ## DATABASE SHUTDOWN SECTION ## + /*** + * In embedded mode, an application should shut down Derby. Shutdown + * throws the XJ015 exception to confirm success. + ***/ + boolean gotSQLExc = false; + try { + DriverManager.getConnection("jdbc:derby:;shutdown=true"); + } catch (SQLException se) { + if (se.getSQLState().equals("XJ015")) { + gotSQLExc = true; + } + } + if (!gotSQLExc) { + System.out.println("Database did not shut down normally"); + } else { + System.out.println("Database shut down normally"); + } + } + public static void main(String[] args) throws SQLException { + // createStatTable(); + // insertData(null); + + Date from = new Date(); + from.setMonth(1); + System.out.println(new StatProcessor(readData( + new Timestamp(from.getTime()), + new Timestamp(new Date().getTime()), null, null)).reportStat()); + + } +} diff --git a/webservices/compbio/ws/execstat/StatProcessor.java b/webservices/compbio/ws/execstat/StatProcessor.java index 0697eb8..16b3234 100644 --- a/webservices/compbio/ws/execstat/StatProcessor.java +++ b/webservices/compbio/ws/execstat/StatProcessor.java @@ -6,6 +6,7 @@ import java.util.Comparator; import java.util.Date; import java.util.List; +import compbio.engine.client.ConfExecutable; import compbio.ws.client.Services; public class StatProcessor { @@ -16,6 +17,14 @@ public class StatProcessor { this.stats = stats; } + /* + * TODO List getNewStat() throws SQLException { Set jobids + * = new HashSet(); for(JobStat js: stats) { jobids.add(js.jobname); + * } StatDB.removeRecordedJobs(jobids); List newjobs = new + * HashSet(); for(String jobid: jobids) { if(newjobs.co) + * jobids.add(js.jobname); } } + */ + List getAbandonedJobs() { List abJobs = new ArrayList(); for (JobStat js : stats) { @@ -96,10 +105,18 @@ public class StatProcessor { report += "Cancelled Jobs: " + getCancelledJobs().size() + "\n"; report += "Total Runtime (s): " + getTotalRuntime() + "\n"; report += "Unsuccessful Jobs: " + getUnsuccessfulJobs().size() + "\n"; - report += "10 longest jobs: \n\n" + sortByRuntime().subList(0, 9) - + "\n"; - report += "10 biggest jobs: \n\n" + sortByResultSize().subList(0, 9) - + "\n"; + if (sortByRuntime().size() > 10) { + report += "10 longest jobs: \n\n" + sortByRuntime().subList(0, 9) + + "\n"; + } else { + report += "longest jobs: \n\n" + sortByRuntime() + "\n"; + } + if (sortByResultSize().size() > 10) + report += "10 biggest jobs: \n\n" + + sortByResultSize().subList(0, 9) + "\n"; + else { + report += "biggest jobs: \n\n" + sortByResultSize() + "\n"; + } return report; } @@ -127,6 +144,7 @@ public class StatProcessor { }; Services webService; + String clusterJobId; String jobname; long start; long finish; @@ -135,11 +153,12 @@ public class StatProcessor { boolean isCollected; boolean isCancelled; - JobStat(Services webService, String jobname, long start, long finish, - long inputSize, long resultSize, boolean isCollected, - boolean isCancelled) { + JobStat(Services webService, String clusterJobId, String jobname, + long start, long finish, long inputSize, long resultSize, + boolean isCollected, boolean isCancelled) { super(); this.webService = webService; + this.clusterJobId = clusterJobId; this.jobname = jobname; this.start = start; this.finish = finish; @@ -149,6 +168,10 @@ public class StatProcessor { this.isCancelled = isCancelled; } + public boolean isClusterJob() { + return jobname.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX); + } + @Override public int hashCode() { final int prime = 31; @@ -221,7 +244,9 @@ public class StatProcessor { } report += "Input size " + inputSize + "\n"; report += "Result size " + resultSize + "\n"; + report += "ClusterJobID " + clusterJobId + "\n"; report += "Collected? " + isCollected + "\n"; + report += "Cancelled? " + isCancelled + "\n"; return report; } @@ -251,7 +276,9 @@ public class StatProcessor { } report += inputSize + "\t"; report += resultSize + "\t"; + report += clusterJobId + "\t"; report += isCollected + "\t"; + report += isCancelled + "\t"; return report; } diff --git a/webservices/compbio/ws/execstat/StatWriter.java b/webservices/compbio/ws/execstat/StatWriter.java deleted file mode 100644 index d1b461e..0000000 --- a/webservices/compbio/ws/execstat/StatWriter.java +++ /dev/null @@ -1,100 +0,0 @@ -package compbio.ws.execstat; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; - -import compbio.ws.execstat.StatProcessor.JobStat; - -public class StatWriter { - - /* the default framework is embedded */ - // private final String framework = "embedded"; - private static final String driver = "org.apache.derby.jdbc.EmbeddedDriver"; - private static final String protocol = "jdbc:derby:"; - private static final String statDBName = "ExecutionStatistic"; - - static Connection getDBConnection() throws SQLException { - // TODO - System.setProperty("derby.system.home", "."); - - Connection conn = DriverManager.getConnection(protocol + statDBName - + ";create=true"); - - // We want to control transactions manually. Autocommit is on by - // default in JDBC. - conn.setAutoCommit(true); - return conn; - } - - // ServiceName,jobname,start,finish,inputSize,resultSize,isCancelled,isCollected - static void createStatTable() throws SQLException { - Connection conn = getDBConnection(); - /* - * Creating a statement object that we can use for running various SQL - * statements commands against the database. - */ - Statement s = conn.createStatement(); - String create = "create table exec_stat(" - + "number INT GENERATED ALWAYS AS IDENTITY," - + "service_name VARCHAR(15) NOT NULL, " - + "job_id VARCHAR(35) NOT NULL PRIMARY KEY, " - + "start TIMESTAMP," + "finish TIMESTAMP," - + "inputsize BIGINT," + "resultsize BIGINT," - + "isCancelled SMALLINT NOT NULL," - + "isCollected SMALLINT NOT NULL)"; - // We create a table... - System.out.println(create); - s.execute(create); - s.close(); - conn.close(); - } - - static void insertData(JobStat jobstatus) throws SQLException { - Connection conn = getDBConnection(); - String insert = "insert into exec_stat (service_name, job_id, start, finish, " - + "inputsize, resultsize, isCancelled, isCollected) " - + "VALUES (?, ?, ?, ?, ?, ?, ?, ? )"; - PreparedStatement pstm = conn.prepareStatement(insert); - - pstm.setString(1, "webservice"); - pstm.setString(2, "@Clustal#980273495452357"); - pstm.setTimestamp(3, new Timestamp(190385934834l)); - pstm.setTimestamp(4, new Timestamp(190332423423l)); - pstm.setLong(5, 1232); - pstm.setLong(6, 1432422); - pstm.setShort(7, (short) 1); - pstm.setShort(8, (short) 0); - pstm.executeUpdate(); - pstm.close(); - conn.close(); - } - - void shutdownDBServer() { - // ## DATABASE SHUTDOWN SECTION ## - /*** - * In embedded mode, an application should shut down Derby. Shutdown - * throws the XJ015 exception to confirm success. - ***/ - boolean gotSQLExc = false; - try { - DriverManager.getConnection("jdbc:derby:;shutdown=true"); - } catch (SQLException se) { - if (se.getSQLState().equals("XJ015")) { - gotSQLExc = true; - } - } - if (!gotSQLExc) { - System.out.println("Database did not shut down normally"); - } else { - System.out.println("Database shut down normally"); - } - } - public static void main(String[] args) throws SQLException { - // createStatTable(); - insertData(null); - } -}