--- /dev/null
+#D:\workspace\JABA2\ExecutionStatistic\r
+# ********************************************************************\r
+# *** Please do NOT edit this file. ***\r
+# *** CHANGING THE CONTENT OF THIS FILE MAY CAUSE DATA CORRUPTION. ***\r
+# ********************************************************************\r
+#Wed Mar 09 18:04:00 GMT 2011\r
+SysschemasIndex2Identifier=225\r
+SyscolumnsIdentifier=144\r
+SysconglomeratesIndex1Identifier=49\r
+SysconglomeratesIdentifier=32\r
+SyscolumnsIndex2Identifier=177\r
+SysschemasIndex1Identifier=209\r
+SysconglomeratesIndex3Identifier=81\r
+SystablesIndex2Identifier=129\r
+SyscolumnsIndex1Identifier=161\r
+derby.serviceProtocol=org.apache.derby.database.Database\r
+SysschemasIdentifier=192\r
+derby.storage.propertiesId=16\r
+SysconglomeratesIndex2Identifier=65\r
+derby.serviceLocale=en_GB\r
+SystablesIdentifier=96\r
+SystablesIndex1Identifier=113\r
import java.io.FileFilter;\r
import java.io.FileWriter;\r
import java.io.IOException;\r
+import java.sql.SQLException;\r
import java.text.SimpleDateFormat;\r
import java.util.ArrayList;\r
import java.util.HashMap;\r
+import java.util.HashSet;\r
import java.util.List;\r
import java.util.Map;\r
\r
* \r
* @param args\r
* @throws IOException\r
+ * @throws SQLException\r
*/\r
- public static void main(String[] args) throws IOException {\r
+ public static void main(String[] args) throws IOException, SQLException {\r
\r
// updateTime(new File(\r
// "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED"));\r
String workDir = PropertyHelperManager.getLocalPath()\r
+ getLocalJobDir().trim();\r
System.out.println(workDir);\r
- File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout",\r
+ File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/local_jobsout",\r
directories);\r
List<StatProcessor.JobStat> stats = new ArrayList<StatProcessor.JobStat>();\r
for (File file : files) {\r
System.out.println("!!!!!!!!!!!!!!!!!!");\r
System.out.println();\r
System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat());\r
+\r
+ StatDB.insertData(new HashSet<StatProcessor.JobStat>(sp\r
+ .getSingleWSStat(Services.TcoffeeWS).stats));\r
}\r
\r
static FileFilter directories = new FileFilter() {\r
return starttime;\r
}\r
\r
+ String getClusterJobID() {\r
+ String clustjobId = "";\r
+ File jobid = files.get("JOBID");\r
+ try {\r
+ if (jobid != null) {\r
+ clustjobId = FileUtil.readFileToString(jobid);\r
+ }\r
+ } catch (IOException ioe) {\r
+ ioe.printStackTrace();\r
+ // TODO LOG\r
+ }\r
+ return clustjobId.trim();\r
+ }\r
+\r
long getFinishedTime() {\r
long ftime = UNDEFINED;\r
File finished = files.get(JobStatus.FINISHED.toString());\r
}\r
\r
StatProcessor.JobStat getJobStat() {\r
- return new StatProcessor.JobStat(getService(), jobdir.getName(),\r
- getStartTime(), getFinishedTime(), getInputSize(),\r
- getResultSize(), isCollected(), isCancelled());\r
+ return new StatProcessor.JobStat(getService(), getClusterJobID(),\r
+ jobdir.getName(), getStartTime(), getFinishedTime(),\r
+ getInputSize(), getResultSize(), isCollected(),\r
+ isCancelled());\r
}\r
\r
@Override\r
--- /dev/null
+package compbio.ws.execstat;\r
+\r
+import java.sql.Connection;\r
+import java.sql.DriverManager;\r
+import java.sql.PreparedStatement;\r
+import java.sql.ResultSet;\r
+import java.sql.SQLException;\r
+import java.sql.Statement;\r
+import java.sql.Timestamp;\r
+import java.util.ArrayList;\r
+import java.util.Date;\r
+import java.util.List;\r
+import java.util.Set;\r
+\r
+import compbio.ws.client.Services;\r
+import compbio.ws.execstat.StatProcessor.JobStat;\r
+\r
+public class StatDB {\r
+\r
+ /* the default framework is embedded */\r
+ // private final String framework = "embedded";\r
+ private static final String driver = "org.apache.derby.jdbc.EmbeddedDriver";\r
+ private static final String protocol = "jdbc:derby:";\r
+ private static final String statDBName = "ExecutionStatistic";\r
+\r
+ private static Connection getDBConnection() throws SQLException {\r
+ // TODO\r
+ System.setProperty("derby.system.home", ".");\r
+ Connection conn = DriverManager.getConnection(protocol + statDBName\r
+ + ";create=true");\r
+\r
+ // We want to control transactions manually. Autocommit is on by\r
+ // default in JDBC.\r
+ // conn.setAutoCommit(false);\r
+ return conn;\r
+ }\r
+\r
+ // ServiceName,jobname,start,finish,inputSize,resultSize,isCancelled,isCollected\r
+ /**\r
+ * \r
+ * rs.getBoolean(i) will return true for any non-zero value and false for 0\r
+ * on SMALLINT data column.\r
+ * \r
+ * @throws SQLException\r
+ */\r
+ private static void createStatTable() throws SQLException {\r
+ Connection conn = getDBConnection();\r
+ /*\r
+ * Creating a statement object that we can use for running various SQL\r
+ * statements commands against the database.\r
+ */\r
+ Statement s = conn.createStatement();\r
+ String create = "create table exec_stat("\r
+ + "number INT GENERATED ALWAYS AS IDENTITY,"\r
+ + "service_name VARCHAR(15) NOT NULL, "\r
+ + "cluster_job_id VARCHAR(30), "\r
+ + "job_id VARCHAR(35) NOT NULL PRIMARY KEY, "\r
+ + "start TIMESTAMP," + "finish TIMESTAMP,"\r
+ + "inputsize BIGINT," + "resultsize BIGINT,"\r
+ + "isCancelled SMALLINT NOT NULL,"\r
+ + "isCollected SMALLINT NOT NULL, "\r
+ + "isClusterJob SMALLINT NOT NULL)";\r
+ // We create a table...\r
+ System.out.println(create);\r
+ s.execute(create);\r
+ s.close();\r
+ conn.close();\r
+ }\r
+\r
+ static void insertData(Set<JobStat> jobstatus) throws SQLException {\r
+ System.out.println("Inserting " + jobstatus.size());\r
+ Connection conn = getDBConnection();\r
+ conn.setAutoCommit(false);\r
+ String insert = "insert into exec_stat (service_name, cluster_job_id, job_id, start, finish, "\r
+ + "inputsize, resultsize, isCancelled, isCollected, isClusterJob) "\r
+ + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";\r
+ PreparedStatement pstm = conn.prepareStatement(insert);\r
+ for (JobStat js : jobstatus) {\r
+ pstm.setString(1, js.webService.toString());\r
+ pstm.setString(2, js.clusterJobId);\r
+ pstm.setString(3, js.jobname);\r
+ pstm.setTimestamp(4, new Timestamp(js.start));\r
+ pstm.setTimestamp(5, new Timestamp(js.finish));\r
+ pstm.setLong(6, js.inputSize);\r
+ pstm.setLong(7, js.resultSize);\r
+ pstm.setBoolean(8, js.isCancelled);\r
+ pstm.setBoolean(9, js.isCollected);\r
+ pstm.setBoolean(10, js.isClusterJob());\r
+ pstm.executeUpdate();\r
+ }\r
+ conn.commit();\r
+ pstm.close();\r
+ conn.close();\r
+ }\r
+\r
+ static List<JobStat> readData(Timestamp from, Timestamp to,\r
+ Services wservice, Boolean clusterOnly) throws SQLException {\r
+ Connection conn = getDBConnection();\r
+ String query = "select service_name, cluster_job_id, job_id, start, finish, inputsize, "\r
+ + "resultsize, isCancelled, isCollected from exec_stat where start BETWEEN ? and ? ";\r
+\r
+ if (wservice != null) {\r
+ query += " and service_name=? ";\r
+ }\r
+\r
+ if (clusterOnly != null) {\r
+ if (clusterOnly) {\r
+ query += " and isClusterJob!=0 ";\r
+ } else {\r
+ query += " and isClusterJob=0 ";\r
+ }\r
+ }\r
+\r
+ PreparedStatement pstm = conn.prepareStatement(query);\r
+ pstm.setTimestamp(1, from);\r
+ pstm.setTimestamp(2, to);\r
+ if (wservice != null) {\r
+ pstm.setString(3, wservice.toString());\r
+ }\r
+ pstm.execute();\r
+ List<JobStat> stats = new ArrayList<StatProcessor.JobStat>();\r
+ ResultSet rs = pstm.getResultSet();\r
+ while (rs.next()) {\r
+ stats.add(new JobStat(Services.getService(rs.getString(1)), rs\r
+ .getString(2), rs.getString(3), rs.getTimestamp(4)\r
+ .getTime(), rs.getTimestamp(5).getTime(), rs.getLong(6), rs\r
+ .getLong(7), rs.getBoolean(8), rs.getBoolean(9)));\r
+ }\r
+ rs.close();\r
+ pstm.close();\r
+ conn.close();\r
+ return stats;\r
+ }\r
+\r
+ static void removeRecordedJobs(Set<String> fsJobs) throws SQLException {\r
+ Connection conn = getDBConnection();\r
+ String query = "select job_id from exec_stat";\r
+\r
+ Statement st = conn.createStatement();\r
+ ResultSet result = st.executeQuery(query);\r
+\r
+ while (result.next()) {\r
+ String recordedJob = result.getString(1);\r
+ if (fsJobs.contains(recordedJob)) {\r
+ fsJobs.remove(recordedJob);\r
+ }\r
+ }\r
+ result.close();\r
+ conn.close();\r
+ }\r
+\r
+ void shutdownDBServer() {\r
+ // ## DATABASE SHUTDOWN SECTION ##\r
+ /***\r
+ * In embedded mode, an application should shut down Derby. Shutdown\r
+ * throws the XJ015 exception to confirm success.\r
+ ***/\r
+ boolean gotSQLExc = false;\r
+ try {\r
+ DriverManager.getConnection("jdbc:derby:;shutdown=true");\r
+ } catch (SQLException se) {\r
+ if (se.getSQLState().equals("XJ015")) {\r
+ gotSQLExc = true;\r
+ }\r
+ }\r
+ if (!gotSQLExc) {\r
+ System.out.println("Database did not shut down normally");\r
+ } else {\r
+ System.out.println("Database shut down normally");\r
+ }\r
+ }\r
+ public static void main(String[] args) throws SQLException {\r
+ // createStatTable();\r
+ // insertData(null);\r
+\r
+ Date from = new Date();\r
+ from.setMonth(1);\r
+ System.out.println(new StatProcessor(readData(\r
+ new Timestamp(from.getTime()),\r
+ new Timestamp(new Date().getTime()), null, null)).reportStat());\r
+\r
+ }\r
+}\r
import java.util.Date;\r
import java.util.List;\r
\r
+import compbio.engine.client.ConfExecutable;\r
import compbio.ws.client.Services;\r
\r
public class StatProcessor {\r
this.stats = stats;\r
}\r
\r
+ /*\r
+ * TODO List<JobStat> getNewStat() throws SQLException { Set<String> jobids\r
+ * = new HashSet<String>(); for(JobStat js: stats) { jobids.add(js.jobname);\r
+ * } StatDB.removeRecordedJobs(jobids); List<String> newjobs = new\r
+ * HashSet<String>(); for(String jobid: jobids) { if(newjobs.co)\r
+ * jobids.add(js.jobname); } }\r
+ */\r
+\r
List<JobStat> getAbandonedJobs() {\r
List<JobStat> abJobs = new ArrayList<StatProcessor.JobStat>();\r
for (JobStat js : stats) {\r
report += "Cancelled Jobs: " + getCancelledJobs().size() + "\n";\r
report += "Total Runtime (s): " + getTotalRuntime() + "\n";\r
report += "Unsuccessful Jobs: " + getUnsuccessfulJobs().size() + "\n";\r
- report += "10 longest jobs: \n\n" + sortByRuntime().subList(0, 9)\r
- + "\n";\r
- report += "10 biggest jobs: \n\n" + sortByResultSize().subList(0, 9)\r
- + "\n";\r
+ if (sortByRuntime().size() > 10) {\r
+ report += "10 longest jobs: \n\n" + sortByRuntime().subList(0, 9)\r
+ + "\n";\r
+ } else {\r
+ report += "longest jobs: \n\n" + sortByRuntime() + "\n";\r
+ }\r
+ if (sortByResultSize().size() > 10)\r
+ report += "10 biggest jobs: \n\n"\r
+ + sortByResultSize().subList(0, 9) + "\n";\r
+ else {\r
+ report += "biggest jobs: \n\n" + sortByResultSize() + "\n";\r
+ }\r
return report;\r
}\r
\r
};\r
\r
Services webService;\r
+ String clusterJobId;\r
String jobname;\r
long start;\r
long finish;\r
boolean isCollected;\r
boolean isCancelled;\r
\r
- JobStat(Services webService, String jobname, long start, long finish,\r
- long inputSize, long resultSize, boolean isCollected,\r
- boolean isCancelled) {\r
+ JobStat(Services webService, String clusterJobId, String jobname,\r
+ long start, long finish, long inputSize, long resultSize,\r
+ boolean isCollected, boolean isCancelled) {\r
super();\r
this.webService = webService;\r
+ this.clusterJobId = clusterJobId;\r
this.jobname = jobname;\r
this.start = start;\r
this.finish = finish;\r
this.isCancelled = isCancelled;\r
}\r
\r
+ public boolean isClusterJob() {\r
+ return jobname.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX);\r
+ }\r
+\r
@Override\r
public int hashCode() {\r
final int prime = 31;\r
}\r
report += "Input size " + inputSize + "\n";\r
report += "Result size " + resultSize + "\n";\r
+ report += "ClusterJobID " + clusterJobId + "\n";\r
report += "Collected? " + isCollected + "\n";\r
+ report += "Cancelled? " + isCancelled + "\n";\r
return report;\r
}\r
\r
}\r
report += inputSize + "\t";\r
report += resultSize + "\t";\r
+ report += clusterJobId + "\t";\r
report += isCollected + "\t";\r
+ report += isCancelled + "\t";\r
return report;\r
}\r
\r
+++ /dev/null
-package compbio.ws.execstat;\r
-\r
-import java.sql.Connection;\r
-import java.sql.DriverManager;\r
-import java.sql.PreparedStatement;\r
-import java.sql.SQLException;\r
-import java.sql.Statement;\r
-import java.sql.Timestamp;\r
-\r
-import compbio.ws.execstat.StatProcessor.JobStat;\r
-\r
-public class StatWriter {\r
-\r
- /* the default framework is embedded */\r
- // private final String framework = "embedded";\r
- private static final String driver = "org.apache.derby.jdbc.EmbeddedDriver";\r
- private static final String protocol = "jdbc:derby:";\r
- private static final String statDBName = "ExecutionStatistic";\r
-\r
- static Connection getDBConnection() throws SQLException {\r
- // TODO\r
- System.setProperty("derby.system.home", ".");\r
-\r
- Connection conn = DriverManager.getConnection(protocol + statDBName\r
- + ";create=true");\r
-\r
- // We want to control transactions manually. Autocommit is on by\r
- // default in JDBC.\r
- conn.setAutoCommit(true);\r
- return conn;\r
- }\r
-\r
- // ServiceName,jobname,start,finish,inputSize,resultSize,isCancelled,isCollected\r
- static void createStatTable() throws SQLException {\r
- Connection conn = getDBConnection();\r
- /*\r
- * Creating a statement object that we can use for running various SQL\r
- * statements commands against the database.\r
- */\r
- Statement s = conn.createStatement();\r
- String create = "create table exec_stat("\r
- + "number INT GENERATED ALWAYS AS IDENTITY,"\r
- + "service_name VARCHAR(15) NOT NULL, "\r
- + "job_id VARCHAR(35) NOT NULL PRIMARY KEY, "\r
- + "start TIMESTAMP," + "finish TIMESTAMP,"\r
- + "inputsize BIGINT," + "resultsize BIGINT,"\r
- + "isCancelled SMALLINT NOT NULL,"\r
- + "isCollected SMALLINT NOT NULL)";\r
- // We create a table...\r
- System.out.println(create);\r
- s.execute(create);\r
- s.close();\r
- conn.close();\r
- }\r
-\r
- static void insertData(JobStat jobstatus) throws SQLException {\r
- Connection conn = getDBConnection();\r
- String insert = "insert into exec_stat (service_name, job_id, start, finish, "\r
- + "inputsize, resultsize, isCancelled, isCollected) "\r
- + "VALUES (?, ?, ?, ?, ?, ?, ?, ? )";\r
- PreparedStatement pstm = conn.prepareStatement(insert);\r
-\r
- pstm.setString(1, "webservice");\r
- pstm.setString(2, "@Clustal#980273495452357");\r
- pstm.setTimestamp(3, new Timestamp(190385934834l));\r
- pstm.setTimestamp(4, new Timestamp(190332423423l));\r
- pstm.setLong(5, 1232);\r
- pstm.setLong(6, 1432422);\r
- pstm.setShort(7, (short) 1);\r
- pstm.setShort(8, (short) 0);\r
- pstm.executeUpdate();\r
- pstm.close();\r
- conn.close();\r
- }\r
-\r
- void shutdownDBServer() {\r
- // ## DATABASE SHUTDOWN SECTION ##\r
- /***\r
- * In embedded mode, an application should shut down Derby. Shutdown\r
- * throws the XJ015 exception to confirm success.\r
- ***/\r
- boolean gotSQLExc = false;\r
- try {\r
- DriverManager.getConnection("jdbc:derby:;shutdown=true");\r
- } catch (SQLException se) {\r
- if (se.getSQLState().equals("XJ015")) {\r
- gotSQLExc = true;\r
- }\r
- }\r
- if (!gotSQLExc) {\r
- System.out.println("Database did not shut down normally");\r
- } else {\r
- System.out.println("Database shut down normally");\r
- }\r
- }\r
- public static void main(String[] args) throws SQLException {\r
- // createStatTable();\r
- insertData(null);\r
- }\r
-}\r
git://source.jalview.org / jabaws.git / commitdiff