From bf5e83e2959a3b88963cb9a8da68fc50ce4329af Mon Sep 17 00:00:00 2001 From: Sasha Sherstnev Date: Thu, 5 Dec 2013 14:25:55 +0000 Subject: [PATCH] Add posibility to look at failed jobs --- .classpath | 2 +- .../cassandra/CassandraNativeConnector.java | 4 +- datadb/compbio/cassandra/CassandraReader.java | 58 +++++++----- datadb/compbio/cassandra/CassandraWriter.java | 93 +++++++++++--------- datadb/compbio/cassandra/JpredParserHTTP.java | 24 +++++ .../controllers/DailyStatisticsController.java | 13 ++- server/compbio/listeners/ServletJobsByDay.java | 25 +++--- server/compbio/statistic/CassandraRequester.java | 10 ++- webapp/view/reportJobStatistics.jsp | 24 +++-- webapp/view/reportJobStatisticsOneDay.jsp | 2 +- 10 files changed, 165 insertions(+), 90 deletions(-) diff --git a/.classpath b/.classpath index 5931fd0..2816de8 100644 --- a/.classpath +++ b/.classpath @@ -33,7 +33,7 @@ - + diff --git a/datadb/compbio/cassandra/CassandraNativeConnector.java b/datadb/compbio/cassandra/CassandraNativeConnector.java index 4b40518..0a5a795 100644 --- a/datadb/compbio/cassandra/CassandraNativeConnector.java +++ b/datadb/compbio/cassandra/CassandraNativeConnector.java @@ -88,6 +88,8 @@ public class CassandraNativeConnector { session.execute("CREATE INDEX IF NOT EXISTS ProteinSeq ON ProteinRow (protein);"); session.execute("CREATE INDEX IF NOT EXISTS ProteinIp ON ProteinLog (ip);"); + session.execute("CREATE INDEX IF NOT EXISTS ON ProteinLog (ExecutionStatus);"); + session.execute("CREATE INDEX IF NOT EXISTS ON FailLog (FinalStatus);"); // session.execute("CREATE INDEX IF NOT EXISTS JobDateStamp ON ProteinData (jobtime);"); } @@ -128,5 +130,5 @@ public class CassandraNativeConnector { System.out.println(" Metrics Options = " + mopt.toString()); System.out.println(" Query Options = " + qopt.toString()); } - + } diff --git a/datadb/compbio/cassandra/CassandraReader.java b/datadb/compbio/cassandra/CassandraReader.java index 69f7c08..00a9f27 100644 --- a/datadb/compbio/cassandra/CassandraReader.java +++ b/datadb/compbio/cassandra/CassandraReader.java @@ -11,6 +11,7 @@ import org.apache.log4j.Logger; import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.ResultSet; +import compbio.engine.JobStatus; public class CassandraReader { private Session session; @@ -18,7 +19,7 @@ public class CassandraReader { public CassandraReader() { Session inis = CassandraNativeConnector.getSession(); - setSession (inis); + setSession(inis); } public void setSession(Session s) { @@ -54,25 +55,44 @@ public class CassandraReader { * getting data from the db */ public DateBean ReadProteinData(long day, String date) { - final long startTime = System.currentTimeMillis(); String com = "SELECT JobID, Protein FROM ProteinData WHERE jobtime = " + day + ";"; System.out.println("Command: " + com); ResultSet results = session.execute(com); if (results.isExhausted()) return null; - final long queryTime = System.currentTimeMillis(); List rows = results.all(); - System.out.println("Query time is " + (queryTime - startTime) + " msec"); DateBean res = new DateBean(date); - int c = 0; for (Row r : rows) { res.setJobidAndSeq(r.getString("JobID"), r.getString("Protein")); - ++c; } - final long endTime = System.currentTimeMillis(); - System.out.println(c + " rows analysed, execution time is " + (endTime - startTime) + " msec"); return res; } + + /** + * getting data from the db + */ + public DateBean ReadFailedJobs(long day, String date, JobStatus status) { + // FailLog (jobtime, JobID, ExecTime, ip, FinalStatus) + String com = "SELECT JobID FROM FailLog WHERE jobtime = " + day + " and FinalStatus = '" + status.name() + "';"; + ResultSet results = session.execute(com); + if (results.isExhausted()) + return null; + List rows = results.all(); + DateBean res = new DateBean(date); + for (Row r : rows) { + String jobid = r.getString("JobID"); + String com1 = "SELECT Protein FROM ProteinLog WHERE JobID = '" + jobid + "';"; + System.out.println("Command: " + com1); + ResultSet results2 = session.execute(com1); + List jrows = results2.all(); + if (1 == jrows.size()) { + String protein = jrows.get(0).getString("Protein"); + res.setJobidAndSeq(jobid, protein); + } + } + return res; + } + /* * getting data from the db JobDateInfo */ @@ -81,11 +101,11 @@ public class CassandraReader { if (results.isExhausted()) return null; Row therow = results.one(); - Total res = new Total(therow.getLong("Total"), therow.getLong("TotalOK"), therow.getLong("TotalStopped"), + Total res = new Total(therow.getLong("Total"), therow.getLong("TotalOK"), therow.getLong("TotalStopped"), therow.getLong("TotalError"), therow.getLong("TotalTimeOut")); if (!results.isExhausted()) { - Date date = new Date (queryDate); - log.warn("CassandraReader.ReadDateTable: date row for " + date.toString () + " ("+ queryDate + ") duplicated "); + Date date = new Date(queryDate); + log.warn("CassandraReader.ReadDateTable: date row for " + date.toString() + " (" + queryDate + ") duplicated "); } return res; } @@ -106,12 +126,11 @@ public class CassandraReader { System.out.println("Query time is " + (queryTime - startTime) + " msec"); System.out.println(" rows analysed, " + rows.size()); List res = new ArrayList(); - ProteinBean structure = new ProteinBean(queryProtein, rows.get(0).getMap( - "Predictions", String.class, String.class)); + ProteinBean structure = new ProteinBean(queryProtein, rows.get(0).getMap("Predictions", String.class, String.class)); System.out.println("second size : " + rows.size()); int c = 0; for (Row r : rows) { - structure.setJobid(r.getString("JobID")); + structure.setJobid(r.getString("JobID")); ++c; } res.add(structure); @@ -120,7 +139,6 @@ public class CassandraReader { return res; } - /* * getting jobs by ip */ @@ -138,9 +156,9 @@ public class CassandraReader { System.out.println(" rows analysed, " + rows.size()); int c = 0; for (Row r : rows) { - if (r.getString("FinalStatus").equals("OK")) { + if (r.getString("FinalStatus").equals("OK")) { String date = r.getString("DataBegin"); - res.put(r.getString("JobID"), new String[] {date.substring(0, date.indexOf(":")), r.getString("Protein")}); + res.put(r.getString("JobID"), new String[] { date.substring(0, date.indexOf(":")), r.getString("Protein") }); ++c; } } @@ -168,7 +186,7 @@ public class CassandraReader { for (Row r : rows) { String prot = r.getString("Protein"); if (prot.matches("(.*)" + queryProtein + "(.*)")) { - ProteinBean structure = new ProteinBean(prot, r.getMap("Predictions", String.class, String.class)); + ProteinBean structure = new ProteinBean(prot, r.getMap("Predictions", String.class, String.class)); structure.setJobid(r.getString("JobID")); res.add(structure); ++c; @@ -255,8 +273,8 @@ public class CassandraReader { if (results1.isExhausted()) return null; Row row1 = results1.one(); - JobBean res = new JobBean(row.getString("Protein"), row.getString("JobID"), row.getString("DataBegin"), - row.getString("DataEnd"), row.getString("ip"), row1.getMap("Predictions", String.class, String.class)); + JobBean res = new JobBean(row.getString("Protein"), row.getString("JobID"), row.getString("DataBegin"), row.getString("DataEnd"), + row.getString("ip"), row1.getMap("Predictions", String.class, String.class)); System.out.println("Query time is " + (queryTime - startTime) + " msec"); final long endTime = System.currentTimeMillis(); System.out.println(" rows analysed, execution time is " + (endTime - startTime) + " msec"); diff --git a/datadb/compbio/cassandra/CassandraWriter.java b/datadb/compbio/cassandra/CassandraWriter.java index 640079f..15e30e8 100644 --- a/datadb/compbio/cassandra/CassandraWriter.java +++ b/datadb/compbio/cassandra/CassandraWriter.java @@ -1,8 +1,5 @@ package compbio.cassandra; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.List; import org.apache.log4j.Logger; @@ -52,24 +49,20 @@ public class CassandraWriter { */ public int FormQueryTables(JpredJob job) { if (JobisNotInsterted(job.getJobID())) { - int njobsOk = 0; - int njobsStop = 0; - int njobsError = 0; - int njobsTimeOut = 0; String id = job.getJobID(); - String ip = job.getIP(); String protein = job.getProtein(); String finalstatus = job.getFinalStatus(); String execstatus = job.getExecutionStatus(); - String com1 = "INSERT INTO ProteinLog " + "(JobID, IP, DataBegin, DataEnd, FinalStatus, ExecutionStatus, Protein)" - + " VALUES ('" + id + "','" + ip + "','" + job.getStartingTimeStr() + "','" + job.getEndTimeStr() + "','" + finalstatus - + "','" + execstatus + "','" + protein + "');"; + + String com1 = "INSERT INTO ProteinLog (JobID, IP, DataBegin, DataEnd, FinalStatus, ExecutionStatus, Protein) VALUES ('" + id + + "','" + job.getIP() + "','" + job.getStartingTimeStr() + "','" + job.getEndTimeStr() + "','" + finalstatus + "','" + + execstatus + "','" + protein + "');"; session.execute(com1); - if (execstatus.equals("OK")) { - String com2 = "INSERT INTO ProteinData " + "(jobtime, JobID, ExecTime, Protein)" + " VALUES (" + job.getStartingDate() + ",'" + id - + "'," + job.getExecutionTime() + ",'" + protein + "');"; + if (finalstatus.equals("OK")) { + String com2 = "INSERT INTO ProteinData " + "(jobtime, JobID, ExecTime, Protein)" + " VALUES (" + job.getStartingDate() + + ",'" + id + "'," + job.getExecutionTime() + ",'" + protein + "');"; session.execute(com2); - + String allpredictions = ""; List pr = job.getPredictions(); for (FastaSequence pred : pr) { @@ -82,54 +75,58 @@ public class CassandraWriter { final_prediction = allpredictions.substring(0, allpredictions.length() - 1); } - String check2 = "SELECT * FROM ProteinRow WHERE JobID = '" + job.getJobID() + "';"; - ResultSet results2 = session.execute(check2); + ResultSet results2 = session.execute("SELECT * FROM ProteinRow WHERE JobID = '" + job.getJobID() + "';"); if (results2.isExhausted()) { - String com3 = "INSERT INTO ProteinRow " + "(Protein, JobID, Predictions)" + " VALUES ('" + protein + "','" + id + "',{" + String com3 = "INSERT INTO ProteinRow (Protein, JobID, Predictions) VALUES ('" + protein + "','" + id + "',{" + final_prediction + "});"; session.execute(com3); } - njobsOk = 1; } else { - String com5 = "INSERT INTO FailLog " + "(jobtime, JobID, ExecTime, ip, FinalStatus)" + " VALUES (" - + job.getStartingDate() + ",'" + id - + "'," + job.getExecutionTime() + ",'" + ip + "', '" + finalstatus + "');"; + String com5 = "INSERT INTO FailLog (jobtime, JobID, ExecTime, ip, FinalStatus) VALUES (" + job.getStartingDate() + ",'" + + id + "'," + job.getExecutionTime() + ",'" + job.getIP() + "', '" + finalstatus + "');"; session.execute(com5); - if (finalstatus.equals("TIMEDOUT")) - njobsTimeOut = 1; - else if (finalstatus.equals("JPREDERROR")) - njobsError = 1; - else if (finalstatus.equals("STOPPED")) - njobsStop = 1; } - // update some internal query tables - String check3 = "SELECT * FROM MainParameters WHERE Name = 'EarliestJobDate';"; - ResultSet results3 = session.execute(check3); + + // update Main parameters if the job is the earliest job so far + ResultSet results3 = session.execute("SELECT * FROM MainParameters WHERE Name = 'EarliestJobDate';"); boolean updateparameter = true; if (!results3.isExhausted()) { - Row r = results3.one(); + Row r = results3.one(); if (job.getStartingDate() >= Long.parseLong(r.getString("Value"))) updateparameter = false; } if (updateparameter) { - String com = "INSERT INTO MainParameters " + "(Name, Value)" + " VALUES ('EarliestJobDate','" + job.getStartingDate() - + "');"; - session.execute(com); + session.execute("INSERT INTO MainParameters (Name, Value) VALUES ('EarliestJobDate','" + job.getStartingDate() + "');"); } - String check4 = "SELECT * FROM JobDateInfo WHERE jobday = " + job.getStartingDate() + ";"; - ResultSet results4 = session.execute(check4); - updateparameter = true; - int njobs = 1; + + // update internal job counts (used by the Daily Statistics + // requests) + //TODO I don't like the bit of code. There should not be so many counters... + int njobsTotal = 1; + int njobsOk = 0; + int njobsStop = 0; + int njobsError = 0; + int njobsTimeOut = 0; + if (finalstatus.equals("OK")) + njobsOk = 1; + else if (finalstatus.equals("TIMEDOUT")) + njobsTimeOut = 1; + else if (finalstatus.equals("JPREDERROR")) + njobsError = 1; + else if (finalstatus.equals("STOPPED")) + njobsStop = 1; + ResultSet results4 = session.execute("SELECT * FROM JobDateInfo WHERE jobday = " + job.getStartingDate() + ";"); if (!results4.isExhausted()) { Row r = results4.one(); - njobs += r.getLong("Total"); + njobsTotal += r.getLong("Total"); njobsOk += r.getLong("TotalOK"); njobsError += r.getLong("TotalError"); njobsStop += r.getLong("TotalStopped"); njobsTimeOut += r.getLong("TotalTimeOut"); } String com = "INSERT INTO JobDateInfo " + "(jobday, Total, TotalOK, TotalStopped, TotalError, TotalTimeOut)" + " VALUES (" - + job.getStartingDate() + "," + njobs + "," + njobsOk + "," + njobsStop + "," + njobsError + "," + njobsTimeOut + ");"; + + job.getStartingDate() + "," + njobsTotal + "," + njobsOk + "," + njobsStop + "," + njobsError + "," + njobsTimeOut + + ");"; session.execute(com); return 1; } @@ -144,9 +141,17 @@ public class CassandraWriter { if (JobisNotArchived(job.getJobID())) { String id = job.getJobID(); String log = job.getLog().replaceAll("'", ""); - String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime, FinalStatus, ExecutionStatus, LOG, ArchiveLink) VALUES ('" + id + "','" - + job.getProtein() + "','" + job.getIP() + "'," + job.getStartingTime() + "," + job.getExecutionTime() + ",'" + job.getFinalStatus() + "','" - + job.getExecutionStatus() + "','" + log + "','" + archivepath + "');"; + String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime, FinalStatus, ExecutionStatus, LOG, ArchiveLink) VALUES ('" + + id + + "','" + + job.getProtein() + + "','" + + job.getIP() + + "'," + + job.getStartingTime() + + "," + + job.getExecutionTime() + + ",'" + job.getFinalStatus() + "','" + job.getExecutionStatus() + "','" + log + "','" + archivepath + "');"; session.execute(com); List predictions = job.getPredictions(); for (FastaSequence p : predictions) { diff --git a/datadb/compbio/cassandra/JpredParserHTTP.java b/datadb/compbio/cassandra/JpredParserHTTP.java index ac9ed4f..453f01f 100644 --- a/datadb/compbio/cassandra/JpredParserHTTP.java +++ b/datadb/compbio/cassandra/JpredParserHTTP.java @@ -93,6 +93,17 @@ public class JpredParserHTTP implements JpredParser { } return protein; } + private String parseSeqFile(final InputStream stream, String jobid) throws FileNotFoundException { + final FastaReader fr = new FastaReader(stream); + String protein = ""; + final FastaSequence fs = fr.next(); + protein = fs.getSequence().replaceAll("\n", ""); + if (fr.hasNext()) { + // this is an aligment job... + return "alignment"; + } + return protein; + } private String parseLogFile(final InputStream stream) throws IOException { String out = ""; @@ -170,9 +181,22 @@ public class JpredParserHTTP implements JpredParser { } if (!running) { + // logging the job job.setAlignment(alignment); job.setPredictions(predictions); + if (job.getExecutionStatus().equals("FAIL")) { + URL sequrl = new URL(maindir + job.getJobID() + ".seq"); + HttpURLConnection httpConnection_sequrl = (HttpURLConnection) sequrl.openConnection(); + if (199 < httpConnection_sequrl.getResponseCode() && httpConnection_sequrl.getResponseCode() < 300) { + try { + job.setProtein(parseSeqFile(sequrl.openStream(), job.getJobID())); + } catch (IOException e) { + e.printStackTrace(); + } + } + } cw.FormQueryTables(job); + // archiving the job if (archiving) { ArchivedJob ajob = new ArchivedJob(job.getJobID()); diff --git a/server/compbio/controllers/DailyStatisticsController.java b/server/compbio/controllers/DailyStatisticsController.java index 0fdd4b8..142a298 100644 --- a/server/compbio/controllers/DailyStatisticsController.java +++ b/server/compbio/controllers/DailyStatisticsController.java @@ -4,7 +4,6 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import java.util.List; import java.util.Map; import org.springframework.stereotype.Controller; @@ -12,7 +11,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; -import compbio.cassandra.DataBase; +import compbio.engine.JobStatus; import compbio.cassandra.DateBean; import compbio.cassandra.TotalJobsStatisticBean; import compbio.statistic.CassandraRequester; @@ -61,7 +60,8 @@ public class DailyStatisticsController { } @RequestMapping(value = "/stat/oneday", method = RequestMethod.GET) - public String findJobsInOneDay(@RequestParam("date") String date, Map model) throws ParseException { + public String findJobsInOneDay(@RequestParam("date") String date, @RequestParam("status") String status, Map model) throws ParseException { + final long startTime = System.currentTimeMillis(); String realdate; @@ -78,13 +78,18 @@ public class DailyStatisticsController { realdate = date; } + if (null == JobStatus.getJobStatus(status)) { + return "support/Notimplemented"; + } + CassandraRequester cr = new CassandraRequester(); // IMPORTANT: input should be suppied in the format: DD/MM/YYYY - DateBean r = cr.readJobByDay(realdate); + DateBean r = cr.readJobByDay(realdate, JobStatus.getJobStatus(status)); model.put("results", r); if (r != null) model.put("njobs", r.getJobidAndSeq().size()); model.put("date", realdate); + model.put("status", status); final long endTime = System.currentTimeMillis(); model.put("timeExecution", (endTime - startTime)); return "/reportJobStatisticsOneDay"; diff --git a/server/compbio/listeners/ServletJobsByDay.java b/server/compbio/listeners/ServletJobsByDay.java index 6a861a9..3d2bf4b 100644 --- a/server/compbio/listeners/ServletJobsByDay.java +++ b/server/compbio/listeners/ServletJobsByDay.java @@ -17,19 +17,22 @@ public class ServletJobsByDay extends HttpServlet { private static final long serialVersionUID = 1L; /** - * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse + * response) + */ + /* + * protected void doGet(HttpServletRequest request, HttpServletResponse + * response) throws ServletException, IOException { String date = + * request.getParameter("date"); CassandraRequester cr = new + * CassandraRequester(); request.setAttribute("result", + * cr.readJobByDay(date)); request.setAttribute("date", date); + * RequestDispatcher rd = + * request.getRequestDispatcher("/ReportJobsByADay.jsp"); + * rd.forward(request, response); } */ - protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - String date = request.getParameter("date"); - CassandraRequester cr = new CassandraRequester(); - request.setAttribute("result", cr.readJobByDay(date)); - request.setAttribute("date", date); - RequestDispatcher rd = request.getRequestDispatcher("/ReportJobsByADay.jsp"); - rd.forward(request, response); - } - /** - * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse + * response) */ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { doGet(request, response); diff --git a/server/compbio/statistic/CassandraRequester.java b/server/compbio/statistic/CassandraRequester.java index 5ed9398..e95291a 100755 --- a/server/compbio/statistic/CassandraRequester.java +++ b/server/compbio/statistic/CassandraRequester.java @@ -5,7 +5,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -20,6 +19,7 @@ import compbio.cassandra.Total; import compbio.cassandra.TotalByCounterBean; import compbio.cassandra.TotalJobsStatisticBean; import compbio.cassandra.UserBean; +import compbio.engine.JobStatus; public class CassandraRequester { private CassandraReader db = new CassandraReader(); @@ -165,7 +165,7 @@ public class CassandraRequester { /* * query: jobs and sequence at date */ - public DateBean readJobByDay(String date) { + public DateBean readJobByDay(String date, JobStatus status) { if (!isThisDateValid(date, formatDDMMYY)) { System.out.println("CassandraRequester.readJobByDay: Wrong date format for " + date); return null; @@ -174,7 +174,11 @@ public class CassandraRequester { long day = DateParsing(date, formatDDMMYY); if (day < earlestDate || day > currentDate) return null; - return db.ReadProteinData(day, date); + + if (status == JobStatus.OK) { + return db.ReadProteinData(day, date); + } + return db.ReadFailedJobs(day, date, status); } /* diff --git a/webapp/view/reportJobStatistics.jsp b/webapp/view/reportJobStatistics.jsp index 509a539..5a833ec 100644 --- a/webapp/view/reportJobStatistics.jsp +++ b/webapp/view/reportJobStatistics.jsp @@ -70,10 +70,25 @@ ${res.key} ${value.total} - ${value.totalOK} - ${value.totalStopped} - ${value.totalError} - ${value.totalTimeOut} + ${value.totalOK} + + + 0 + ${value.totalStopped} + + + + + 0 + ${value.totalError} + + + + + 0 + ${value.totalTimeOut} + + @@ -81,7 +96,6 @@ - diff --git a/webapp/view/reportJobStatisticsOneDay.jsp b/webapp/view/reportJobStatisticsOneDay.jsp index 1194596..1176913 100644 --- a/webapp/view/reportJobStatisticsOneDay.jsp +++ b/webapp/view/reportJobStatisticsOneDay.jsp @@ -11,7 +11,7 @@
- +
  • -- 1.7.10.2