+ "ExecutionStatus ascii, Protein ascii, PRIMARY KEY(JobID));");
session.execute("CREATE TABLE IF NOT EXISTS ProteinData "
- + "(jobtime bigint, JobID ascii, Protein ascii, PRIMARY KEY(JobID));");
+ + "(jobtime bigint, JobID ascii, ExecTime int, Protein ascii, PRIMARY KEY(jobtime, JobID));");
+ session.execute("CREATE TABLE IF NOT EXISTS FailLog "
+ + "(jobtime bigint, JobID ascii, ExecTime int, ip ascii, FinalStatus ascii, PRIMARY KEY(jobtime, JobID));");
+
session.execute("CREATE TABLE IF NOT EXISTS JpredArchive "
- + "(JobID ascii, Protein varchar, IP ascii, StartTime bigint, ExecTime int, alignment map<ascii,ascii>, "
+ + "(JobID ascii, Protein varchar, IP ascii, StartTime bigint, ExecTime int, FinalStatus ascii, ExecutionStatus ascii, alignment map<ascii,ascii>, "
+ "predictions map<ascii,ascii>, ArchiveLink varchar, LOG varchar, PRIMARY KEY(JobID));");
session.execute("CREATE TABLE IF NOT EXISTS JobDateInfo "
- + "(jobday bigint, Total bigint, Program varchar, Version varchar, PRIMARY KEY(jobday));");
+ + "(jobday bigint, Total bigint, TotalOK bigint, TotalStopped bigint, TotalError bigint, TotalTimeOut bigint, Program varchar, Version varchar, PRIMARY KEY(jobday));");
session.execute("CREATE INDEX IF NOT EXISTS ProteinSeq ON ProteinRow (protein);");
- session.execute("CREATE INDEX IF NOT EXISTS JobDateStamp ON ProteinData (jobtime);");
- }
+ session.execute("CREATE INDEX IF NOT EXISTS ProteinIp ON ProteinLog (ip);");
+ // session.execute("CREATE INDEX IF NOT EXISTS JobDateStamp ON ProteinData (jobtime);");
+ }
public void Closing() {
session.shutdown();
/*
* getting data from the db JobDateInfo
*/
- public long ReadDateTable(long queryDate) {
- ResultSet results = session.execute("SELECT Total FROM JobDateInfo WHERE jobday = " + queryDate + ";");
+ public List<Long> ReadDateTable(long queryDate) {
+ ResultSet results = session.execute("SELECT * FROM JobDateInfo WHERE jobday = " + queryDate + ";");
if (results.isExhausted())
- return 0;
+ return null;
Row therow = results.one();
- long res = therow.getLong("Total");
+ List<Long> res = new ArrayList<Long>();
+ res.add(therow.getLong("Total"));
+ res.add(therow.getLong("TotalOK"));
+ res.add(therow.getLong("TotalStopped"));
+ res.add(therow.getLong("TotalError"));
+ res.add(therow.getLong("TotalTimeOut"));
if (!results.isExhausted()) {
Date date = new Date (queryDate);
log.warn("CassandraReader.ReadDateTable: date row for " + date.toString () + " ("+ queryDate + ") duplicated ");
*/
public Map<String, Integer> ReadProteinSequenceByCounter() {
final long startTime = System.currentTimeMillis();
- String com = "SELECT Protein FROM ProteinRow;";
+ String com = "SELECT Protein, JobID FROM ProteinRow;";
System.out.println("Command: " + com);
ResultSet results = session.execute(com);
if (results.isExhausted())
int c = 0;
for (Row r : rows) {
String protein = r.getString("Protein");
+ String id = r.getString("JobID");
+ System.out.println(id + ", " + protein);
if (res.containsKey(protein))
res.put(protein, res.get(protein) + 1);
else
package compbio.cassandra;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+
import org.apache.log4j.Logger;
import com.datastax.driver.core.ResultSet;
public class CassandraRemover {
private Session session;
+ static SimpleDateFormat dateformatter = new SimpleDateFormat("yyyy/MM/dd");
private static Logger log = Logger.getLogger(CassandraNativeConnector.class);
public CassandraRemover() {
}
/*
- * getting a record from CF for current jobId
+ * delete a record from CF for current jobId
*/
- public StructureJobLog ReadJobLog(String jobid) {
- final long startTime = System.currentTimeMillis();
- String com = "SELECT Protein, StartTime FROM ProteinLog WHERE JobID = '" + jobid + "';";
+ private void RemoveJob(String jobid, long date) {
+ String com0 = "DELETE FROM ProteinLog WHERE JobID = '" + jobid + "';";
+ System.out.println("Command: " + com0);
+ session.execute(com0);
+ String com1 = "DELETE FROM ProteinRow WHERE JobID = '" + jobid + "';";
+ System.out.println("Command: " + com1);
+ session.execute(com1);
+ String com2 = "DELETE FROM ProteinData WHERE JobID = '" + jobid + "' AND jobtime = " + date + ";";
+ System.out.println("Command: " + com2);
+ session.execute(com2);
+ String com3 = "UPDATE jpredarchive SET finalstatus = 'DELETED' WHERE JobID = '" + jobid + "' ;";
+ System.out.println("Command: " + com3);
+ session.execute(com3);
+ String com = "SELECT total FROM JobDateInfo WHERE jobday = " + date + ";";
+ System.out.println("Command: " + com);
+ ResultSet results = session.execute(com);
+ long njobs = results.one().getLong("total");
+ System.out.println("njobs: " + njobs);
+ String com4 = "INSERT INTO JobDateInfo " + "(jobday, Total)" + " VALUES (" + date + "," + (njobs -1) + ");";
+ System.out.println("Command: " + com4);
+ session.execute(com4);
+ System.out.println("Remove jobs: " + jobid);
+ }
+
+ public void RemoveJobById (String jobid) {
+ if (jobid == null)
+ return;
+ Long date = FindDate(jobid);
+ if (date == null)
+ return;
+ RemoveJob(jobid, date);
+ }
+
+ public void RemoveJobByDate (String date1, String date2) {
+ System.out.println("Start " + date1 + ", " + date2);
+ if (date1 == null || date2 == null)
+ return;
+ Long dateBegin = convertDate(date1);
+ Long dateEnd = convertDate(date2);
+ System.out.println("Date to long done!: ");
+ if (dateBegin == null || dateEnd == null)
+ return;
+ Calendar start = Calendar.getInstance();
+ start.setTime(new Date(dateBegin));
+ Calendar end = Calendar.getInstance();
+ end.setTime(new Date(dateEnd));
+ System.out.println("Date to cal done!: ");
+ for (Date date = start.getTime(); !start.after(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
+ String com = "SELECT JobID FROM ProteinData WHERE jobtime = " + date.getTime() + ";";
+ System.out.println("Command: " + com);
+ ResultSet results = session.execute(com);
+ if (!results.isExhausted()) {
+ List<Row> rows = results.all();
+ for (Row r : rows) {
+ String jobid = r.getString("JobID");
+ if (jobid != null)
+ RemoveJob(jobid, date.getTime());
+ }
+ }
+ }
+ }
+
+
+ public void RemoveJobByIp (String ip) {
+ if (ip == null)
+ return;
+ String com = "SELECT databegin, JobID FROM ProteinLog WHERE ip = '" + ip + "';";
+ System.out.println("Command: " + com);
+ ResultSet results = session.execute(com);
+ if (!results.isExhausted()) {
+ List<Row> rows = results.all();
+ for (Row r : rows) {
+ Long date = convertDate(r.getString("databegin"));
+ String jobid = r.getString("JobID");
+ if (date == null || jobid == null)
+ continue;
+ RemoveJob(jobid, date);
+ }
+ }
+ }
+
+ public void RemoveJobBySequence (String seq) {
+ if (seq == null)
+ return;
+ String com = "SELECT JobID FROM ProteinRow WHERE Protein = '" + seq + "';";
+ System.out.println("Command: " + com);
+ ResultSet results = session.execute(com);
+ if (!results.isExhausted()) {
+ List<Row> rows = results.all();
+ for (Row r : rows) {
+ String jobid = r.getString("JobID");
+ if (jobid == null)
+ continue;
+ Long date = FindDate(jobid);
+ if (date == null)
+ continue;
+ RemoveJob(jobid, date);
+ }
+ }
+ }
+
+ private Long FindDate(String jobid) {
+ String com = "SELECT databegin FROM ProteinLog WHERE JobID = '" + jobid + "';";
System.out.println("Command: " + com);
ResultSet results = session.execute(com);
if (results.isExhausted())
return null;
- final long queryTime = System.currentTimeMillis();
- Row row = results.one();
- String com1 = "SELECT * FROM ProteinRow WHERE JobID = '" + jobid + "' ALLOW FILTERING;";
- System.out.println("Command: " + com1);
- ResultSet results1 = session.execute(com1);
- if (results1.isExhausted())
- return null;
- Row row1 = results1.one();
- StructureJobLog res = new StructureJobLog(row.getString("Protein"), row.getString("JobID"), row.getString("DataBegin"),
- row.getString("DataEnd"), row.getString("ip"), row1.getMap("Predictions", String.class, String.class));
- System.out.println("Query time is " + (queryTime - startTime) + " msec");
- final long endTime = System.currentTimeMillis();
- System.out.println(" rows analysed, execution time is " + (endTime - startTime) + " msec");
- return res;
+ Long date = convertDate(results.one().getString("databegin"));
+ return date;
}
+ protected long convertDate (String d) {
+ try {
+ if (null != d) {
+ Date startdate = dateformatter.parse(d);
+ return startdate.getTime();
+ }
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ return 0L;
+ }
}
package compbio.cassandra;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import java.util.List;
import org.apache.log4j.Logger;
*/
public int FormQueryTables(JpredJob job) {
if (JobisNotInsterted(job.getJobID())) {
+ int njobsOk = 0;
+ int njobsStop = 0;
+ int njobsError = 0;
+ int njobsTimeOut = 0;
String id = job.getJobID();
String ip = job.getIP();
String protein = job.getProtein();
+ " VALUES ('" + id + "','" + ip + "','" + job.getStartingTimeStr() + "','" + job.getEndTimeStr() + "','" + finalstatus
+ "','" + execstatus + "','" + protein + "');";
session.execute(com1);
-
- String com2 = "INSERT INTO ProteinData " + "(jobtime, JobID, Protein)" + " VALUES (" + job.getStartingDate() + ",'" + id
- + "','" + protein + "');";
- session.execute(com2);
-
- String allpredictions = "";
- List<FastaSequence> pr = job.getPredictions();
- for (FastaSequence pred : pr) {
- String predictionname = pred.getId();
- String prediction = pred.getSequence().replaceAll("\n", "");
- allpredictions += "'" + predictionname + "':'" + prediction + "',";
- }
- String final_prediction = "";
- if (!allpredictions.equals("")) {
- final_prediction = allpredictions.substring(0, allpredictions.length() - 1);
- }
-
- String check2 = "SELECT * FROM ProteinRow WHERE JobID = '" + job.getJobID() + "';";
- ResultSet results2 = session.execute(check2);
- if (results2.isExhausted()) {
- String com3 = "INSERT INTO ProteinRow " + "(Protein, JobID, Predictions)" + " VALUES ('" + protein + "','" + id + "',{"
- + final_prediction + "});";
- session.execute(com3);
+ if (execstatus.equals("OK")) {
+ String com2 = "INSERT INTO ProteinData " + "(jobtime, JobID, ExecTime, Protein)" + " VALUES (" + job.getStartingDate() + ",'" + id
+ + "'," + job.getExecutionTime() + ",'" + protein + "');";
+ session.execute(com2);
+
+ String allpredictions = "";
+ List<FastaSequence> pr = job.getPredictions();
+ for (FastaSequence pred : pr) {
+ String predictionname = pred.getId();
+ String prediction = pred.getSequence().replaceAll("\n", "");
+ allpredictions += "'" + predictionname + "':'" + prediction + "',";
+ }
+ String final_prediction = "";
+ if (!allpredictions.equals("")) {
+ final_prediction = allpredictions.substring(0, allpredictions.length() - 1);
+ }
+
+ String check2 = "SELECT * FROM ProteinRow WHERE JobID = '" + job.getJobID() + "';";
+ ResultSet results2 = session.execute(check2);
+ if (results2.isExhausted()) {
+ String com3 = "INSERT INTO ProteinRow " + "(Protein, JobID, Predictions)" + " VALUES ('" + protein + "','" + id + "',{"
+ + final_prediction + "});";
+ session.execute(com3);
+ }
+ njobsOk = 1;
+ } else {
+ String com5 = "INSERT INTO FailLog " + "(jobtime, JobID, ExecTime, ip, FinalStatus)" + " VALUES ("
+ + job.getStartingDate() + ",'" + id
+ + "'," + job.getExecutionTime() + ",'" + ip + "', '" + finalstatus + "');";
+ session.execute(com5);
+ System.out.println(com5);
+ if (finalstatus.equals("TIMEDOUT"))
+ njobsTimeOut = 1;
+ else if (finalstatus.equals("JPREDERROR"))
+ njobsError = 1;
+ else if (finalstatus.equals("STOPPED"))
+ njobsStop = 1;
}
-
// update some internal query tables
String check3 = "SELECT * FROM MainParameters WHERE Name = 'EarliestJobDate';";
ResultSet results3 = session.execute(check3);
boolean updateparameter = true;
if (!results3.isExhausted()) {
- Row r = results3.one();
+ Row r = results3.one();
if (job.getStartingDate() >= Long.parseLong(r.getString("Value")))
updateparameter = false;
}
if (updateparameter) {
- String com = "INSERT INTO MainParameters " + "(Name, Value)" + " VALUES ('EarliestJobDate','" + job.getStartingDateStr()
+ String com = "INSERT INTO MainParameters " + "(Name, Value)" + " VALUES ('EarliestJobDate','" + job.getStartingDate()
+ "');";
session.execute(com);
}
if (!results4.isExhausted()) {
Row r = results4.one();
njobs += r.getLong("Total");
+ njobsOk += r.getLong("TotalOK");
+ njobsError += r.getLong("TotalError");
+ njobsStop += r.getLong("TotalStopped");
+ njobsTimeOut += r.getLong("TotalTimeOut");
}
- String com = "INSERT INTO JobDateInfo " + "(jobday, Total)" + " VALUES (" + job.getStartingDate() + "," + njobs + ");";
+ String com = "INSERT INTO JobDateInfo " + "(jobday, Total, TotalOK, TotalStopped, TotalError, TotalTimeOut)" + " VALUES ("
+ + job.getStartingDate() + "," + njobs + "," + njobsOk + "," + njobsStop + "," + njobsError + "," + njobsTimeOut + ");";
+ System.out.println(com);
session.execute(com);
-
return 1;
}
return 0;
if (JobisNotArchived(job.getJobID())) {
String id = job.getJobID();
String log = job.getLog().replaceAll("'", "");
- String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime,LOG, ArchiveLink) VALUES ('" + id + "','"
- + job.getProtein() + "','" + job.getIP() + "'," + job.getStartingTime() + "," + job.getExecutionTime() + ",'" + log
- + "','" + archivepath + "');";
+ String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime, FinalStatus, ExecutionStatus, LOG, ArchiveLink) VALUES ('" + id + "','"
+ + job.getProtein() + "','" + job.getIP() + "'," + job.getStartingTime() + "," + job.getExecutionTime() + ",'" + job.getFinalStatus() + "','"
+ + job.getExecutionStatus() + "','" + log + "','" + archivepath + "');";
session.execute(com);
-
List<FastaSequence> predictions = job.getPredictions();
for (FastaSequence p : predictions) {
session.execute("UPDATE JpredArchive SET predictions = predictions + {'" + p.getId() + "':'"
public class DataBase {
private String date;
- private int total;
+ private int total; // total jobs
+ private int totalOK; // total jobs with execution status OK
+ private int totalStopped; // total jobs with execution status STOPPED
+ private int totalError; // total jobs with execution status Jpred ERROR
+ private int totalTimeOut; // total jobs with execution status TIMEOUT
private int totalJobs;
- private int totalId;
+ private int totalId; // total jobs for current protein sequence
private String id;
- private String prot;
+ private String prot; // protein sequence
private String jpred;
- private List<String> subProt;
+ private List<String> subProt; // protein sequence divided by several parts for highlighting the particular part
private List<Integer> timeRez;
private List<Integer> timeTotalExec;
private StructureJobLog logInfo;
return total;
}
+ public void setTotalOK(int tot) {
+ this.totalOK = tot;
+ }
+
+ public int getTotalOK() {
+ return totalOK;
+ }
+
+ public void setTotalStopped(int tot) {
+ this.totalStopped = tot;
+ }
+
+ public int getTotalStopped() {
+ return totalStopped;
+ }
+
+ public void setTotalError(int tot) {
+ this.totalError = tot;
+ }
+
+ public int getTotalError() {
+ return totalError;
+ }
+
+ public void setTotalTimeOut(int tot) {
+ this.totalTimeOut = tot;
+ }
+
+ public int getTotalTimeOut() {
+ return totalTimeOut;
+ }
public void setTotalJobs(int totalJobs) {
this.totalJobs = totalJobs;
private String parsePredictions(final InputStream stream, String jobid) throws FileNotFoundException {
final FastaReader fr = new FastaReader(stream);
String protein = "";
- alignment = new ArrayList<FastaSequence>();
- predictions = new ArrayList<FastaSequence>();
+// alignment = new ArrayList<FastaSequence>();
+// predictions = new ArrayList<FastaSequence>();
while (fr.hasNext()) {
final FastaSequence fs = fr.next();
String seqid = fs.getId();
}
private int analyseJob(String[] jobinfo) throws IOException {
+ alignment = new ArrayList<FastaSequence>();
+ predictions = new ArrayList<FastaSequence>();
boolean running = true;
boolean ConcisefileExists = false;
boolean LogfileExists = false;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import compbio.cassandra.CassandraRemover;
import compbio.statistic.CassandraRequester;
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
- CassandraRequester cr = new CassandraRequester();
+ CassandraRemover cr = new CassandraRemover();
String flagId = request.getParameter("byId");
String jobId = request.getParameter("id");
+ String flagDate = request.getParameter("byDate");
String date1 = request.getParameter("date1");
String date2 = request.getParameter("date2");
- if (flagId.equals("on"))
- request.setAttribute("result", cr.countJobs(date1, date2));
- System.out.println(flagId);
+ String flagIp = request.getParameter("byIp");
+ String ip = request.getParameter("ip");
+ String flagSeq = request.getParameter("bySequence");
+ String seq = request.getParameter("seq");
+ if (flagId != null)
+ cr.RemoveJobById(jobId);
+ if (flagDate != null)
+ cr.RemoveJobByDate(date1, date2);
+ if (flagIp != null)
+ cr.RemoveJobByIp(ip);
+ if (flagSeq != null)
+ cr.RemoveJobBySequence(seq);
// request.setAttribute("IdJob", id);
// RequestDispatcher rd = request.getRequestDispatcher("/ReportLogInfo.jsp");
// rd.forward(request, response);
end.setTime(new Date(dateEnd));
query = new ArrayList<DataBase>();
for (Date date = start.getTime(); !start.after(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
- long res = db.ReadDateTable(date.getTime());
- DataBase db = new DataBase();
- db.setTotal((int)res);
- db.setDate(DateFormat(date.getTime()));
- query.add(db);
+ List<Long> res = db.ReadDateTable(date.getTime());
+ DataBase bean = new DataBase();
+ bean.setTotal((int)(long)res.get(0));
+ bean.setTotalOK((int)(long)res.get(1));
+ bean.setTotalStopped((int)(long)res.get(2));
+ bean.setTotalError((int)(long)res.get(3));
+ bean.setTotalTimeOut((int)(long)res.get(4));
+ bean.setDate(DateFormat(date.getTime()));
+ query.add(bean);
}
System.out.println("StatisticsProt.readLength: total number of dates = " + query.size());
return query;
query = new ArrayList<DataBase>();
Map<String, Integer> map = db.ReadProteinSequenceByCounter();
for (Map.Entry<String, Integer> entry : map.entrySet()) {
- if (entry.getValue() > minimalcounter) {
- DataBase db = new DataBase();
- db.setTotalId(entry.getValue());
- db.setProt(entry.getKey());
- query.add(db);
+ if (entry.getValue() > minimalcounter && entry.getKey().length() > 0) {
+ DataBase bean = new DataBase();
+ bean.setTotalId(entry.getValue());
+ bean.setProt(entry.getKey());
+ query.add(bean);
}
}
return query;
<h3>Time execution: ${timeExecution} ms</h3>
<c:set var="sum" value="0" />
+<c:set var="sumOK" value="0" />
+<c:set var="sumStopped" value="0" />
+<c:set var="sumError" value="0" />
+<c:set var="sumTimeOut" value="0" />
<c:forEach items="${result}" var="res">
<c:set var="tot" value="${res.total}" />
<c:set var="sum" value="${sum + tot}" />
+ <c:set var="totOK" value="${res.totalOK}" />
+ <c:set var="sumOK" value="${sumOK + totOK}" />
+ <c:set var="totStopped" value="${res.totalStopped}" />
+ <c:set var="sumStopped" value="${sumStopped + totStopped}" />
+ <c:set var="totError" value="${res.totalError}" />
+ <c:set var="sumError" value="${sumError + totError}" />
+ <c:set var="totTimeOut" value="${res.totalTimeOut}" />
+ <c:set var="sumTimeOut" value="${sumTimeOut + totTimeOut}" />
</c:forEach>
<thead>
<tr>
<th rowspan="2" style="text-align: centre">Date</th>
- <th colspan="4" style="text-align: centre">Number of Proteins</th>
+ <th rowspan="2" style="text-align: centre">Total jobs</th>
+ <th colspan="5" style="text-align: centre">Number of jobs</th>
</tr>
<tr>
- <th style="text-align: centre">Total</th>
- <th style="text-align: centre">Failed</th>
- <th style="text-align: centre">Cancelled</th>
- <th style="text-align: centre">Abandoned</th>
+ <th style="text-align: centre">Status "OK"</th>
+ <th style="text-align: centre">Status "Stopped"</th>
+ <th style="text-align: centre">Status "Error"</th>
+ <th style="text-align: centre">Status "Time out"</th>
</tr>
</thead>
<tbody>
<tr style="font-weight: bolder;">
<td>Total:</td>
<td style="text-align: right">${sum}</td>
- <td style="text-align: right">0</td>
- <td style="text-align: right">0</td>
- <td style="text-align: right">0</td>
+ <td style="text-align: right">${sumOK}</td>
+ <td style="text-align: right">${sumStopped}</td>
+ <td style="text-align: right">${sumError}</td>
+ <td style="text-align: right">${sumTimeOut}</td>
</tr>
<c:forEach items="${result}" var="res">
<tr>
<td>${res.date}</td>
<td style="text-align: right"><c:out value="${res.total}" /></td>
- <td style="text-align: right">0</td>
- <td style="text-align: right">0</td>
- <td style="text-align: right">0</td>
+ <td style="text-align: right">${res.totalOK}</td>
+ <td style="text-align: right">${res.totalStopped}</td>
+ <td style="text-align: right">${res.totalError}</td>
+ <td style="text-align: right">${res.totalTimeOut}</td>
</tr>
</c:forEach>
</div>
<input type="checkbox" name="byDate" onclick="show(this, 'date');">records for the period of dates
<div id='date' class="row" style="visibility:hidden;">
- from <input type="text" name="data1" value = <%= currentdate%> style=" width : 145px;"/>
- to <input type="text" name="data2" value = <%= currentdate%> />
+ from <input type="text" name="date1" value = <%= currentdate%> style=" width : 145px;"/>
+ to <input type="text" name="date2" value = <%= currentdate%> />
</div>
<input type="checkbox" name="byIp" onclick="show(this, 'ip');">records with ip
<div id='ip' class="row" style="visibility:hidden;">
git://source.jalview.org / proteocache.git / commitdiff