From db09514e58b52d44ef52d7c4591924a762c99c4b Mon Sep 17 00:00:00 2001 From: Natasha Sherstneva Date: Mon, 20 Jan 2014 10:56:16 +0000 Subject: [PATCH] inserted 2 columns: version and pragramme name --- conf/Proteocache.properties | 5 +-- datadb/compbio/beans/JobBean.java | 14 +++++++- .../cassandra/CassandraNativeConnector.java | 25 ++++++-------- .../compbio/cassandra/CassandraNewTableWriter.java | 34 ++++++++++++++++++++ datadb/compbio/cassandra/CassandraReaderOld.java | 2 +- datadb/compbio/cassandra/CassandraWriter.java | 10 +++--- datadb/compbio/cassandra/JpredParserHTTP.java | 16 ++++++--- datadb/compbio/cassandra/readers/JobReader.java | 2 +- engine/compbio/engine/Job.java | 21 ++++++++++++ server/compbio/listeners/ContextListener.java | 13 ++++++++ webapp/view/reports/Job.jsp | 2 ++ 11 files changed, 115 insertions(+), 29 deletions(-) diff --git a/conf/Proteocache.properties b/conf/Proteocache.properties index fb855a3..b5f588f 100644 --- a/conf/Proteocache.properties +++ b/conf/Proteocache.properties @@ -1,10 +1,11 @@ ################################################################################# # Cassandra host or IP # test server is 10.0.115.190 -#cassandra.host=localhost -cassandra.host=gjb-www-1.cluster.lifesci.dundee.ac.uk +cassandra.host=localhost +#cassandra.host=gjb-www-1.cluster.lifesci.dundee.ac.uk cassandra.newtables.update=false +cassandra.version.update=false ################################################################################# # Jpred sources diff --git a/datadb/compbio/beans/JobBean.java b/datadb/compbio/beans/JobBean.java index 8176748..05c798a 100644 --- a/datadb/compbio/beans/JobBean.java +++ b/datadb/compbio/beans/JobBean.java @@ -11,10 +11,14 @@ public class JobBean implements PredictionIndex { private String dateEnd; private String sequence; private String ip; + private String programmeName; + private String version; private LinkedHashMap predictions; - public JobBean(String seq, String id, String dateStart, String dateEnd, String ip, Map pred) { + public JobBean(String seq, String id, String dateStart, String dateEnd, String ip, Map pred, String prog, String version) { this.sequence = seq; + this.programmeName = prog; + this.version = version; this.jobid = id; this.dateStart = dateStart; this.dateEnd = dateEnd; @@ -50,6 +54,14 @@ public class JobBean implements PredictionIndex { public String getIp() { return ip; } + + public String getProgrammeName() { + return programmeName; + } + + public String getVersion() { + return version; + } public Map getPrediction() { return predictions; diff --git a/datadb/compbio/cassandra/CassandraNativeConnector.java b/datadb/compbio/cassandra/CassandraNativeConnector.java index 9d8d01d..beb83c4 100644 --- a/datadb/compbio/cassandra/CassandraNativeConnector.java +++ b/datadb/compbio/cassandra/CassandraNativeConnector.java @@ -73,7 +73,7 @@ public class CassandraNativeConnector { session.execute("CREATE TABLE IF NOT EXISTS ProteinLog " + "(JobID ascii, DataBegin ascii, DataEnd ascii, ip ascii, FinalStatus ascii, " - + "ExecutionStatus ascii, Protein ascii, PRIMARY KEY(JobID));"); + + "ExecutionStatus ascii, Protein ascii, ProgrammeName ascii, Version ascii, PRIMARY KEY(JobID));"); session.execute("CREATE TABLE IF NOT EXISTS ProteinData " + "(jobtime bigint, JobID ascii, ExecTime int, Protein ascii, PRIMARY KEY(jobtime, JobID));"); @@ -83,7 +83,7 @@ public class CassandraNativeConnector { session.execute("CREATE TABLE IF NOT EXISTS JpredArchive " + "(JobID ascii, Protein varchar, IP ascii, StartTime bigint, ExecTime int, FinalStatus ascii, ExecutionStatus ascii, alignment map, " - + "predictions map, ArchiveLink varchar, LOG varchar, PRIMARY KEY(JobID));"); + + "predictions map, ArchiveLink varchar, LOG varchar, ProgrammeName ascii, Version ascii, PRIMARY KEY(JobID));"); session.execute("CREATE TABLE IF NOT EXISTS JobDateInfo " + "(jobday bigint, Total bigint, TotalOK bigint, TotalStopped bigint, TotalError bigint, TotalTimeOut bigint, Program varchar, Version varchar, PRIMARY KEY(jobday));"); @@ -91,20 +91,13 @@ public class CassandraNativeConnector { String com = "CREATE TABLE IF NOT EXISTS Users " + "(name varchar, id bigint, email varchar, password varchar, organisation varchar, position varchar, signedtolist boolean, registrationdate bigint, PRIMARY KEY(id));"; - try { - session.execute(com); - } catch (QueryExecutionException e) { - String mess = "CassandraNativeConnector.CreateMainTables: query execution exception..."; - System.out.println(mess); - log.error(mess); - log.error(e.getLocalizedMessage(), e.getCause()); - } catch (QueryValidationException e) { - String mess = "CassandraNativeConnector.CreateMainTables: query validation exception... Command: " + com; - System.out.println(mess); - log.error(mess); - log.error(e.getLocalizedMessage(), e.getCause()); - } - + + + // session.execute("ALTER TABLE ProteinLog ADD ProgrammeName ascii;"); + // session.execute("ALTER TABLE ProteinLog ADD Version ascii;"); + // session.execute("ALTER TABLE JpredArchive ADD ProgrammeName ascii;"); + // session.execute("ALTER TABLE JpredArchive ADD Version ascii;"); + session.execute("CREATE INDEX IF NOT EXISTS ProteinSeq ON ProteinRow (protein);"); session.execute("CREATE INDEX IF NOT EXISTS ProteinIp ON ProteinLog (ip);"); session.execute("CREATE INDEX IF NOT EXISTS ON ProteinLog (ExecutionStatus);"); diff --git a/datadb/compbio/cassandra/CassandraNewTableWriter.java b/datadb/compbio/cassandra/CassandraNewTableWriter.java index 0d68b1a..6cb9c98 100644 --- a/datadb/compbio/cassandra/CassandraNewTableWriter.java +++ b/datadb/compbio/cassandra/CassandraNewTableWriter.java @@ -2,6 +2,7 @@ package compbio.cassandra; import java.util.Calendar; import java.util.Date; +import java.util.List; import org.apache.log4j.Logger; @@ -44,6 +45,39 @@ public class CassandraNewTableWriter { } return false; } + + /* + * update programme name and version + */ + public void FillNewParameter() { + System.out.println("Updating...."); + + final long StartTime = System.currentTimeMillis(); + long erldate = CassandraNativeConnector.getEarliestDateInDB(); + Calendar runnicCal = Calendar.getInstance(); + runnicCal.setTime(new Date(erldate)); + Calendar endcal = Calendar.getInstance(); + long endTime = endcal.getTime().getTime(); + for (Date date = runnicCal.getTime(); date.getTime() < endTime; runnicCal.add(Calendar.DATE, 1), date = runnicCal.getTime()) { + try { + String com = "SELECT JobID, Protein FROM ProteinData WHERE jobtime = " + date.getTime() + ";"; + System.out.println(com); + ResultSet results = session.execute(com); + if (results == null || results.isExhausted()) + continue; + List rows = results.all(); + System.out.println(rows.size()); + for (Row r : rows) { + session.execute("UPDATE ProteinLog SET ProgrammeName = 'Jpred', Version = '3.0.1' WHERE JobID = '" + r.getString("JobID") + "';"); + session.execute("UPDATE JpredArchive SET ProgrammeName = 'Jpred', Version = '3.0.1' WHERE JobID = '" + r.getString("JobID") + "';"); + } + } catch (QueryExecutionException e) { + e.printStackTrace(); + } + } + System.out.println("Table ProteinLog filled: total time is " + (System.currentTimeMillis() - StartTime) + " msec"); + } + /* * fill new table diff --git a/datadb/compbio/cassandra/CassandraReaderOld.java b/datadb/compbio/cassandra/CassandraReaderOld.java index c7176b0..62979ad 100644 --- a/datadb/compbio/cassandra/CassandraReaderOld.java +++ b/datadb/compbio/cassandra/CassandraReaderOld.java @@ -314,7 +314,7 @@ public class CassandraReaderOld { return null; Row row1 = results1.one(); JobBean res = new JobBean(row.getString("Protein"), row.getString("JobID"), row.getString("DataBegin"), row.getString("DataEnd"), - row.getString("ip"), row1.getMap("Predictions", String.class, String.class)); + row.getString("ip"), row1.getMap("Predictions", String.class, String.class), row.getString("ProgrammeName"), row.getString("Version")); System.out.println("Query time is " + (queryTime - startTime) + " msec"); final long endTime = System.currentTimeMillis(); System.out.println(" rows analysed, execution time is " + (endTime - startTime) + " msec"); diff --git a/datadb/compbio/cassandra/CassandraWriter.java b/datadb/compbio/cassandra/CassandraWriter.java index 710d78b..05a03f0 100644 --- a/datadb/compbio/cassandra/CassandraWriter.java +++ b/datadb/compbio/cassandra/CassandraWriter.java @@ -74,10 +74,12 @@ public class CassandraWriter { String protein = job.getProtein(); String finalstatus = job.getFinalStatus(); String execstatus = job.getExecutionStatus(); + String programmeName = job.getProgrammeName(); + String version = job.getVersion(); - String com1 = "INSERT INTO ProteinLog (JobID, IP, DataBegin, DataEnd, FinalStatus, ExecutionStatus, Protein) VALUES ('" + id + String com1 = "INSERT INTO ProteinLog (JobID, IP, DataBegin, DataEnd, FinalStatus, ExecutionStatus, Protein, ProgrammeName, Version) VALUES ('" + id + "','" + job.getIP() + "','" + job.getStartingTimeStr() + "','" + job.getEndTimeStr() + "','" + finalstatus + "','" - + execstatus + "','" + protein + "');"; + + execstatus + "','" + protein + "','" + programmeName + "','" + version + "');"; ResultSet insert = execute(com1); if (null == insert) { System.out.println("CassandraWriter.FormQueryTables: couldn't insert into ProteinLog"); @@ -191,7 +193,7 @@ public class CassandraWriter { if (JobisNotArchived(job.getJobID())) { String id = job.getJobID(); String log = job.getLog().replaceAll("'", ""); - String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime, FinalStatus, ExecutionStatus, LOG, ArchiveLink) VALUES ('" + String com = "INSERT INTO JpredArchive (JobID, Protein, IP, StartTime, ExecTime, FinalStatus, ExecutionStatus, LOG, ArchiveLink, ProgrammeName, Version) VALUES ('" + id + "','" + job.getProtein() @@ -201,7 +203,7 @@ public class CassandraWriter { + job.getStartingTime() + "," + job.getExecutionTime() - + ",'" + job.getFinalStatus() + "','" + job.getExecutionStatus() + "','" + log + "','" + archivepath + "');"; + + ",'" + job.getFinalStatus() + "','" + job.getExecutionStatus() + "','" + log + "','" + archivepath + "','" + job.getProgrammeName() + "','" + job.getVersion() + "');"; if (null == execute(com)) { System.out.println("CassandraWriter.ArchiveData: couldn't insert into JpredArchive"); } diff --git a/datadb/compbio/cassandra/JpredParserHTTP.java b/datadb/compbio/cassandra/JpredParserHTTP.java index d221209..1d888e4 100644 --- a/datadb/compbio/cassandra/JpredParserHTTP.java +++ b/datadb/compbio/cassandra/JpredParserHTTP.java @@ -13,6 +13,8 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import compbio.cassandra.JpredParser; import compbio.data.sequence.FastaReader; @@ -106,12 +108,17 @@ public class JpredParserHTTP implements JpredParser { return protein; } - private String parseLogFile(final InputStream stream) throws IOException { + private String parseLogFile(final InputStream stream, JpredJob job) throws IOException { String out = ""; BufferedReader buffer = new BufferedReader(new InputStreamReader(stream)); String line; + if (null != (out = buffer.readLine()) && (out.contains("version"))) { + Matcher matcher = Pattern.compile("((\\d|\\.)+)").matcher(out); + if (matcher.find()) + job.setVersion(matcher.group(0)); + } while (null != (line = buffer.readLine())) { - out += line; + out += line; } return out; } @@ -140,7 +147,7 @@ public class JpredParserHTTP implements JpredParser { if (199 < httpConnection_conciseurl.getResponseCode() && httpConnection_conciseurl.getResponseCode() < 300) { ConcisefileExists = true; running = false; - try { + try { job.setProtein(parsePredictions(conciseurl.openStream(), job.getJobID())); } catch (IOException e) { e.printStackTrace(); @@ -151,7 +158,8 @@ public class JpredParserHTTP implements JpredParser { } if (199 < httpConnection_logurl.getResponseCode() && httpConnection_logurl.getResponseCode() < 300) { LogfileExists = true; - job.setLog(parseLogFile(logurl.openStream())); + job.setProgrammeName("Jpred"); + job.setLog(parseLogFile(logurl.openStream(), job)); } else { // The job has not been started at all... job.setExecutionStatus("FAIL"); diff --git a/datadb/compbio/cassandra/readers/JobReader.java b/datadb/compbio/cassandra/readers/JobReader.java index 178ff4b..c632542 100644 --- a/datadb/compbio/cassandra/readers/JobReader.java +++ b/datadb/compbio/cassandra/readers/JobReader.java @@ -29,7 +29,7 @@ public class JobReader extends CassandraReader { return null; Row row1 = results1.one(); JobBean res = new JobBean(row.getString("Protein"), row.getString("JobID"), row.getString("DataBegin"), row.getString("DataEnd"), - row.getString("ip"), row1.getMap("Predictions", String.class, String.class)); + row.getString("ip"), row1.getMap("Predictions", String.class, String.class), row.getString("ProgrammeName"), row.getString("Version")); return res; } } diff --git a/engine/compbio/engine/Job.java b/engine/compbio/engine/Job.java index e1de779..162a246 100644 --- a/engine/compbio/engine/Job.java +++ b/engine/compbio/engine/Job.java @@ -10,6 +10,8 @@ public class Job { private String FinalStatus; private String IP; private String Email; + private String programmeName; + private String version; private String StartingDateStr; private long StartingDate; @@ -18,6 +20,7 @@ public class Job { private String StartingTimeStr; private long StartingTime; private long ExecutionTime; + static SimpleDateFormat dateformatter = new SimpleDateFormat("yyyy/MM/dd");; static SimpleDateFormat timeformatter = new SimpleDateFormat("yyyy/MM/dd:H:m:s"); @@ -26,6 +29,8 @@ public class Job { ExecutionStatus = "OK"; FinalStatus = "OK"; jobid = ""; + programmeName =""; + version = ""; StartingDateStr = "1970/1/1"; StartingTimeStr = "1970/1/1:0:0:0"; @@ -166,4 +171,20 @@ public class Job { public String getJobID() { return jobid; } + + public void setProgrammeName(String prog) { + this.programmeName = prog; + } + + public String getProgrammeName() { + return programmeName; + } + + public void setVersion(String ver) { + this.version = ver; + } + + public String getVersion() { + return version; + } } diff --git a/server/compbio/listeners/ContextListener.java b/server/compbio/listeners/ContextListener.java index 356828c..f673e30 100644 --- a/server/compbio/listeners/ContextListener.java +++ b/server/compbio/listeners/ContextListener.java @@ -39,6 +39,7 @@ public class ContextListener implements ServletContextListener { public static boolean READ_WEB_JPRED = false; public static boolean READ_LOCALFILE_JPRED = false; public static boolean FILL_NEWTABLE = false; + public static boolean UPDATE_TABLE = false; private static boolean initBooleanValue(String key) { assert key != null; @@ -62,8 +63,20 @@ public class ContextListener implements ServletContextListener { READ_WEB_JPRED = initBooleanValue("cassandra.jpred.web.update"); READ_LOCALFILE_JPRED = initBooleanValue("cassandra.jpred.local.update"); FILL_NEWTABLE = initBooleanValue("cassandra.newtables.update"); + UPDATE_TABLE = initBooleanValue("cassandra.version.update"); executor = Executors.newScheduledThreadPool(3); + if (UPDATE_TABLE) { + System.out.println("Initializating update version parameter"); + //update_scheduler = Executors.newSingleThreadScheduledExecutor(); + //update_scheduler.schedule(new Runnable() { + executor.schedule(new Runnable() { + @Override + public void run() { + updater.FillNewParameter(); + } + }, 10, TimeUnit.SECONDS); + } if (FILL_NEWTABLE) { System.out.println("Initializating new table update scheduler"); //update_scheduler = Executors.newSingleThreadScheduledExecutor(); diff --git a/webapp/view/reports/Job.jsp b/webapp/view/reports/Job.jsp index 64d7896..db8f352 100644 --- a/webapp/view/reports/Job.jsp +++ b/webapp/view/reports/Job.jsp @@ -21,6 +21,8 @@

Start timestamp : ${result.dateStart}

End timestamp : ${result.dateEnd}

+

Run : ${result.programmeName} server

+

Version : ${result.version}

IP: ${result.ip}

 -- 1.7.10.2