datadb/compbio/cassandra/JpredParserHTTP.java

   1 package compbio.cassandra;
   2
   3 import java.io.BufferedReader;
   4 import java.io.FileNotFoundException;
   5 import java.io.IOException;
   6 import java.io.InputStream;
   7 import java.io.InputStreamReader;
   8 import java.net.HttpURLConnection;
   9 import java.net.MalformedURLException;
  10 import java.net.URL;
  11 import java.net.URLConnection;
  12 import java.util.ArrayList;
  13 import java.util.Calendar;
  14 import java.util.Date;
  15 import java.util.List;
  16
  17 import compbio.cassandra.JpredParser;
  18 import compbio.data.sequence.FastaReader;
  19 import compbio.data.sequence.FastaSequence;
  20 import compbio.engine.JpredJob;
  21
  22 public class JpredParserHTTP implements JpredParser {
  23         private CassandraWriter cw = new CassandraWriter();
  24         private String dirprefix;
  25         private List<FastaSequence> alignment;
  26         private List<FastaSequence> predictions;
  27         private int countNoData;
  28
  29         public JpredParserHTTP() {
  30                 dirprefix = "http://www.compbio.dundee.ac.uk/www-jpred/results";
  31         }
  32
  33         public JpredParserHTTP(String sourceurl) {
  34                 dirprefix = sourceurl;
  35         }
  36
  37         public void setSource(String newsourceprefix) {
  38                 dirprefix = newsourceprefix;
  39         }
  40
  41         public void Parsing(String source, int nDays) throws IOException {
  42                 Calendar cal = Calendar.getInstance();
  43                 cal.add(Calendar.DATE, -nDays);
  44                 for (int i = 0; i < nDays; ++i) {
  45                         cal.add(Calendar.DATE, 1);
  46                         String date = cal.get(Calendar.YEAR) + "/" + (cal.get(Calendar.MONTH) + 1) + "/" + cal.get(Calendar.DATE);
  47                         ParsingForDate(source, date);
  48                 }
  49         }
  50
  51         /*
  52          * The method parses the Jpred output concise file in the FASTA format If
  53          * there is a record with ID = QUERY or jobid, this a "one protein" job
  54          * otherwise this is an alignment job
  55          */
  56         private String parsePredictions(final InputStream stream, String jobid) throws FileNotFoundException {
  57                 final FastaReader fr = new FastaReader(stream);
  58                 String protein = "";
  59                 alignment = new ArrayList<FastaSequence>();
  60                 predictions = new ArrayList<FastaSequence>();
  61                 while (fr.hasNext()) {
  62                         final FastaSequence fs = fr.next();
  63                         String seqid = fs.getId();
  64                         String seq = fs.getSequence().replaceAll("\n", "");
  65                         if (seqid.equals("jnetpred") || seqid.equals("Lupas_21") || seqid.equals("Lupas_14") || seqid.equals("Lupas_28")
  66                                         || seqid.equals("JNETSOL25") || seqid.equals("JNETSOL5") || seqid.equals("JNETSOL0") || seqid.equals("JNETCONF")
  67                                         || seqid.equals("JNETHMM") || seqid.equals("JNETPSSM") || seqid.equals("JNETCONF")) {
  68                                 predictions.add(fs);
  69                         } else {
  70                                 alignment.add(fs);
  71                                 if (seqid.equals("QUERY") || seqid.equals(jobid))
  72                                         protein = seq;
  73                         }
  74                 }
  75                 return protein;
  76         }
  77
  78         private String parseLogFile(final InputStream stream) throws IOException {
  79                 String out = "";
  80                 BufferedReader buffer = new BufferedReader(new InputStreamReader(stream));
  81                 String line;
  82                 while (null != (line = buffer.readLine())) {
  83                         out += line;
  84                 }
  85                 return out;
  86         }
  87
  88         private int analyseJob(String[] jobinfo) throws IOException {
  89                 boolean running = true;
  90                 boolean ConcisefileExists = false;
  91                 boolean LogfileExists = false;
  92                 JpredJob job = new JpredJob (jobinfo[jobinfo.length - 1], jobinfo[0], jobinfo[1]);
  93                 job.setIP(jobinfo[2]);
  94                 Date currDate = new Date();
  95                 String maindir = dirprefix + "/" + job.getJobID() + "/";
  96
  97                 //System.out.println("analyzing job " + job.getJobID());
  98                 try {
  99                         URL dirurl = new URL(maindir);
 100                         HttpURLConnection httpConnection_dirurl = (HttpURLConnection) dirurl.openConnection();
 101                         if (httpConnection_dirurl.getResponseCode() < 199 || 300 <= httpConnection_dirurl.getResponseCode()) {
 102                                 return 0;
 103                         }
 104                         URL conciseurl = new URL(maindir + job.getJobID() + ".concise.fasta");
 105                         URL archiveurl = new URL(maindir + job.getJobID() + ".tar.gz");
 106                         URL logurl = new URL(maindir + "LOG");
 107                         HttpURLConnection httpConnection_conciseurl = (HttpURLConnection) conciseurl.openConnection();
 108                         HttpURLConnection httpConnection_logurl = (HttpURLConnection) logurl.openConnection();
 109                         HttpURLConnection httpConnection_archiveurl = (HttpURLConnection) archiveurl.openConnection();
 110                         if (199 < httpConnection_conciseurl.getResponseCode() && httpConnection_conciseurl.getResponseCode() < 300) {
 111                                 ConcisefileExists = true;
 112                                 running = false;
 113                                 try {
 114                                         job.setProtein(parsePredictions(conciseurl.openStream(), job.getJobID()));
 115                                 } catch (IOException e) {
 116                                         e.printStackTrace();
 117                                 }
 118                         } else {
 119                                 // The job still can be running of failed...
 120                                 ++countNoData;
 121                         }
 122                         if (199 < httpConnection_logurl.getResponseCode() && httpConnection_logurl.getResponseCode() < 300) {
 123                                 LogfileExists = true;
 124                                 job.setLog(parseLogFile(logurl.openStream()));
 125                         } else {
 126                                 // The job has not been started at all...
 127                                 job.setExecutionStatus("FAIL");
 128                                 job.setFinalStatus("STOPPED");
 129                                 running = false;
 130                         }
 131                         if (job.getLog().matches("(.*)TIMEOUT\\syour\\sjob\\stimed\\sout(.*)")) {
 132                                 // blast job was too long (more than 3600 secs by default)...
 133                                 job.setExecutionStatus("FAIL");
 134                                 job.setFinalStatus("TIMEDOUT");
 135                                 running = false;
 136                         } else if (job.getLog().matches("(.*)Jpred\\serror:\\sDied\\sat(.*)")) {
 137                                 // an internal Jpred error...
 138                                 job.setExecutionStatus("FAIL");
 139                                 job.setFinalStatus("JPREDERROR");
 140                                 running = false;
 141                         } else if ((currDate.getTime() - job.getEndTime()) / 1000 > 3601 && LogfileExists && !ConcisefileExists) {
 142                                 // the job was stopped with unknown reason...
 143                                 job.setExecutionStatus("FAIL");
 144                                 job.setFinalStatus("STOPPED");
 145                                 running = false;
 146                         }
 147
 148                         httpConnection_conciseurl.disconnect();
 149                         httpConnection_logurl.disconnect();
 150                         httpConnection_archiveurl.disconnect();
 151                 } catch (MalformedURLException e) {
 152                         e.printStackTrace();
 153                 }
 154
 155                 if (!running) {
 156                         job.setAlignment(alignment);
 157                         job.setPredictions(predictions);
 158                         cw.FormQueryTables(job);
 159                         cw.ArchiveData(job, "undefined");
 160                         return 1;
 161                 }
 162
 163                 return 0;
 164         }
 165
 166         private void ParsingForDate(String input, String date) {
 167                 int totalcount = 0;
 168                 int countinsertions = 0;
 169                 int countinserted = 0;
 170                 int countNotanalyzed = 0;
 171                 countNoData = 0;
 172
 173                 System.out.println("Inserting jobs for " + date);
 174                 try {
 175                         URL url = new URL(input);
 176                         URLConnection conn = url.openConnection();
 177                         BufferedReader alljobs = new BufferedReader(new InputStreamReader(conn.getInputStream()));
 178                         String line;
 179
 180                         while ((line = alljobs.readLine()) != null) {
 181                                 if (line.matches(date + ":(.*)jp_[^\\s]+")) {
 182                                         totalcount++;
 183                                         String[] job = line.split("\\s+");
 184                                         String jobid = job[job.length - 1];
 185                                         if (cw.JobisNotInsterted(jobid)) {
 186                                                 countinsertions += analyseJob(job);
 187                                         } else {
 188                                                 ++countinserted;
 189                                         }
 190                                 } else {
 191                                         ++countNotanalyzed;
 192                                 }
 193                         }
 194                         alljobs.close();
 195                         System.out.println("Total number of jobs = " + totalcount);
 196                         System.out.println("   " + countinserted + " jobs inserted already");
 197                         System.out.println("   " + countNotanalyzed + " not analysed jobs");
 198                         System.out.println("   " + countNoData + " jobs without *.concise.fasta file (RUNNING or FAILED)");
 199                         System.out.println("   " + countinsertions + " new job insertions\n");
 200                 } catch (MalformedURLException e) {
 201                         e.printStackTrace();
 202                 } catch (IOException e) {
 203                         e.printStackTrace();
 204                 }
 205                 ;
 206         }
 207 }