Add another parser for local files with metainformation

[proteocache.git] / datadb / compbio / cassandra / JpredParserLocalFile.java

diff --git a/datadb/compbio/cassandra/JpredParserLocalFile.java b/datadb/compbio/cassandra/JpredParserLocalFile.java
new file mode 100644 (file)
index 0000000..c37ec7a
--- /dev/null
+++ b/datadb/compbio/cassandra/JpredParserLocalFile.java
@@ -0,0 +1,131 @@
+package compbio.cassandra;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.io.FileInputStream;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+
+public class JpredParserLocalFile {
+       private CassandraCreate cc = new CassandraCreate();
+       private String dirprefix;
+       
+       public void setSource (String newsourceprefix) {
+               this.dirprefix = newsourceprefix;
+       }
+
+       JpredParserLocalFile() {
+               this.dirprefix = "/home/asherstnev/Projects/Java.projects/proteocache/data_stress_test/data.dat";
+       }
+       
+       JpredParserLocalFile(String sourceurl) {
+               this.dirprefix = sourceurl;
+       }
+
+       public void Parsing(String source, int nDays) {
+               Calendar cal = Calendar.getInstance();
+               cal.add(Calendar.DATE, -nDays);
+               for (int i = 0; i < nDays; ++i) {
+                       cal.add(Calendar.DATE, 1);
+                       int month = cal.get(Calendar.MONTH) + 1;
+                       int year = cal.get(Calendar.YEAR);
+                       int day = cal.get(Calendar.DATE);
+                       String date = year + "/" + month + "/" + day;
+                       ParsingForDate(source, date);
+               }
+       }
+
+       private void ParsingForDate(String input, String date) {
+               int totalcount = 0;
+               int countNoData = 0;
+               int countUnclearFASTAid = 0;
+               int countinsertions = 0;
+               int countinserted = 0;
+               int counAlignments = 0;
+               int countStrange = 0;
+
+               System.out.println("Inserting jobs for " + date);
+               try {
+                       File file = new File(input);
+                       BufferedReader alljobs = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
+                       String line;
+
+                       while ((line = alljobs.readLine()) != null) {
+                               if (line.matches(date + "(.*)jp_[^\\s]+")) {
+                                       String[] table = line.split("\\s+");
+                                       String id = table[table.length - 1];
+                                       totalcount++;
+                                       if (!cc.CheckID(id)) {
+                                               String confilename = dirprefix + "/" + id + "/" + id + ".concise";
+                                               File confile = new File(confilename);
+                                               if (confile.exists()) {
+                                                       try {
+                                                               final FastaReader fr = new FastaReader(confilename);
+                                                               final List<FastaSequence> seqs = new ArrayList<FastaSequence>();
+                                                               String newprotein = "";
+                                                               while (fr.hasNext()) {
+                                                                       final FastaSequence fs = fr.next();
+                                                                       if (fs.getId().equals("QUERY") || fs.getId().equals(id))
+                                                                               newprotein = fs.getSequence().replaceAll("\n", "");
+                                                                       else
+                                                                               seqs.add(fs);
+                                                               }
+                                                               if (newprotein.equals("")) {
+                                                                       countUnclearFASTAid++;
+                                                               } else {
+                                                                       SimpleDateFormat formatter = new SimpleDateFormat("yyyy/MM/dd");
+                                                                       String dateInString1 = table[0].substring(0, table[0].indexOf(":"));
+                                                                       long dateWork1 = 0;
+                                                                       try {
+                                                                               Date dat1 = formatter.parse(dateInString1);
+                                                                               dateWork1 = dat1.getTime();
+                                                                       } catch (ParseException e) {
+                                                                               e.printStackTrace();
+                                                                       }
+                                                                       cc.InsertData(dateWork1, table[0], table[1], table[2], id, "OK", "OK", newprotein, seqs);
+                                                                       ++countinsertions;
+                                                                       // flush every 100 insertions
+                                                                       if (0 == countinsertions % 100) {
+                                                                               cc.flushData();
+                                                                       }
+                                                               }
+                                                       } catch (IOException e) {
+                                                               e.printStackTrace();
+                                                       }
+                                               } else {
+                                                       countNoData++;
+                                               }
+                                       } else {
+                                               ++countinserted;
+                                       }
+                               } else {
+                                       if (line.matches(date + "(.*)Sequence0/(.*)")) {
+                                               ++counAlignments;
+                                       } else {
+                                               ++countStrange;
+                                       }
+                               }
+                       }
+                       alljobs.close();
+                       System.out.println("Total number of jobs = " + totalcount);
+                       System.out.println("   " + countinserted + " jobs inserted already");
+                       System.out.println("   " + counAlignments + " jalview jobs");
+                       System.out.println("   " + countStrange + " not analysed jobs");
+                       System.out.println("   " + countNoData + " jobs without *.concise.fasta file");
+                       System.out.println("   " + countUnclearFASTAid + " jobs with unclear FASTA protein id in *.concise.fasta");
+                       System.out.println("   " + countinsertions + " new job insertions\n");
+               } catch (MalformedURLException e) {
+                       e.printStackTrace();
+               } catch (IOException e) {
+                       e.printStackTrace();
+               }
+       }
+}