Fix problem with too many open files and problem with not-flushed

[proteocache.git] / datadb / compbio / cassandra / JpredParserHTTP.java

diff --git a/datadb/compbio/cassandra/JpredParserHTTP.java b/datadb/compbio/cassandra/JpredParserHTTP.java
index e308a25..052ff6a 100644 (file)
--- a/datadb/compbio/cassandra/JpredParserHTTP.java
+++ b/datadb/compbio/cassandra/JpredParserHTTP.java
@@ -41,11 +41,13 @@ public class JpredParserHTTP implements JpredParser {
                        int year = cal.get(Calendar.YEAR);
                        int day = cal.get(Calendar.DATE);
                        String date = year + "/" + month + "/" + day;
-                       ParsingForDate(source, date);
+                       if (0 < ParsingForDate(source, date)) {
+                               cc.flushData();
+                       }
                }
        }
 
-       private void ParsingForDate(String input, String date) {
+       private int ParsingForDate(String input, String date) {
                int totalcount = 0;
                int countNoData = 0;
                int countUnclearFASTAid = 0;
@@ -53,6 +55,7 @@ public class JpredParserHTTP implements JpredParser {
                int countinserted = 0;
                int counAlignments = 0;
                int countStrange = 0;
+               int njobs = 0;
 
                System.out.println("Inserting jobs for " + date);
                try {
@@ -100,9 +103,11 @@ public class JpredParserHTTP implements JpredParser {
                                                                        }
                                                                        cc.InsertData(dateWork1, table[0], table[1], table[2], id, "OK", "OK", newprotein, seqs);
                                                                        ++countinsertions;
-                                                                       // flush every 100 insertions
-                                                                       if (0 == countinsertions % 100) {
+                                                                       ++njobs;
+                                                                       // flush every 50 insertions
+                                                                       if (0 == countinsertions % 50) {
                                                                                cc.flushData();
+                                                                               njobs -= 50;
                                                                        }
                                                                }
                                                        } catch (IOException e) {
@@ -135,5 +140,6 @@ public class JpredParserHTTP implements JpredParser {
                } catch (IOException e) {
                        e.printStackTrace();
                }
+               return njobs;
        }
 }