JAL-1061,JAL-1075 - first pass at implementing sequence record fetching across multip...
authorjprocter <jprocter@compbio.dundee.ac.uk>
Mon, 14 May 2012 10:12:43 +0000 (11:12 +0100)
committerjprocter <jprocter@compbio.dundee.ac.uk>
Mon, 14 May 2012 10:14:11 +0000 (11:14 +0100)
 Multiple source fetching not yet ready for production use (disabled for v2.8 release)

src/jalview/gui/SequenceFetcher.java

index 49511ad..111db90 100755 (executable)
@@ -18,6 +18,7 @@
 package jalview.gui;
 
 import java.util.*;
+import java.util.List;
 
 import java.awt.*;
 import java.awt.event.*;
@@ -29,6 +30,7 @@ import javax.swing.tree.TreeModel;
 
 import jalview.datamodel.*;
 import jalview.io.*;
+import jalview.util.DBRefUtils;
 import jalview.ws.dbsources.das.api.DasSourceRegistryI;
 import jalview.ws.seqfetcher.DbSourceProxy;
 
@@ -438,137 +440,245 @@ public class SequenceFetcher extends JPanel implements Runnable
       resetDialog();
       return;
     }
-    ArrayList<String> aresultq = new ArrayList<String>();
-    ArrayList<AlignmentI> aresult = new ArrayList<AlignmentI>();
-    DbSourceProxy proxy = database.getSelectedSources().get(0);
-    Enumeration en = new StringTokenizer(textArea.getText(), ";");
-    boolean isAliSource = false;
-    try
-    {
-      guiWindow.setProgressBar(
-              "Fetching Sequences from " + proxy.getDbName(), Thread
-                      .currentThread().hashCode());
-      isAliSource = proxy.isA(DBRefSource.ALIGNMENTDB);
-      if (proxy.getAccessionSeparator() == null)
+    // indicate if successive sources should be merged into one alignment.
+    boolean addToLast=false;
+    ArrayList<String> aresultq = new ArrayList<String>(), presultTitle = new ArrayList<String>();
+    ArrayList<AlignmentI> presult = new ArrayList<AlignmentI>(), aresult = new ArrayList<AlignmentI>();
+    Iterator<DbSourceProxy> proxies = database.getSelectedSources()
+            .iterator();
+    String[] qries;
+    List<String> nextfetch = Arrays.asList(qries = textArea.getText()
+            .split(";"));
+    Iterator<String> en = Arrays.asList(new String[0]).iterator();
+    int nqueries = qries.length;
+    while (proxies.hasNext() && (en.hasNext() || nextfetch.size() > 0))
+    {
+      if (!en.hasNext() && nextfetch.size() > 0)
       {
-        while (en.hasMoreElements())
+        en = nextfetch.iterator();
+        nqueries = nextfetch.size();
+        // save the remaining queries in the original array
+        qries = nextfetch.toArray(new String[nqueries]);
+        nextfetch = new ArrayList<String>();
+      }
+
+      DbSourceProxy proxy = proxies.next();
+      boolean isAliSource = false;
+      try
+      {
+        // update status
+        guiWindow.setProgressBar("Fetching " + nqueries
+                + " sequence queries from " + proxy.getDbName(), Thread
+                .currentThread().hashCode());
+        isAliSource = proxy.isA(DBRefSource.ALIGNMENTDB);
+        if (proxy.getAccessionSeparator() == null)
         {
-          String item = (String) en.nextElement();
-          try
+          while (en.hasNext())
           {
-            if (aresult != null)
+            String item = en.next();
+            try
             {
+              if (aresult != null)
+              {
+                try
+                {
+                  // give the server a chance to breathe
+                  Thread.sleep(5);
+                } catch (Exception e)
+                {
+                  //
+                }
+
+              }
+
+              AlignmentI indres = null;
               try
               {
-                // give the server a chance to breathe
-                Thread.sleep(5);
-              } catch (Exception e)
+                indres = proxy.getSequenceRecords(item);
+              } catch (OutOfMemoryError oome)
               {
-                //
+                new OOMWarning("fetching " + item + " from "
+                        + proxy.getDbName(), oome, this);
               }
-
+              if (indres != null)
+              {
+                aresultq.add(item);
+                aresult.add(indres);
+              }
+              else
+              {
+                nextfetch.add(item);
+              }
+            } catch (Exception e)
+            {
+              jalview.bin.Cache.log.info("Error retrieving " + item
+                      + " from " + proxy.getDbName(), e);
+              nextfetch.add(item);
             }
-
-            AlignmentI indres = null;
-            try
+          }
+        }
+        else
+        {
+          StringBuffer multiacc = new StringBuffer();
+          ArrayList<String> tosend = new ArrayList<String>();
+          while (en.hasNext())
+          {
+            String nel = en.next();
+            tosend.add(nel);
+            multiacc.append(nel);
+            if (en.hasNext())
             {
-              indres = proxy.getSequenceRecords(item);
-            } catch (OutOfMemoryError oome)
+              multiacc.append(proxy.getAccessionSeparator());
+            }
+          }
+          try
+          {
+            AlignmentI rslt;
+            SequenceI[] rs;
+            List<String> nores = new ArrayList<String>();
+            rslt = proxy.getSequenceRecords(multiacc.toString());
+            if (rslt == null || rslt.getHeight() == 0)
             {
-              new OOMWarning("fetching " + item + " from "
-                      + proxy.getDbName(), oome, this);
+              // no results - pass on all queries to next source
+              nextfetch.addAll(tosend);
             }
-            if (indres != null)
+            else
             {
-              aresultq.add(item);
-              aresult.add(indres);
+              aresultq.add(multiacc.toString());
+              aresult.add(rslt);
+
+              rs = rslt.getSequencesArray();
+              // search for each query in the dbrefs associated with each
+              // sequence
+              // returned.
+              // ones we do not find will be used to query next source (if any)
+              for (String q : tosend)
+              {
+                DBRefEntry dbr = new DBRefEntry(), found[] = null;
+                dbr.setSource(proxy.getDbSource());
+                boolean rfound = false;
+                for (int r = 0; r < rs.length; r++)
+                {
+                  if (rs[r] != null
+                          && (found = DBRefUtils.searchRefs(
+                                  rs[r].getDBRef(), dbr)) != null
+                          && found.length > 0)
+                  {
+                    rfound = true;
+                    rs[r] = null;
+                    continue;
+                  }
+                }
+                if (!rfound)
+                {
+                  nextfetch.add(q);
+                }
+              }
             }
-          } catch (Exception e)
+          } catch (OutOfMemoryError oome)
           {
-            jalview.bin.Cache.log.info("Error retrieving " + item
-                    + " from " + proxy.getDbName(), e);
+            new OOMWarning("fetching " + multiacc + " from "
+                    + database.getSelectedItem(), oome, this);
           }
         }
+
+      } catch (Exception e)
+      {
+        showErrorMessage("Error retrieving " + textArea.getText()
+                + " from " + database.getSelectedItem());
+        // error
+        // +="Couldn't retrieve sequences from "+database.getSelectedItem();
+        System.err.println("Retrieval failed for source ='"
+                + database.getSelectedItem() + "' and query\n'"
+                + textArea.getText() + "'\n");
+        e.printStackTrace();
+      } catch (OutOfMemoryError e)
+      {
+        // resets dialog box - so we don't use OOMwarning here.
+        showErrorMessage("Out of Memory when retrieving "
+                + textArea.getText()
+                + " from "
+                + database.getSelectedItem()
+                + "\nPlease see the Jalview FAQ for instructions for increasing the memory available to Jalview.\n");
+        e.printStackTrace();
+      } catch (Error e)
+      {
+        showErrorMessage("Serious Error retrieving " + textArea.getText()
+                + " from " + database.getSelectedItem());
+        e.printStackTrace();
       }
-      else
+      // Stack results ready for opening in alignment windows
+      if (aresult != null && aresult.size() > 0)
       {
-        StringBuffer multiacc = new StringBuffer();
-        while (en.hasMoreElements())
+        AlignmentI ar = null;
+        if (isAliSource)
         {
-          multiacc.append(en.nextElement());
-          if (en.hasMoreElements())
+          addToLast=false;
+          // new window for each result
+          while (aresult.size() > 0)
           {
-            multiacc.append(proxy.getAccessionSeparator());
+            presult.add(aresult.remove(0));
+            presultTitle.add(aresultq.remove(0) + " "
+                    + getDefaultRetrievalTitle());
           }
         }
-        try
-        {
-          aresultq.add(multiacc.toString());
-          aresult.add(proxy.getSequenceRecords(multiacc.toString()));
-        } catch (OutOfMemoryError oome)
+        else
         {
-          new OOMWarning("fetching " + multiacc + " from "
-                  + database.getSelectedItem(), oome, this);
+          String titl=null;
+          if (addToLast && presult.size()>0)
+          {
+            ar=presult.remove(presult.size()-1);
+            titl=presultTitle.remove(presultTitle.size()-1);
+          }
+          // concatenate all results in one window
+          while (aresult.size() > 0)
+          {
+            if (ar == null)
+            {
+              ar = aresult.remove(0);
+            }
+            else
+            {
+              ar.append(aresult.remove(0));
+            }
+            ;
+          }
+          addToLast=true;
+          presult.add(ar);
+          presultTitle.add(titl);
         }
-
       }
-
-    } catch (Exception e)
-    {
-      showErrorMessage("Error retrieving " + textArea.getText() + " from "
-              + database.getSelectedItem());
-      // error +="Couldn't retrieve sequences from "+database.getSelectedItem();
-      System.err.println("Retrieval failed for source ='"
-              + database.getSelectedItem() + "' and query\n'"
-              + textArea.getText() + "'\n");
-      e.printStackTrace();
-    } catch (OutOfMemoryError e)
-    {
-      // resets dialog box - so we don't use OOMwarning here.
-      showErrorMessage("Out of Memory when retrieving "
-              + textArea.getText()
-              + " from "
-              + database.getSelectedItem()
-              + "\nPlease see the Jalview FAQ for instructions for increasing the memory available to Jalview.\n");
-      e.printStackTrace();
-    } catch (Error e)
-    {
-      showErrorMessage("Serious Error retrieving " + textArea.getText()
-              + " from " + database.getSelectedItem());
-      e.printStackTrace();
+      guiWindow.setProgressBar("Finished querying", Thread.currentThread().hashCode());
     }
-    if (aresult != null && aresult.size() > 0)
+    guiWindow.setProgressBar((presult.size()>0) ? "Parsing results.":"Processing ..", Thread.currentThread().hashCode());
+    // process results
+    while (presult.size() > 0)
     {
-      AlignmentI ar = null;
-      if (isAliSource)
+      parseResult(presult.remove(0), presultTitle.remove(0), null);
+    }
+    // only remove visual delay after we finished parsing.
+    guiWindow.setProgressBar(null, Thread.currentThread().hashCode());
+    if (nextfetch.size() > 0)
+    {
+      StringBuffer sb = new StringBuffer();
+      sb.append("Didn't retrieve the following "
+              + (nextfetch.size() == 1 ? "query" : nextfetch.size()
+                      + " queries") + ": \n");
+      int l = sb.length(),lr=0;
+      for (String s : nextfetch)
       {
-        // new window for each result
-        while (aresult.size() > 0)
+        if (l != sb.length())
         {
-          parseResult(aresult.remove(0), aresultq.remove(0) + " "
-                  + getDefaultRetrievalTitle(), null);
+          sb.append("; ");
         }
-      }
-      else
-      {
-        // concatenate all results in one window
-        while (aresult.size() > 0)
+        if (lr-sb.length()>40)
         {
-          if (ar == null)
-          {
-            ar = aresult.remove(0);
-          }
-          else
-          {
-            ar.append(aresult.remove(0));
-          }
-          ;
+          sb.append("\n");
         }
-        parseResult(ar, null, null);
+        sb.append(s);
       }
+      showErrorMessage(sb.toString());
     }
-    // only remove visual delay after we finished parsing.
-    guiWindow.setProgressBar(null, Thread.currentThread().hashCode());
     resetDialog();
   }