X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fext%2Fensembl%2FEnsemblRestClient.java;h=24375886005dce6264b763e5dc6f12fdb0ebba2c;hb=136c0793b90b72b928c4d77dc109dd5c644e00d3;hp=6a564f10e9655a90f91cc5a9f0fa35d27f12b9ff;hpb=d547268f85f377f14dbeb169072caebad45f9e2b;p=jalview.git

diff --git a/src/jalview/ext/ensembl/EnsemblRestClient.java b/src/jalview/ext/ensembl/EnsemblRestClient.java
index 6a564f1..2437588 100644
--- a/src/jalview/ext/ensembl/EnsemblRestClient.java
+++ b/src/jalview/ext/ensembl/EnsemblRestClient.java
@@ -1,6 +1,28 @@
+/*
+ * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
+ * Copyright (C) $$Year-Rel$$ The Jalview Authors
+ * 
+ * This file is part of Jalview.
+ * 
+ * Jalview is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License 
+ * as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
+ *  
+ * Jalview is distributed in the hope that it will be useful, but 
+ * WITHOUT ANY WARRANTY; without even the implied warranty 
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR 
+ * PURPOSE.  See the GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
+ * The Jalview Authors are detailed in the 'AUTHORS' file.
+ */
 package jalview.ext.ensembl;
 
+import jalview.io.DataSourceType;
 import jalview.io.FileParse;
+import jalview.util.StringUtils;
 
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
@@ -10,10 +32,16 @@ import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.ws.rs.HttpMethod;
 
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
 import com.stevesoft.pat.Regex;
 
 /**
@@ -23,31 +51,48 @@ import com.stevesoft.pat.Regex;
  */
 abstract class EnsemblRestClient extends EnsemblSequenceFetcher
 {
-  private final static String ENSEMBL_REST = "http://rest.ensembl.org";
+  private static final int DEFAULT_READ_TIMEOUT = 5 * 60 * 1000; // 5 minutes
+
+  private static final int CONNECT_TIMEOUT_MS = 10 * 1000; // 10 seconds
+
+  /*
+   * update these constants when Jalview has been checked / updated for
+   * changes to Ensembl REST API (ref JAL-2105)
+   * @see https://github.com/Ensembl/ensembl-rest/wiki/Change-log
+   * @see http://rest.ensembl.org/info/rest?content-type=application/json
+   */
+  private static final String LATEST_ENSEMBLGENOMES_REST_VERSION = "5.0";
+
+  private static final String LATEST_ENSEMBL_REST_VERSION = "5.0";
 
-  protected final static String ENSEMBL_GENOMES_REST = "http://rest.ensemblgenomes.org";
+  private static final String REST_CHANGE_LOG = "https://github.com/Ensembl/ensembl-rest/wiki/Change-log";
+
+  private static Map<String, EnsemblInfo> domainData;
 
   // @see https://github.com/Ensembl/ensembl-rest/wiki/Output-formats
   private static final String PING_URL = "http://rest.ensembl.org/info/ping.json";
 
-  private final static long RETEST_INTERVAL = 10000L; // 10 seconds
-
-  private static final Regex TRANSCRIPT_REGEX = new Regex(
-            "(ENS)([A-Z]{3}|)T[0-9]{11}$");
+  private final static long AVAILABILITY_RETEST_INTERVAL = 10000L; // 10 seconds
 
-  private static final Regex GENE_REGEX = new Regex(
-            "(ENS)([A-Z]{3}|)G[0-9]{11}$");
+  private final static long VERSION_RETEST_INTERVAL = 1000L * 3600; // 1 hr
 
-  private String domain = ENSEMBL_REST;
+  private static final Regex PROTEIN_REGEX = new Regex(
+          "(ENS)([A-Z]{3}|)P[0-9]{11}$");
 
-  private static boolean ensemblRestAvailable = false;
+  private static final Regex TRANSCRIPT_REGEX = new Regex(
+          "(ENS)([A-Z]{3}|)T[0-9]{11}$");
 
-  private static long lastCheck = -1;
+  private static final Regex GENE_REGEX = new Regex(
+          "(ENS)([A-Z]{3}|)G[0-9]{11}$");
 
-  /*
-   * absolute time to wait till if we overloaded the REST service
-   */
-  private static long retryAfter;
+  static
+  {
+    domainData = new HashMap<String, EnsemblInfo>();
+    domainData.put(ENSEMBL_REST, new EnsemblInfo(ENSEMBL_REST,
+            LATEST_ENSEMBL_REST_VERSION));
+    domainData.put(ENSEMBL_GENOMES_REST, new EnsemblInfo(
+            ENSEMBL_GENOMES_REST, LATEST_ENSEMBLGENOMES_REST_VERSION));
+  }
 
   protected volatile boolean inProgress = false;
 
@@ -66,35 +111,31 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
    */
   public EnsemblRestClient(String d)
   {
-    domain = d;
+    setDomain(d);
   }
 
   /**
-   * Returns the domain name to query e.g. http://rest.ensembl.org or
-   * http://rest.ensemblgenomes.org
+   * Answers true if the query matches the regular expression pattern for an
+   * Ensembl transcript stable identifier
    * 
+   * @param query
    * @return
    */
-  String getDomain()
-  {
-    return domain;
-  }
-
-  void setDomain(String d)
+  public boolean isTranscriptIdentifier(String query)
   {
-    domain = d;
+    return query == null ? false : TRANSCRIPT_REGEX.search(query);
   }
 
   /**
    * Answers true if the query matches the regular expression pattern for an
-   * Ensembl transcript stable identifier
+   * Ensembl protein stable identifier
    * 
    * @param query
    * @return
    */
-  public boolean isTranscriptIdentifier(String query)
+  public boolean isProteinIdentifier(String query)
   {
-    return query == null ? false : TRANSCRIPT_REGEX.search(query);
+    return query == null ? false : PROTEIN_REGEX.search(query);
   }
 
   /**
@@ -157,30 +198,47 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   protected abstract String getResponseMimeType();
 
   /**
-   * Tries to connect to Ensembl's REST 'ping' endpoint, and returns true if
-   * successful, else false
+   * Checks Ensembl's REST 'ping' endpoint, and returns true if response
+   * indicates available, else false
    * 
+   * @see http://rest.ensembl.org/documentation/info/ping
    * @return
    */
   private boolean checkEnsembl()
   {
+    BufferedReader br = null;
     try
     {
       // note this format works for both ensembl and ensemblgenomes
       // info/ping.json works for ensembl only (March 2016)
       URL ping = new URL(getDomain()
               + "/info/ping?content-type=application/json");
-      HttpURLConnection conn = (HttpURLConnection) ping.openConnection();
-      int rc = conn.getResponseCode();
-      conn.disconnect();
-      if (rc >= 200 && rc < 300)
-      {
-        return true;
-      }
+
+      /*
+       * expect {"ping":1} if ok
+       * if ping takes more than 2 seconds to respond, treat as if unavailable
+       */
+      br = getHttpResponse(ping, null, 2 * 1000);
+      JSONParser jp = new JSONParser();
+      JSONObject val = (JSONObject) jp.parse(br);
+      String pingString = val.get("ping").toString();
+      return pingString != null;
     } catch (Throwable t)
     {
       System.err.println("Error connecting to " + PING_URL + ": "
               + t.getMessage());
+    } finally
+    {
+      if (br != null)
+      {
+        try
+        {
+          br.close();
+        } catch (IOException e)
+        {
+          // ignore
+        }
+      }
     }
     return false;
   }
@@ -196,33 +254,55 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
           throws IOException
   {
     URL url = getUrl(ids);
-  
+
     BufferedReader reader = getHttpResponse(url, ids);
-    FileParse fp = new FileParse(reader, url.toString(), "HTTP_POST");
+    if (reader == null)
+    {
+      // request failed
+      return null;
+    }
+    FileParse fp = new FileParse(reader, url.toString(), DataSourceType.URL);
     return fp;
   }
 
   /**
+   * Gets a reader to the HTTP response, using the default read timeout of 5
+   * minutes
+   * 
+   * @param url
+   * @param ids
+   * @return
+   * @throws IOException
+   */
+  protected BufferedReader getHttpResponse(URL url, List<String> ids)
+          throws IOException
+  {
+    return getHttpResponse(url, ids, DEFAULT_READ_TIMEOUT);
+  }
+
+  /**
    * Writes the HTTP request and gets the response as a reader.
    * 
    * @param url
    * @param ids
    *          written as Json POST body if more than one
+   * @param readTimeout
+   *          in milliseconds
    * @return
    * @throws IOException
    *           if response code was not 200, or other I/O error
    */
-  protected BufferedReader getHttpResponse(URL url, List<String> ids)
-          throws IOException
+  protected BufferedReader getHttpResponse(URL url, List<String> ids,
+          int readTimeout) throws IOException
   {
     // long now = System.currentTimeMillis();
     HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-  
+
     /*
      * POST method allows multiple queries in one request; it is supported for
      * sequence queries, but not for overlap
      */
-    boolean multipleIds = ids.size() > 1;// useGetRequest();
+    boolean multipleIds = ids != null && ids.size() > 1;
     connection.setRequestMethod(multipleIds ? HttpMethod.POST
             : HttpMethod.GET);
     connection.setRequestProperty("Content-Type",
@@ -233,29 +313,33 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
     connection.setDoInput(true);
     connection.setDoOutput(multipleIds);
 
+    connection.setConnectTimeout(CONNECT_TIMEOUT_MS);
+    connection.setReadTimeout(readTimeout);
+
     if (multipleIds)
     {
       writePostBody(connection, ids);
     }
-  
-    InputStream response = connection.getInputStream();
+
     int responseCode = connection.getResponseCode();
-  
+
     if (responseCode != 200)
     {
       /*
        * note: a GET request for an invalid id returns an error code e.g. 415
        * but POST request returns 200 and an empty Fasta response 
        */
-      throw new IOException(
-              "Response code was not 200. Detected response was "
-                      + responseCode);
+      System.err.println("Response code " + responseCode + " for " + url);
+      return null;
     }
+    // get content
+    InputStream response = connection.getInputStream();
+
     // System.out.println(getClass().getName() + " took "
     // + (System.currentTimeMillis() - now) + "ms to fetch");
 
     checkRateLimits(connection);
-  
+
     BufferedReader reader = null;
     reader = new BufferedReader(new InputStreamReader(response, "UTF-8"));
     return reader;
@@ -284,13 +368,14 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
     // to test:
     // retryDelay = "5";
 
+    EnsemblInfo info = domainData.get(getDomain());
     if (retryDelay != null)
     {
       System.err.println("Ensembl REST service rate limit exceeded, wait "
               + retryDelay + " seconds before retrying");
       try
       {
-        retryAfter = System.currentTimeMillis()
+        info.retryAfter = System.currentTimeMillis()
                 + (1000 * Integer.valueOf(retryDelay));
       } catch (NumberFormatException e)
       {
@@ -300,46 +385,64 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
     }
     else
     {
-      retryAfter = 0;
+      info.retryAfter = 0;
       // debug:
       // System.out.println(String.format(
       // "%s Ensembl requests remaining of %s (reset in %ss)",
       // remaining, limit, reset));
     }
   }
+
   /**
    * Rechecks if Ensembl is responding, unless the last check was successful and
    * the retest interval has not yet elapsed. Returns true if Ensembl is up,
-   * else false.
+   * else false. Also retrieves and saves the current version of Ensembl data
+   * and REST services at intervals.
    * 
    * @return
    */
   protected boolean isEnsemblAvailable()
   {
+    EnsemblInfo info = domainData.get(getDomain());
+
     long now = System.currentTimeMillis();
 
     /*
      * check if we are waiting for 'Retry-After' to expire
      */
-    if (retryAfter > now)
+    if (info.retryAfter > now)
     {
-      System.err.println("Still " + (1 + (retryAfter - now) / 1000)
+      System.err.println("Still " + (1 + (info.retryAfter - now) / 1000)
               + " secs to wait before retrying Ensembl");
       return false;
     }
     else
     {
-      retryAfter = 0;
+      info.retryAfter = 0;
     }
 
-    boolean retest = now - lastCheck > RETEST_INTERVAL;
-    if (ensemblRestAvailable && !retest)
+    /*
+     * recheck if Ensembl is up if it was down, or the recheck period has elapsed
+     */
+    boolean retestAvailability = (now - info.lastAvailableCheckTime) > AVAILABILITY_RETEST_INTERVAL;
+    if (!info.restAvailable || retestAvailability)
     {
-      return true;
+      info.restAvailable = checkEnsembl();
+      info.lastAvailableCheckTime = now;
     }
-    ensemblRestAvailable = checkEnsembl();
-    lastCheck = now;
-    return ensemblRestAvailable;
+
+    /*
+     * refetch Ensembl versions if the recheck period has elapsed
+     */
+    boolean refetchVersion = (now - info.lastVersionCheckTime) > VERSION_RETEST_INTERVAL;
+    if (refetchVersion)
+    {
+      checkEnsemblRestVersion();
+      checkEnsemblDataVersion();
+      info.lastVersionCheckTime = now;
+    }
+
+    return info.restAvailable;
   }
 
   /**
@@ -378,4 +481,105 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
     wr.close();
   }
 
+  /**
+   * Fetches and checks Ensembl's REST version number
+   * 
+   * @return
+   */
+  private void checkEnsemblRestVersion()
+  {
+    EnsemblInfo info = domainData.get(getDomain());
+
+    JSONParser jp = new JSONParser();
+    URL url = null;
+    try
+    {
+      url = new URL(getDomain()
+              + "/info/rest?content-type=application/json");
+      BufferedReader br = getHttpResponse(url, null);
+      JSONObject val = (JSONObject) jp.parse(br);
+      String version = val.get("release").toString();
+      String majorVersion = version.substring(0, version.indexOf("."));
+      String expected = info.expectedRestVersion;
+      String expectedMajorVersion = expected.substring(0,
+              expected.indexOf("."));
+      info.restMajorVersionMismatch = false;
+      try
+      {
+        /*
+         * if actual REST major version is ahead of what we expect,
+         * record this in case we want to warn the user
+         */
+        if (Float.valueOf(majorVersion) > Float
+                .valueOf(expectedMajorVersion))
+        {
+          info.restMajorVersionMismatch = true;
+        }
+      } catch (NumberFormatException e)
+      {
+        System.err.println("Error in REST version: " + e.toString());
+      }
+
+      /*
+       * check if REST version is later than what Jalview has tested against,
+       * if so warn; we don't worry if it is earlier (this indicates Jalview has
+       * been tested in advance against the next pending REST version)
+       */
+      boolean laterVersion = StringUtils.compareVersions(version, expected) == 1;
+      if (laterVersion)
+      {
+        System.err
+                .println(String
+                        .format("EnsemblRestClient expected %s REST version %s but found %s, see %s",
+                                getDbSource(), expected, version,
+                                REST_CHANGE_LOG));
+      }
+      info.restVersion = version;
+    } catch (Throwable t)
+    {
+      System.err.println("Error checking Ensembl REST version: "
+              + t.getMessage());
+    }
+  }
+
+  public boolean isRestMajorVersionMismatch()
+  {
+    return domainData.get(getDomain()).restMajorVersionMismatch;
+  }
+
+  /**
+   * Fetches and checks Ensembl's data version number
+   * 
+   * @return
+   */
+  private void checkEnsemblDataVersion()
+  {
+    JSONParser jp = new JSONParser();
+    URL url = null;
+    try
+    {
+      url = new URL(getDomain()
+              + "/info/data?content-type=application/json");
+      BufferedReader br = getHttpResponse(url, null);
+      JSONObject val = (JSONObject) jp.parse(br);
+      JSONArray versions = (JSONArray) val.get("releases");
+      domainData.get(getDomain()).dataVersion = versions.get(0).toString();
+    } catch (Throwable t)
+    {
+      System.err.println("Error checking Ensembl data version: "
+              + t.getMessage());
+    }
+  }
+
+  public String getEnsemblDataVersion()
+  {
+    return domainData.get(getDomain()).dataVersion;
+  }
+
+  @Override
+  public String getDbVersion()
+  {
+    return getEnsemblDataVersion();
+  }
+
 }