X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fext%2Fensembl%2FEnsemblRestClient.java;fp=src%2Fjalview%2Fext%2Fensembl%2FEnsemblRestClient.java;h=c06d13e09c31c3ba694d8007974ed4bf5fc1dbb0;hb=2e6299399bd798ad2959a2b80fd3ac2d2a1374df;hp=81bc560dfb15da43c32fa56ca3071ad897ee0eaf;hpb=b691a827f39b1d02cd01ee2f2e0b2077c01818bd;p=jalview.git

diff --git a/src/jalview/ext/ensembl/EnsemblRestClient.java b/src/jalview/ext/ensembl/EnsemblRestClient.java
index 81bc560..c06d13e 100644
--- a/src/jalview/ext/ensembl/EnsemblRestClient.java
+++ b/src/jalview/ext/ensembl/EnsemblRestClient.java
@@ -56,6 +56,12 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
 
   private static final int CONNECT_TIMEOUT_MS = 10 * 1000; // 10 seconds
 
+  private static final int MAX_RETRIES = 3;
+
+  private static final int HTTP_OK = 200;
+
+  private static final int HTTP_OVERLOAD = 429;
+
   /*
    * update these constants when Jalview has been checked / updated for
    * changes to Ensembl REST API (ref JAL-2105)
@@ -205,7 +211,7 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
    * @see http://rest.ensembl.org/documentation/info/ping
    * @return
    */
-  private boolean checkEnsembl()
+  boolean checkEnsembl()
   {
     BufferedReader br = null;
     try
@@ -220,6 +226,10 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
        * if ping takes more than 2 seconds to respond, treat as if unavailable
        */
       br = getHttpResponse(ping, null, 2 * 1000);
+      if (br == null)
+      {
+        return false;
+      }
       JSONParser jp = new JSONParser();
       JSONObject val = (JSONObject) jp.parse(br);
       String pingString = val.get("ping").toString();
@@ -282,7 +292,7 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   }
 
   /**
-   * Writes the HTTP request and gets the response as a reader.
+   * Sends the HTTP request and gets the response as a reader
    * 
    * @param url
    * @param ids
@@ -296,18 +306,25 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   protected BufferedReader getHttpResponse(URL url, List<String> ids,
           int readTimeout) throws IOException
   {
-    // long now = System.currentTimeMillis();
-    int maxRetries = 3;
-    int retriesLeft = maxRetries;
+    int retriesLeft = MAX_RETRIES;
     HttpURLConnection connection = null;
+    int responseCode = 0;
+
     while (retriesLeft > 0)
     {
       connection = tryConnection(url, ids, readTimeout);
+      responseCode = connection.getResponseCode();
+      if (responseCode == HTTP_OVERLOAD) // 429
+      {
+        retriesLeft--;
+        checkRetryAfter(connection);
+      }
+      else
+      {
+        retriesLeft = 0;
+      }
     }
-
-    int responseCode = connection.getResponseCode();
-
-    if (responseCode != 200)
+    if (responseCode != HTTP_OK) // 200
     {
       /*
        * note: a GET request for an invalid id returns an error code e.g. 415
@@ -316,14 +333,12 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
       System.err.println("Response code " + responseCode + " for " + url);
       return null;
     }
-    // get content
+
     InputStream response = connection.getInputStream();
 
     // System.out.println(getClass().getName() + " took "
     // + (System.currentTimeMillis() - now) + "ms to fetch");
 
-    checkRateLimits(connection);
-
     BufferedReader reader = null;
     reader = new BufferedReader(new InputStreamReader(response, "UTF-8"));
     return reader;
@@ -340,6 +355,7 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   protected HttpURLConnection tryConnection(URL url, List<String> ids,
           int readTimeout) throws IOException, ProtocolException
   {
+    // System.out.println(System.currentTimeMillis() + " " + url);
     HttpURLConnection connection = (HttpURLConnection) url.openConnection();
 
     /*
@@ -368,51 +384,36 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   }
 
   /**
-   * Inspect response headers for any sign of server overload and respect any
-   * 'retry-after' directive
+   * Inspects response headers for a 'retry-after' directive, and waits for the
+   * directed period (if less than 10 seconds)
    * 
    * @see https://github.com/Ensembl/ensembl-rest/wiki/Rate-Limits
    * @param connection
    */
-  void checkRateLimits(HttpURLConnection connection)
+  void checkRetryAfter(HttpURLConnection connection)
   {
-    // number of requests allowed per time interval:
-    String limit = connection.getHeaderField("X-RateLimit-Limit");
-    // length of quota time interval in seconds:
-    // String period = connection.getHeaderField("X-RateLimit-Period");
-    // seconds remaining until usage quota is reset:
-    String reset = connection.getHeaderField("X-RateLimit-Reset");
-    // number of requests remaining from quota for current period:
-    String remaining = connection.getHeaderField("X-RateLimit-Remaining");
-    // number of seconds to wait before retrying (if remaining == 0)
     String retryDelay = connection.getHeaderField("Retry-After");
 
     // to test:
     // retryDelay = "5";
 
-    EnsemblInfo info = domainData.get(getDomain());
     if (retryDelay != null)
     {
-      System.err.println("Ensembl REST service rate limit exceeded, wait "
-              + retryDelay + " seconds before retrying");
       try
       {
-        info.retryAfter = System.currentTimeMillis()
-                + (1000 * Integer.valueOf(retryDelay));
-      } catch (NumberFormatException e)
+        int retrySecs = Integer.valueOf(retryDelay);
+        if (retrySecs > 0 && retrySecs < 10)
+        {
+          System.err
+                  .println("Ensembl REST service rate limit exceeded, waiting "
+                          + retryDelay + " seconds before retrying");
+          Thread.sleep(1000 * retrySecs);
+        }
+      } catch (NumberFormatException | InterruptedException e)
       {
-        System.err
-                .println("Unexpected value for Retry-After: " + retryDelay);
+        System.err.println("Error handling Retry-After: " + e.getMessage());
       }
     }
-    else
-    {
-      info.retryAfter = 0;
-      // debug:
-      // System.out.println(String.format(
-      // "%s Ensembl requests remaining of %s (reset in %ss)",
-      // remaining, limit, reset));
-    }
   }
 
   /**
@@ -430,20 +431,6 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
     long now = System.currentTimeMillis();
 
     /*
-     * check if we are waiting for 'Retry-After' to expire
-     */
-    if (info.retryAfter > now)
-    {
-      System.err.println("Still " + (1 + (info.retryAfter - now) / 1000)
-              + " secs to wait before retrying Ensembl");
-      return false;
-    }
-    else
-    {
-      info.retryAfter = 0;
-    }
-
-    /*
      * recheck if Ensembl is up if it was down, or the recheck period has elapsed
      */
     boolean retestAvailability = (now
@@ -580,18 +567,36 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher
   {
     JSONParser jp = new JSONParser();
     URL url = null;
+    BufferedReader br = null;
+
     try
     {
       url = new URL(
               getDomain() + "/info/data?content-type=application/json");
-      BufferedReader br = getHttpResponse(url, null);
-      JSONObject val = (JSONObject) jp.parse(br);
-      JSONArray versions = (JSONArray) val.get("releases");
-      domainData.get(getDomain()).dataVersion = versions.get(0).toString();
+      br = getHttpResponse(url, null);
+      if (br != null)
+      {
+        JSONObject val = (JSONObject) jp.parse(br);
+        JSONArray versions = (JSONArray) val.get("releases");
+        domainData.get(getDomain()).dataVersion = versions.get(0)
+                .toString();
+      }
     } catch (Throwable t)
     {
       System.err.println(
               "Error checking Ensembl data version: " + t.getMessage());
+    } finally
+    {
+      if (br != null)
+      {
+        try
+        {
+          br.close();
+        } catch (IOException e)
+        {
+          // ignore
+        }
+      }
     }
   }