X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;ds=sidebyside;f=src%2Fjalview%2Fext%2Fensembl%2FEnsemblRestClient.java;h=f9cfe052a2f7487113cb48fe68a4e5f017ffbc94;hb=08c202dd3ef0f34bde305c11da9eec03520174dc;hp=297f71b725267e194970d64e297c6f129d3ed34f;hpb=e24933a537e0f640c75d4685c468615872bc77fc;p=jalview.git diff --git a/src/jalview/ext/ensembl/EnsemblRestClient.java b/src/jalview/ext/ensembl/EnsemblRestClient.java index 297f71b..f9cfe05 100644 --- a/src/jalview/ext/ensembl/EnsemblRestClient.java +++ b/src/jalview/ext/ensembl/EnsemblRestClient.java @@ -1,6 +1,7 @@ package jalview.ext.ensembl; import jalview.io.FileParse; +import jalview.util.StringUtils; import java.io.BufferedReader; import java.io.DataOutputStream; @@ -10,10 +11,16 @@ import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.ws.rs.HttpMethod; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + import com.stevesoft.pat.Regex; /** @@ -23,15 +30,23 @@ import com.stevesoft.pat.Regex; */ abstract class EnsemblRestClient extends EnsemblSequenceFetcher { - protected final static String ENSEMBL_REST = "http://rest.ensembl.org"; + /* + * update these constants when Jalview has been checked / updated for + * changes to Ensembl REST API + * @see https://github.com/Ensembl/ensembl-rest/wiki/Change-log + */ + private static final String LATEST_ENSEMBLGENOMES_REST_VERSION = "4.4"; - protected static final String SEQUENCE_ID_URL = ENSEMBL_REST - + "/sequence/id"; + private static final String LATEST_ENSEMBL_REST_VERSION = "4.5"; + + private static Map domainData; // @see https://github.com/Ensembl/ensembl-rest/wiki/Output-formats private static final String PING_URL = "http://rest.ensembl.org/info/ping.json"; - private final static long RETEST_INTERVAL = 10000L; // 10 seconds + private final static long AVAILABILITY_RETEST_INTERVAL = 10000L; // 10 seconds + + private final static long VERSION_RETEST_INTERVAL = 1000L * 3600; // 1 hr private static final Regex TRANSCRIPT_REGEX = new Regex( "(ENS)([A-Z]{3}|)T[0-9]{11}$"); @@ -39,18 +54,55 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher private static final Regex GENE_REGEX = new Regex( "(ENS)([A-Z]{3}|)G[0-9]{11}$"); - private static boolean ensemblRestAvailable = false; - - private static long lastCheck = -1; + static + { + domainData = new HashMap(); + domainData.put(ENSEMBL_REST, new EnsemblInfo(ENSEMBL_REST, + LATEST_ENSEMBL_REST_VERSION)); + domainData.put(ENSEMBL_GENOMES_REST, new EnsemblInfo( + ENSEMBL_GENOMES_REST, LATEST_ENSEMBLGENOMES_REST_VERSION)); + } protected volatile boolean inProgress = false; - public static boolean isTranscriptIdentifier(String query) + /** + * Default constructor to use rest.ensembl.org + */ + public EnsemblRestClient() + { + this(ENSEMBL_REST); + } + + /** + * Constructor given the target domain to fetch data from + * + * @param d + */ + public EnsemblRestClient(String d) + { + setDomain(d); + } + + /** + * Answers true if the query matches the regular expression pattern for an + * Ensembl transcript stable identifier + * + * @param query + * @return + */ + public boolean isTranscriptIdentifier(String query) { return query == null ? false : TRANSCRIPT_REGEX.search(query); } - public static boolean isGeneIdentifier(String query) + /** + * Answers true if the query matches the regular expression pattern for an + * Ensembl gene stable identifier + * + * @param query + * @return + */ + public boolean isGeneIdentifier(String query) { return query == null ? false : GENE_REGEX.search(query); } @@ -103,27 +155,40 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher protected abstract String getResponseMimeType(); /** - * Tries to connect to Ensembl's REST 'ping' endpoint, and returns true if - * successful, else false + * Checks Ensembl's REST 'ping' endpoint, and returns true if response + * indicates available, else false * + * @see http://rest.ensembl.org/documentation/info/ping * @return */ private boolean checkEnsembl() { + HttpURLConnection conn = null; try { - URL ping = new URL(PING_URL); - HttpURLConnection conn = (HttpURLConnection) ping.openConnection(); - int rc = conn.getResponseCode(); - conn.disconnect(); - if (rc >= 200 && rc < 300) - { - return true; - } + // note this format works for both ensembl and ensemblgenomes + // info/ping.json works for ensembl only (March 2016) + URL ping = new URL(getDomain() + + "/info/ping?content-type=application/json"); + + /* + * expect {"ping":1} if ok + */ + BufferedReader br = getHttpResponse(ping, null); + JSONParser jp = new JSONParser(); + JSONObject val = (JSONObject) jp.parse(br); + String pingString = val.get("ping").toString(); + return pingString != null; } catch (Throwable t) { System.err.println("Error connecting to " + PING_URL + ": " + t.getMessage()); + } finally + { + if (conn != null) + { + conn.disconnect(); + } } return false; } @@ -165,7 +230,7 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher * POST method allows multiple queries in one request; it is supported for * sequence queries, but not for overlap */ - boolean multipleIds = ids.size() > 1;// useGetRequest(); + boolean multipleIds = ids != null && ids.size() > 1; connection.setRequestMethod(multipleIds ? HttpMethod.POST : HttpMethod.GET); connection.setRequestProperty("Content-Type", @@ -196,6 +261,8 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher } // System.out.println(getClass().getName() + " took " // + (System.currentTimeMillis() - now) + "ms to fetch"); + + checkRateLimits(connection); BufferedReader reader = null; reader = new BufferedReader(new InputStreamReader(response, "UTF-8")); @@ -203,23 +270,103 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher } /** + * Inspect response headers for any sign of server overload and respect any + * 'retry-after' directive + * + * @see https://github.com/Ensembl/ensembl-rest/wiki/Rate-Limits + * @param connection + */ + void checkRateLimits(HttpURLConnection connection) + { + // number of requests allowed per time interval: + String limit = connection.getHeaderField("X-RateLimit-Limit"); + // length of quota time interval in seconds: + // String period = connection.getHeaderField("X-RateLimit-Period"); + // seconds remaining until usage quota is reset: + String reset = connection.getHeaderField("X-RateLimit-Reset"); + // number of requests remaining from quota for current period: + String remaining = connection.getHeaderField("X-RateLimit-Remaining"); + // number of seconds to wait before retrying (if remaining == 0) + String retryDelay = connection.getHeaderField("Retry-After"); + + // to test: + // retryDelay = "5"; + + EnsemblInfo info = domainData.get(getDomain()); + if (retryDelay != null) + { + System.err.println("Ensembl REST service rate limit exceeded, wait " + + retryDelay + " seconds before retrying"); + try + { + info.retryAfter = System.currentTimeMillis() + + (1000 * Integer.valueOf(retryDelay)); + } catch (NumberFormatException e) + { + System.err.println("Unexpected value for Retry-After: " + + retryDelay); + } + } + else + { + info.retryAfter = 0; + // debug: + // System.out.println(String.format( + // "%s Ensembl requests remaining of %s (reset in %ss)", + // remaining, limit, reset)); + } + } + + /** * Rechecks if Ensembl is responding, unless the last check was successful and * the retest interval has not yet elapsed. Returns true if Ensembl is up, - * else false. + * else false. Also retrieves and saves the current version of Ensembl data + * and REST services at intervals. * * @return */ protected boolean isEnsemblAvailable() { + EnsemblInfo info = domainData.get(getDomain()); + long now = System.currentTimeMillis(); - boolean retest = now - lastCheck > RETEST_INTERVAL; - if (ensemblRestAvailable && !retest) + + /* + * check if we are waiting for 'Retry-After' to expire + */ + if (info.retryAfter > now) + { + System.err.println("Still " + (1 + (info.retryAfter - now) / 1000) + + " secs to wait before retrying Ensembl"); + return false; + } + else { - return true; + info.retryAfter = 0; } - ensemblRestAvailable = checkEnsembl(); - lastCheck = now; - return ensemblRestAvailable; + + /* + * recheck if Ensembl is up if it was down, or the recheck period has elapsed + */ + boolean retestAvailability = (now - info.lastAvailableCheckTime) > AVAILABILITY_RETEST_INTERVAL; + if (!info.restAvailable || retestAvailability) + { + info.restAvailable = checkEnsembl(); + info.lastAvailableCheckTime = now; + } + + /* + * refetch Ensembl versions if the recheck period has elapsed + */ + boolean refetchVersion = (now - info.lastVersionCheckTime) > VERSION_RETEST_INTERVAL; + if (refetchVersion) + { + checkEnsemblRestVersion(); + checkEnsemblDataVersion(); + info.lastVersionCheckTime = now; + } + + return info.restAvailable; } /** @@ -258,4 +405,104 @@ abstract class EnsemblRestClient extends EnsemblSequenceFetcher wr.close(); } + /** + * Fetches and checks Ensembl's REST version number + * + * @return + */ + private void checkEnsemblRestVersion() + { + EnsemblInfo info = domainData.get(getDomain()); + + JSONParser jp = new JSONParser(); + URL url = null; + try + { + url = new URL(getDomain() + + "/info/rest?content-type=application/json"); + BufferedReader br = getHttpResponse(url, null); + JSONObject val = (JSONObject) jp.parse(br); + String version = val.get("release").toString(); + String majorVersion = version.substring(0, version.indexOf(".")); + String expected = info.expectedRestVersion; + String expectedMajorVersion = expected.substring(0, + expected.indexOf(".")); + info.restMajorVersionMismatch = false; + try + { + /* + * if actual REST major version is ahead of what we expect, + * record this in case we want to warn the user + */ + if (Float.valueOf(majorVersion) > Float + .valueOf(expectedMajorVersion)) + { + info.restMajorVersionMismatch = true; + } + } catch (NumberFormatException e) + { + System.err.println("Error in REST version: " + e.toString()); + } + + /* + * check if REST version is later than what Jalview has tested against, + * if so warn; we don't worry if it is earlier (this indicates Jalview has + * been tested in advance against the next pending REST version) + */ + boolean laterVersion = StringUtils.compareVersions(version, expected) == 1; + if (laterVersion) + { + System.err.println(String.format( + "Expected %s REST version %s but found %s", getDbSource(), + expected, + version)); + } + info.restVersion = version; + } catch (Throwable t) + { + System.err.println("Error checking Ensembl REST version: " + + t.getMessage()); + } + } + + public boolean isRestMajorVersionMismatch() + { + return domainData.get(getDomain()).restMajorVersionMismatch; + } + + /** + * Fetches and checks Ensembl's data version number + * + * @return + */ + private void checkEnsemblDataVersion() + { + JSONParser jp = new JSONParser(); + URL url = null; + try + { + url = new URL(getDomain() + + "/info/data?content-type=application/json"); + BufferedReader br = getHttpResponse(url, null); + JSONObject val = (JSONObject) jp.parse(br); + JSONArray versions = (JSONArray) val.get("releases"); + domainData.get(getDomain()).dataVersion = versions.get(0).toString(); + } catch (Throwable t) + { + System.err.println("Error checking Ensembl data version: " + + t.getMessage()); + } + } + + public String getEnsemblDataVersion() + { + return domainData.get(getDomain()).dataVersion; + } + + @Override + public String getDbVersion() + { + return getEnsemblDataVersion(); + } + }