1 package jalview.ext.ensembl;
3 import jalview.io.FileParse;
4 import jalview.util.StringUtils;
6 import java.io.BufferedReader;
7 import java.io.DataOutputStream;
8 import java.io.IOException;
9 import java.io.InputStream;
10 import java.io.InputStreamReader;
11 import java.net.HttpURLConnection;
12 import java.net.MalformedURLException;
14 import java.util.HashMap;
15 import java.util.List;
18 import javax.ws.rs.HttpMethod;
20 import org.json.simple.JSONArray;
21 import org.json.simple.JSONObject;
22 import org.json.simple.parser.JSONParser;
24 import com.stevesoft.pat.Regex;
27 * Base class for Ensembl REST service clients
31 abstract class EnsemblRestClient extends EnsemblSequenceFetcher
34 * update these constants when Jalview has been checked / updated for
35 * changes to Ensembl REST API
36 * @see https://github.com/Ensembl/ensembl-rest/wiki/Change-log
38 private static final String LATEST_ENSEMBLGENOMES_REST_VERSION = "4.4";
40 private static final String LATEST_ENSEMBL_REST_VERSION = "4.5";
42 private static Map<String, EnsemblInfo> domainData;
44 // @see https://github.com/Ensembl/ensembl-rest/wiki/Output-formats
45 private static final String PING_URL = "http://rest.ensembl.org/info/ping.json";
47 private final static long AVAILABILITY_RETEST_INTERVAL = 10000L; // 10 seconds
49 private final static long VERSION_RETEST_INTERVAL = 1000L * 3600; // 1 hr
51 private static final Regex TRANSCRIPT_REGEX = new Regex(
52 "(ENS)([A-Z]{3}|)T[0-9]{11}$");
54 private static final Regex GENE_REGEX = new Regex(
55 "(ENS)([A-Z]{3}|)G[0-9]{11}$");
59 domainData = new HashMap<String, EnsemblInfo>();
60 domainData.put(ENSEMBL_REST, new EnsemblInfo(ENSEMBL_REST,
61 LATEST_ENSEMBL_REST_VERSION));
62 domainData.put(ENSEMBL_GENOMES_REST, new EnsemblInfo(
63 ENSEMBL_GENOMES_REST, LATEST_ENSEMBLGENOMES_REST_VERSION));
66 protected volatile boolean inProgress = false;
69 * Default constructor to use rest.ensembl.org
71 public EnsemblRestClient()
77 * Constructor given the target domain to fetch data from
81 public EnsemblRestClient(String d)
87 * Answers true if the query matches the regular expression pattern for an
88 * Ensembl transcript stable identifier
93 public boolean isTranscriptIdentifier(String query)
95 return query == null ? false : TRANSCRIPT_REGEX.search(query);
99 * Answers true if the query matches the regular expression pattern for an
100 * Ensembl gene stable identifier
105 public boolean isGeneIdentifier(String query)
107 return query == null ? false : GENE_REGEX.search(query);
111 public boolean queryInProgress()
117 public StringBuffer getRawRecords()
123 * Returns the URL for the client http request
127 * @throws MalformedURLException
129 protected abstract URL getUrl(List<String> ids)
130 throws MalformedURLException;
133 * Returns true if client uses GET method, false if it uses POST
137 protected abstract boolean useGetRequest();
140 * Return the desired value for the Content-Type request header
145 * @see https://github.com/Ensembl/ensembl-rest/wiki/HTTP-Headers
147 protected abstract String getRequestMimeType(boolean multipleIds);
150 * Return the desired value for the Accept request header
153 * @see https://github.com/Ensembl/ensembl-rest/wiki/HTTP-Headers
155 protected abstract String getResponseMimeType();
158 * Checks Ensembl's REST 'ping' endpoint, and returns true if response
159 * indicates available, else false
161 * @see http://rest.ensembl.org/documentation/info/ping
164 private boolean checkEnsembl()
166 HttpURLConnection conn = null;
169 // note this format works for both ensembl and ensemblgenomes
170 // info/ping.json works for ensembl only (March 2016)
171 URL ping = new URL(getDomain()
172 + "/info/ping?content-type=application/json");
175 * expect {"ping":1} if ok
177 BufferedReader br = getHttpResponse(ping, null);
178 JSONParser jp = new JSONParser();
179 JSONObject val = (JSONObject) jp.parse(br);
180 String pingString = val.get("ping").toString();
181 return pingString != null;
182 } catch (Throwable t)
184 System.err.println("Error connecting to " + PING_URL + ": "
197 * returns a reader to a Fasta response from the Ensembl sequence endpoint
201 * @throws IOException
203 protected FileParse getSequenceReader(List<String> ids)
206 URL url = getUrl(ids);
208 BufferedReader reader = getHttpResponse(url, ids);
209 FileParse fp = new FileParse(reader, url.toString(), "HTTP_POST");
214 * Writes the HTTP request and gets the response as a reader.
218 * written as Json POST body if more than one
220 * @throws IOException
221 * if response code was not 200, or other I/O error
223 protected BufferedReader getHttpResponse(URL url, List<String> ids)
226 // long now = System.currentTimeMillis();
227 HttpURLConnection connection = (HttpURLConnection) url.openConnection();
230 * POST method allows multiple queries in one request; it is supported for
231 * sequence queries, but not for overlap
233 boolean multipleIds = ids != null && ids.size() > 1;
234 connection.setRequestMethod(multipleIds ? HttpMethod.POST
236 connection.setRequestProperty("Content-Type",
237 getRequestMimeType(multipleIds));
238 connection.setRequestProperty("Accept", getResponseMimeType());
240 connection.setUseCaches(false);
241 connection.setDoInput(true);
242 connection.setDoOutput(multipleIds);
246 writePostBody(connection, ids);
249 InputStream response = connection.getInputStream();
250 int responseCode = connection.getResponseCode();
252 if (responseCode != 200)
255 * note: a GET request for an invalid id returns an error code e.g. 415
256 * but POST request returns 200 and an empty Fasta response
258 throw new IOException(
259 "Response code was not 200. Detected response was "
262 // System.out.println(getClass().getName() + " took "
263 // + (System.currentTimeMillis() - now) + "ms to fetch");
265 checkRateLimits(connection);
267 BufferedReader reader = null;
268 reader = new BufferedReader(new InputStreamReader(response, "UTF-8"));
273 * Inspect response headers for any sign of server overload and respect any
274 * 'retry-after' directive
276 * @see https://github.com/Ensembl/ensembl-rest/wiki/Rate-Limits
279 void checkRateLimits(HttpURLConnection connection)
281 // number of requests allowed per time interval:
282 String limit = connection.getHeaderField("X-RateLimit-Limit");
283 // length of quota time interval in seconds:
284 // String period = connection.getHeaderField("X-RateLimit-Period");
285 // seconds remaining until usage quota is reset:
286 String reset = connection.getHeaderField("X-RateLimit-Reset");
287 // number of requests remaining from quota for current period:
288 String remaining = connection.getHeaderField("X-RateLimit-Remaining");
289 // number of seconds to wait before retrying (if remaining == 0)
290 String retryDelay = connection.getHeaderField("Retry-After");
295 EnsemblInfo info = domainData.get(getDomain());
296 if (retryDelay != null)
298 System.err.println("Ensembl REST service rate limit exceeded, wait "
299 + retryDelay + " seconds before retrying");
302 info.retryAfter = System.currentTimeMillis()
303 + (1000 * Integer.valueOf(retryDelay));
304 } catch (NumberFormatException e)
306 System.err.println("Unexpected value for Retry-After: "
314 // System.out.println(String.format(
315 // "%s Ensembl requests remaining of %s (reset in %ss)",
316 // remaining, limit, reset));
321 * Rechecks if Ensembl is responding, unless the last check was successful and
322 * the retest interval has not yet elapsed. Returns true if Ensembl is up,
323 * else false. Also retrieves and saves the current version of Ensembl data
324 * and REST services at intervals.
328 protected boolean isEnsemblAvailable()
330 EnsemblInfo info = domainData.get(getDomain());
332 long now = System.currentTimeMillis();
335 * check if we are waiting for 'Retry-After' to expire
337 if (info.retryAfter > now)
339 System.err.println("Still " + (1 + (info.retryAfter - now) / 1000)
340 + " secs to wait before retrying Ensembl");
349 * recheck if Ensembl is up if it was down, or the recheck period has elapsed
351 boolean retestAvailability = (now - info.lastAvailableCheckTime) > AVAILABILITY_RETEST_INTERVAL;
352 if (!info.restAvailable || retestAvailability)
354 info.restAvailable = checkEnsembl();
355 info.lastAvailableCheckTime = now;
359 * refetch Ensembl versions if the recheck period has elapsed
361 boolean refetchVersion = (now - info.lastVersionCheckTime) > VERSION_RETEST_INTERVAL;
364 checkEnsemblRestVersion();
365 checkEnsemblDataVersion();
366 info.lastVersionCheckTime = now;
369 return info.restAvailable;
373 * Constructs, writes and flushes the POST body of the request, containing the
374 * query ids in JSON format
378 * @throws IOException
380 protected void writePostBody(HttpURLConnection connection,
381 List<String> ids) throws IOException
384 StringBuilder postBody = new StringBuilder(64);
385 postBody.append("{\"ids\":[");
387 for (String id : ids)
391 postBody.append(",");
394 postBody.append("\"");
395 postBody.append(id.trim());
396 postBody.append("\"");
398 postBody.append("]}");
399 byte[] thepostbody = postBody.toString().getBytes();
400 connection.setRequestProperty("Content-Length",
401 Integer.toString(thepostbody.length));
402 DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
403 wr.write(thepostbody);
409 * Fetches and checks Ensembl's REST version number
413 private void checkEnsemblRestVersion()
415 EnsemblInfo info = domainData.get(getDomain());
417 JSONParser jp = new JSONParser();
421 url = new URL(getDomain()
422 + "/info/rest?content-type=application/json");
423 BufferedReader br = getHttpResponse(url, null);
424 JSONObject val = (JSONObject) jp.parse(br);
425 String version = val.get("release").toString();
426 String majorVersion = version.substring(0, version.indexOf("."));
427 String expected = info.expectedRestVersion;
428 String expectedMajorVersion = expected.substring(0,
429 expected.indexOf("."));
430 info.restMajorVersionMismatch = false;
434 * if actual REST major version is ahead of what we expect,
435 * record this in case we want to warn the user
437 if (Float.valueOf(majorVersion) > Float
438 .valueOf(expectedMajorVersion))
440 info.restMajorVersionMismatch = true;
442 } catch (NumberFormatException e)
444 System.err.println("Error in REST version: " + e.toString());
448 * check if REST version is later than what Jalview has tested against,
449 * if so warn; we don't worry if it is earlier (this indicates Jalview has
450 * been tested in advance against the next pending REST version)
452 boolean laterVersion = StringUtils.compareVersions(version, expected) == 1;
455 System.err.println(String.format(
456 "Expected %s REST version %s but found %s", getDbSource(),
460 info.restVersion = version;
461 } catch (Throwable t)
463 System.err.println("Error checking Ensembl REST version: "
468 public boolean isRestMajorVersionMismatch()
470 return domainData.get(getDomain()).restMajorVersionMismatch;
474 * Fetches and checks Ensembl's data version number
478 private void checkEnsemblDataVersion()
480 JSONParser jp = new JSONParser();
484 url = new URL(getDomain()
485 + "/info/data?content-type=application/json");
486 BufferedReader br = getHttpResponse(url, null);
487 JSONObject val = (JSONObject) jp.parse(br);
488 JSONArray versions = (JSONArray) val.get("releases");
489 domainData.get(getDomain()).dataVersion = versions.get(0).toString();
490 } catch (Throwable t)
492 System.err.println("Error checking Ensembl data version: "
497 public String getEnsemblDataVersion()
499 return domainData.get(getDomain()).dataVersion;
503 public String getDbVersion()
505 return getEnsemblDataVersion();