*/
public static final String ENSEMBL = "ENSEMBL";
+ public static final String ENSEMBLGENOMES = "ENSEMBLGENOMES";
+
/**
* List of databases whose sequences might have coding regions annotated
*/
--- /dev/null
+package jalview.ext.ensembl;
+
+/**
+ * A data class to model the data and rest version of one Ensembl domain,
+ * currently for rest.ensembl.org and rest.ensemblgenomes.org
+ *
+ * @author gmcarstairs
+ */
+class EnsemblInfo
+{
+ /*
+ * The http domain this object is holding data values for
+ */
+ String domain;
+
+ /*
+ * The latest version Jalview has tested for, e.g. "4.5"; a minor version change should be
+ * ok, a major version change may break stuff
+ */
+ String expectedRestVersion;
+
+ /*
+ * Major / minor / point version e.g. "4.5.1"
+ * @see http://rest.ensembl.org/info/rest/?content-type=application/json
+ */
+ String restVersion;
+
+ /*
+ * data version
+ * @see http://rest.ensembl.org/info/data/?content-type=application/json
+ */
+ String dataVersion;
+
+ /*
+ * true when http://rest.ensembl.org/info/ping/?content-type=application/json
+ * returns response code 200
+ */
+ boolean restAvailable;
+
+ /*
+ * absolute time when availability was last checked
+ */
+ long lastAvailableCheckTime;
+
+ /*
+ * absolute time when version numbers were last checked
+ */
+ long lastVersionCheckTime;
+
+ // flag set to true if REST major version is not the one expected
+ boolean restMajorVersionMismatch;
+
+ /*
+ * absolute time to wait till if we overloaded the REST service
+ */
+ long retryAfter;
+
+ /**
+ * Constructor given expected REST version number e.g 4.5 or 3.4.3
+ *
+ * @param restExpected
+ */
+ EnsemblInfo(String theDomain, String restExpected)
+ {
+ domain = theDomain;
+ expectedRestVersion = restExpected;
+ lastAvailableCheckTime = -1;
+ lastVersionCheckTime = -1;
+ }
+
+}
package jalview.ext.ensembl;
import jalview.io.FileParse;
+import jalview.util.StringUtils;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import javax.ws.rs.HttpMethod;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
import com.stevesoft.pat.Regex;
/**
*/
abstract class EnsemblRestClient extends EnsemblSequenceFetcher
{
- private final static String ENSEMBL_REST = "http://rest.ensembl.org";
+ /*
+ * update these constants when Jalview has been checked / updated for
+ * changes to Ensembl REST API
+ * @see https://github.com/Ensembl/ensembl-rest/wiki/Change-log
+ */
+ private static final String LATEST_ENSEMBLGENOMES_REST_VERSION = "4.4";
- protected final static String ENSEMBL_GENOMES_REST = "http://rest.ensemblgenomes.org";
+ private static final String LATEST_ENSEMBL_REST_VERSION = "4.5";
+
+ private static Map<String, EnsemblInfo> domainData;
// @see https://github.com/Ensembl/ensembl-rest/wiki/Output-formats
private static final String PING_URL = "http://rest.ensembl.org/info/ping.json";
- private final static long RETEST_INTERVAL = 10000L; // 10 seconds
+ private final static long AVAILABILITY_RETEST_INTERVAL = 10000L; // 10 seconds
+
+ private final static long VERSION_RETEST_INTERVAL = 1000L * 3600; // 1 hr
private static final Regex TRANSCRIPT_REGEX = new Regex(
"(ENS)([A-Z]{3}|)T[0-9]{11}$");
private static final Regex GENE_REGEX = new Regex(
"(ENS)([A-Z]{3}|)G[0-9]{11}$");
- private String domain = ENSEMBL_REST;
-
- private static boolean ensemblRestAvailable = false;
-
- private static long lastCheck = -1;
-
- /*
- * absolute time to wait till if we overloaded the REST service
- */
- private static long retryAfter;
+ static
+ {
+ domainData = new HashMap<String, EnsemblInfo>();
+ domainData.put(ENSEMBL_REST, new EnsemblInfo(ENSEMBL_REST,
+ LATEST_ENSEMBL_REST_VERSION));
+ domainData.put(ENSEMBL_GENOMES_REST, new EnsemblInfo(
+ ENSEMBL_GENOMES_REST, LATEST_ENSEMBLGENOMES_REST_VERSION));
+ }
protected volatile boolean inProgress = false;
*/
public EnsemblRestClient(String d)
{
- domain = d;
- }
-
- /**
- * Returns the domain name to query e.g. http://rest.ensembl.org or
- * http://rest.ensemblgenomes.org
- *
- * @return
- */
- String getDomain()
- {
- return domain;
- }
-
- void setDomain(String d)
- {
- domain = d;
+ setDomain(d);
}
/**
* POST method allows multiple queries in one request; it is supported for
* sequence queries, but not for overlap
*/
- boolean multipleIds = ids.size() > 1;// useGetRequest();
+ boolean multipleIds = ids != null && ids.size() > 1;
connection.setRequestMethod(multipleIds ? HttpMethod.POST
: HttpMethod.GET);
connection.setRequestProperty("Content-Type",
// to test:
// retryDelay = "5";
+ EnsemblInfo info = domainData.get(getDomain());
if (retryDelay != null)
{
System.err.println("Ensembl REST service rate limit exceeded, wait "
+ retryDelay + " seconds before retrying");
try
{
- retryAfter = System.currentTimeMillis()
+ info.retryAfter = System.currentTimeMillis()
+ (1000 * Integer.valueOf(retryDelay));
} catch (NumberFormatException e)
{
}
else
{
- retryAfter = 0;
+ info.retryAfter = 0;
// debug:
// System.out.println(String.format(
// "%s Ensembl requests remaining of %s (reset in %ss)",
// remaining, limit, reset));
}
}
+
/**
* Rechecks if Ensembl is responding, unless the last check was successful and
* the retest interval has not yet elapsed. Returns true if Ensembl is up,
- * else false.
+ * else false. Also retrieves and saves the current version of Ensembl data
+ * and REST services at intervals.
*
* @return
*/
protected boolean isEnsemblAvailable()
{
+ EnsemblInfo info = domainData.get(getDomain());
+
long now = System.currentTimeMillis();
/*
* check if we are waiting for 'Retry-After' to expire
*/
- if (retryAfter > now)
+ if (info.retryAfter > now)
{
- System.err.println("Still " + (1 + (retryAfter - now) / 1000)
+ System.err.println("Still " + (1 + (info.retryAfter - now) / 1000)
+ " secs to wait before retrying Ensembl");
return false;
}
else
{
- retryAfter = 0;
+ info.retryAfter = 0;
+ }
+
+ /*
+ * recheck if Ensembl is up if it was down, or the recheck period has elapsed
+ */
+ boolean retestAvailability = (now - info.lastAvailableCheckTime) > AVAILABILITY_RETEST_INTERVAL;
+ if (!info.restAvailable || retestAvailability)
+ {
+ info.restAvailable = checkEnsembl();
+ info.lastAvailableCheckTime = now;
}
- boolean retest = now - lastCheck > RETEST_INTERVAL;
- if (ensemblRestAvailable && !retest)
+ /*
+ * refetch Ensembl versions if the recheck period has elapsed
+ */
+ boolean refetchVersion = (now - info.lastVersionCheckTime) > VERSION_RETEST_INTERVAL;
+ if (refetchVersion)
{
- return true;
+ checkEnsemblRestVersion();
+ checkEnsemblDataVersion();
+ info.lastVersionCheckTime = now;
}
- ensemblRestAvailable = checkEnsembl();
- lastCheck = now;
- return ensemblRestAvailable;
+
+ return info.restAvailable;
}
/**
wr.close();
}
+ /**
+ * Fetches and checks Ensembl's REST version number
+ *
+ * @return
+ */
+ private void checkEnsemblRestVersion()
+ {
+ EnsemblInfo info = domainData.get(getDomain());
+
+ JSONParser jp = new JSONParser();
+ URL url = null;
+ try
+ {
+ url = new URL(getDomain()
+ + "/info/rest?content-type=application/json");
+ BufferedReader br = getHttpResponse(url, null);
+ JSONObject val = (JSONObject) jp.parse(br);
+ String version = val.get("release").toString();
+ String majorVersion = version.substring(0, version.indexOf("."));
+ String expected = info.expectedRestVersion;
+ String expectedMajorVersion = expected.substring(0,
+ expected.indexOf("."));
+ info.restMajorVersionMismatch = false;
+ try
+ {
+ /*
+ * if actual REST major version is ahead of what we expect,
+ * record this in case we want to warn the user
+ */
+ if (Float.valueOf(majorVersion) > Float
+ .valueOf(expectedMajorVersion))
+ {
+ info.restMajorVersionMismatch = true;
+ }
+ } catch (NumberFormatException e)
+ {
+ System.err.println("Error in REST version: " + e.toString());
+ }
+
+ /*
+ * check if REST version is later than what Jalview has tested against,
+ * if so warn; we don't worry if it is earlier (this indicates Jalview has
+ * been tested in advance against the next pending REST version)
+ */
+ boolean laterVersion = StringUtils.compareVersions(version, expected) == 1;
+ if (laterVersion)
+ {
+ System.err.println(String.format(
+ "Expected %s REST version %s but found %s", getDbSource(),
+ expected,
+ version));
+ }
+ info.restVersion = version;
+ } catch (Throwable t)
+ {
+ System.err.println("Error checking Ensembl REST version: "
+ + t.getMessage());
+ }
+ }
+
+ public boolean isRestMajorVersionMismatch()
+ {
+ return domainData.get(getDomain()).restMajorVersionMismatch;
+ }
+
+ /**
+ * Fetches and checks Ensembl's data version number
+ *
+ * @return
+ */
+ private void checkEnsemblDataVersion()
+ {
+ JSONParser jp = new JSONParser();
+ URL url = null;
+ try
+ {
+ url = new URL(getDomain()
+ + "/info/data?content-type=application/json");
+ BufferedReader br = getHttpResponse(url, null);
+ JSONObject val = (JSONObject) jp.parse(br);
+ JSONArray versions = (JSONArray) val.get("releases");
+ domainData.get(getDomain()).dataVersion = versions.get(0).toString();
+ } catch (Throwable t)
+ {
+ System.err.println("Error checking Ensembl data version: "
+ + t.getMessage());
+ }
+ }
+
+ public String getEnsemblDataVersion()
+ {
+ return domainData.get(getDomain()).dataVersion;
+ }
+
+ @Override
+ public String getDbVersion()
+ {
+ return getEnsemblDataVersion();
+ }
+
}
ds.setSourceDBRef(proteinSeq.getSourceDBRef());
Mapping map = new Mapping(ds, mapList);
- DBRefEntry dbr = new DBRefEntry(getDbSource(), getDbVersion(),
- proteinSeq.getName(), map);
+ DBRefEntry dbr = new DBRefEntry(getDbSource(),
+ getEnsemblDataVersion(), proteinSeq.getName(), map);
querySeq.getDatasetSequence().addDBRef(dbr);
/*
/*
* and add a reference to itself
*/
- DBRefEntry self = new DBRefEntry(getDbSource(), "0", seq.getName());
+ DBRefEntry self = new DBRefEntry(getDbSource(),
+ getEnsemblDataVersion(), seq.getName());
seq.addDBRef(self);
}
if (ids.contains(name)
|| ids.contains(name.replace("ENSP", "ENST")))
{
- DBRefUtils.parseToDbRef(sq, DBRefSource.ENSEMBL, "0", name);
+ DBRefUtils.parseToDbRef(sq, getDbSource(),
+ getEnsemblDataVersion(), name);
}
}
if (alignment == null)
private static final Regex ACCESSION_REGEX = new Regex(
"(ENS([A-Z]{3}|)[GTEP]{1}[0-9]{11}$)" + "|" + "(CCDS[0-9.]{3,}$)");
+ protected static final String ENSEMBL_GENOMES_REST = "http://rest.ensemblgenomes.org";
+
+ protected static final String ENSEMBL_REST = "http://rest.ensembl.org";
+
/*
* possible values for the 'feature' parameter of the /overlap REST service
* @see http://rest.ensembl.org/documentation/info/overlap_id
constrained, regulatory
}
+ private String domain = ENSEMBL_REST;
+
@Override
public String getDbSource()
{
// NB ensure Uniprot xrefs are canonicalised from "Ensembl" to "ENSEMBL"
- return DBRefSource.ENSEMBL; // "ENSEMBL"
- }
-
- @Override
- public String getDbVersion()
- {
- return "0";
+ if (ENSEMBL_GENOMES_REST.equals(getDomain()))
+ {
+ return DBRefSource.ENSEMBLGENOMES;
+ }
+ return DBRefSource.ENSEMBL;
}
@Override
{
return true;
}
+
+ /**
+ * Returns the domain name to query e.g. http://rest.ensembl.org or
+ * http://rest.ensemblgenomes.org
+ *
+ * @return
+ */
+ protected String getDomain()
+ {
+ return domain;
+ }
+
+ protected void setDomain(String d)
+ {
+ domain = d;
+ }
}
import jalview.structures.models.AAStructureBindingModel;
import jalview.util.MessageManager;
import jalview.util.Platform;
+import jalview.util.StringUtils;
import jalview.util.jarInputStreamProvider;
import jalview.viewmodel.AlignmentViewport;
import jalview.viewmodel.seqfeatures.FeatureRendererSettings;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.StringTokenizer;
import java.util.Vector;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
}
/**
+ * Answers true if 'version' is equal to or later than 'supported', where each
+ * is formatted as major/minor versions like "2.8.3" or "2.3.4b1" for bugfix
+ * changes. Development and test values for 'version' are leniently treated
+ * i.e. answer true.
*
* @param supported
* - minimum version we are comparing against
* @param version
- * - version of data being processsed.
- * @return true if version is development/null or evaluates to the same or
- * later X.Y.Z (where X,Y,Z are like [0-9]+b?[0-9]*)
+ * - version of data being processsed
+ * @return
*/
public static boolean isVersionStringLaterThan(String supported,
String version)
{
- if (version == null || version.equalsIgnoreCase("DEVELOPMENT BUILD")
+ if (supported == null || version == null
+ || version.equalsIgnoreCase("DEVELOPMENT BUILD")
|| version.equalsIgnoreCase("Test")
|| version.equalsIgnoreCase("AUTOMATED BUILD"))
{
}
else
{
- StringTokenizer currentV = new StringTokenizer(supported, "."), fileV = new StringTokenizer(
- version, ".");
- while (currentV.hasMoreTokens() && fileV.hasMoreTokens())
- {
- // convert b to decimal to catch bugfix releases within a series
- String curT = currentV.nextToken().toLowerCase().replace('b', '.');
- String fileT = fileV.nextToken().toLowerCase().replace('b', '.');
- try
- {
- float supportedVersionToken = Float.parseFloat(curT);
- float myVersiontoken = Float.parseFloat(fileT);
- if (supportedVersionToken > myVersiontoken)
- {
- // current version is newer than the version that wrote the file
- return false;
- }
- if (supportedVersionToken < myVersiontoken)
- {
- // current version is older than the version that wrote the file
- return true;
- }
- } catch (NumberFormatException nfe)
- {
- System.err
- .println("** WARNING: Version comparison failed for tokens ("
- + curT
- + ") and ("
- + fileT
- + ")\n** Current: '"
- + supported + "' and Version: '" + version + "'");
- }
- }
- if (currentV.hasMoreElements())
- {
- // fileV has no minor version but identical series to current
- return false;
- }
+ return StringUtils.compareVersions(version, supported, "b") >= 0;
}
- return true;
}
Vector<JalviewStructureDisplayI> newStructureViewers = null;
}
return result;
}
+
+ /**
+ * Compares two versions formatted as e.g. "3.4.5" and returns -1, 0 or 1 as
+ * the first version precedes, is equal to, or follows the second
+ *
+ * @param v1
+ * @param v2
+ * @return
+ */
+ public static int compareVersions(String v1, String v2)
+ {
+ return compareVersions(v1, v2, null);
+ }
+
+ /**
+ * Compares two versions formatted as e.g. "3.4.5b1" and returns -1, 0 or 1 as
+ * the first version precedes, is equal to, or follows the second
+ *
+ * @param v1
+ * @param v2
+ * @param pointSeparator
+ * a string used to delimit point increments in sub-tokens of the
+ * version
+ * @return
+ */
+ public static int compareVersions(String v1, String v2,
+ String pointSeparator)
+ {
+ if (v1 == null || v2 == null)
+ {
+ return 0;
+ }
+ String[] toks1 = v1.split("\\.");
+ String[] toks2 = v2.split("\\.");
+ int i = 0;
+ for (; i < toks1.length; i++)
+ {
+ if (i >= toks2.length)
+ {
+ /*
+ * extra tokens in v1
+ */
+ return 1;
+ }
+ String tok1 = toks1[i];
+ String tok2 = toks2[i];
+ if (pointSeparator != null)
+ {
+ /*
+ * convert e.g. 5b2 into decimal 5.2 for comparison purposes
+ */
+ tok1 = tok1.replace(pointSeparator, ".");
+ tok2 = tok2.replace(pointSeparator, ".");
+ }
+ try
+ {
+ float f1 = Float.valueOf(tok1);
+ float f2 = Float.valueOf(tok2);
+ int comp = Float.compare(f1, f2);
+ if (comp != 0)
+ {
+ return comp;
+ }
+ } catch (NumberFormatException e)
+ {
+ System.err.println("Invalid version format found: "
+ + e.getMessage());
+ return 0;
+ }
+ }
+
+ if (i < toks2.length)
+ {
+ /*
+ * extra tokens in v2
+ */
+ return -1;
+ }
+
+ /*
+ * same length, all tokens match
+ */
+ return 0;
+ }
}
{
@Test(suiteName = "live")
- public void testLiveCheckEnsembl()
+ public void testIsEnsemblAvailable()
{
EnsemblRestClient sf = new EnsemblRestClient()
{
}
}
- @Test(suiteName = "live")
- public void testLiveCheckEnsembl()
- {
- EnsemblRestClient sf = new EnsemblRestClient()
- {
-
- @Override
- public String getDbName()
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public AlignmentI getSequenceRecords(String queries) throws Exception
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- protected URL getUrl(List<String> ids) throws MalformedURLException
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- protected boolean useGetRequest()
- {
- // TODO Auto-generated method stub
- return false;
- }
-
- @Override
- protected String getRequestMimeType(boolean b)
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- protected String getResponseMimeType()
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- };
- boolean isAvailable = sf.isEnsemblAvailable();
- System.out.println("Ensembl is "
- + (isAvailable ? "UP!"
- : "DOWN or unreachable ******************* BAD!"));
- }
-
@Test(groups = "Functional")
public void getGenomicRangesFromFeatures()
{
*/
assertTrue(Jalview2XML.isVersionStringLaterThan(null, null));
assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", null));
+ assertTrue(Jalview2XML.isVersionStringLaterThan(null, "2.8.3"));
assertTrue(Jalview2XML.isVersionStringLaterThan(null,
"Development Build"));
assertTrue(Jalview2XML.isVersionStringLaterThan(null,
assertEquals(0,
StringUtils.parseInt(String.valueOf(Integer.MAX_VALUE) + "1"));
}
+
+ @Test(groups = { "Functional" })
+ public void testCompareVersions()
+ {
+ assertEquals(0, StringUtils.compareVersions(null, null));
+ assertEquals(0, StringUtils.compareVersions("2.8.3", null));
+
+ /*
+ * same version returns 0
+ */
+ assertEquals(0, StringUtils.compareVersions("2.8", "2.8"));
+ assertEquals(0, StringUtils.compareVersions("2.8.3", "2.8.3"));
+ assertEquals(0, StringUtils.compareVersions("2.8.3b1", "2.8.3b1", "b"));
+ assertEquals(0, StringUtils.compareVersions("2.8.3B1", "2.8.3b1", "b"));
+ assertEquals(0, StringUtils.compareVersions("2.8.3b1", "2.8.3B1", "b"));
+
+ /*
+ * v1 < v2 returns -1
+ */
+ assertEquals(-1, StringUtils.compareVersions("2.8.3", "2.8.4"));
+ assertEquals(-1, StringUtils.compareVersions("2.8.3", "2.9"));
+ assertEquals(-1, StringUtils.compareVersions("2.8.3", "2.9.2"));
+ assertEquals(-1, StringUtils.compareVersions("2.8", "2.8.3"));
+ assertEquals(-1, StringUtils.compareVersions("2.8.3", "2.8.3b1", "b"));
+ assertEquals(-1, StringUtils.compareVersions("2.8.3b1", "2.8.3b2", "b"));
+ assertEquals(-1, StringUtils.compareVersions("2.8", "2.8.0", "b"));
+ assertEquals(-1, StringUtils.compareVersions("2", "12"));
+ assertEquals(-1, StringUtils.compareVersions("3.2.4", "3.12.11"));
+
+ /*
+ * v1 > v2 returns +1
+ */
+ assertEquals(1, StringUtils.compareVersions("2.8.3", "2.8"));
+ assertEquals(1, StringUtils.compareVersions("2.8.0", "2.8"));
+ assertEquals(1, StringUtils.compareVersions("2.8.4", "2.8.3"));
+ assertEquals(1, StringUtils.compareVersions("2.8.3b1", "2.8.3", "b"));
+ assertEquals(1, StringUtils.compareVersions("2.8.3", "2.8.2b1", "b"));
+ assertEquals(1, StringUtils.compareVersions("2.8.0b2", "2.8.0b1", "b"));
+ assertEquals(1, StringUtils.compareVersions("12", "2"));
+ assertEquals(1, StringUtils.compareVersions("3.12.11", "3.2.4"));
+ }
}
git://source.jalview.org / jalview.git / commitdiff