From df81a9e1abd4d4424d6d24b37d93f17bffd3649c Mon Sep 17 00:00:00 2001 From: gmungoc Date: Thu, 2 Jul 2020 15:28:36 +0100 Subject: [PATCH] JAL-3615 used gzip endpoints for Pfam and Rfam redact parent commits related to JalviewJS for 2.11.1.1 patch --- src/jalview/io/FileParse.java | 32 ++++++++++++++++++++++++++++-- src/jalview/ws/dbsources/Pfam.java | 5 +++++ src/jalview/ws/dbsources/PfamFull.java | 2 +- src/jalview/ws/dbsources/PfamSeed.java | 2 +- src/jalview/ws/dbsources/Rfam.java | 5 +++++ src/jalview/ws/dbsources/RfamFull.java | 2 +- src/jalview/ws/dbsources/RfamSeed.java | 3 +-- src/jalview/ws/dbsources/Xfam.java | 1 - test/jalview/io/FileParseTest.java | 34 ++++++++++++++++++++++++++++++++ 9 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 test/jalview/io/FileParseTest.java diff --git a/src/jalview/io/FileParse.java b/src/jalview/io/FileParse.java index 7117d0f..68b5d28 100755 --- a/src/jalview/io/FileParse.java +++ b/src/jalview/io/FileParse.java @@ -39,6 +39,10 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.zip.GZIPInputStream; + +import jalview.ws.dbsources.Pfam; +import jalview.ws.dbsources.Rfam; + /** * implements a random access wrapper around a particular datasource, for * passing to identifyFile and AlignFile objects. @@ -49,6 +53,8 @@ public class FileParse protected static final String TAB = "\t"; + private static final String GZ_EXT = ".gz"; + /** * text specifying source of data. usually filename or url. */ @@ -56,6 +62,7 @@ public class FileParse public File inFile = null; + /** * a viewport associated with the current file operation. May be null. May * move to different object. @@ -182,7 +189,7 @@ public class FileParse } if (!error) { - if (fileStr.toLowerCase().endsWith(".gz")) + if (fileStr.toLowerCase().endsWith(GZ_EXT)) { try { @@ -224,7 +231,7 @@ public class FileParse // GZIPInputStream code borrowed from Aquaria (soon to be open sourced) via // Kenny Sabir Exception e = null; - if (fileStr.toLowerCase().endsWith(".gz")) + if (isGzipped(fileStr)) { try { @@ -256,6 +263,27 @@ public class FileParse } /** + * Answers true if the filename (or URL) has a format which Jalview recognises + * as denoting gzipped content. + *

+ * Currently this means having a ".gz" extension, or ending in "/gzipped" or + * "?gz=1" (used to retrieve gzipped from Pfam and Rfam respectively). + * + * @param filename + * @return + */ + protected static boolean isGzipped(String filename) + { + if (filename == null) + { + return false; + } + String lower = filename.toLowerCase(); + return lower.endsWith(GZ_EXT) || lower.endsWith(Pfam.GZIPPED) + || lower.endsWith(Rfam.GZIPPED); + } + + /** * sets the suffix string (if any) and returns remainder (if suffix was * detected) * diff --git a/src/jalview/ws/dbsources/Pfam.java b/src/jalview/ws/dbsources/Pfam.java index 8877c34..de19078 100644 --- a/src/jalview/ws/dbsources/Pfam.java +++ b/src/jalview/ws/dbsources/Pfam.java @@ -35,6 +35,11 @@ import com.stevesoft.pat.Regex; */ abstract public class Pfam extends Xfam { + /* + * append to URLs to retrieve as a gzipped file + */ + public static final String GZIPPED = "/gzipped"; + static final String PFAM_BASEURL_KEY = "PFAM_BASEURL"; private static final String DEFAULT_PFAM_BASEURL = "https://pfam.xfam.org"; diff --git a/src/jalview/ws/dbsources/PfamFull.java b/src/jalview/ws/dbsources/PfamFull.java index 0600427..d71892b 100644 --- a/src/jalview/ws/dbsources/PfamFull.java +++ b/src/jalview/ws/dbsources/PfamFull.java @@ -34,7 +34,7 @@ public class PfamFull extends Pfam @Override public String getURLSuffix() { - return "/alignment/full"; + return "/alignment/full" + GZIPPED; } /* diff --git a/src/jalview/ws/dbsources/PfamSeed.java b/src/jalview/ws/dbsources/PfamSeed.java index dff8a17..f64d07f 100644 --- a/src/jalview/ws/dbsources/PfamSeed.java +++ b/src/jalview/ws/dbsources/PfamSeed.java @@ -36,7 +36,7 @@ public class PfamSeed extends Pfam @Override public String getURLSuffix() { - return "/alignment/seed"; + return "/alignment/seed" + GZIPPED; } /* diff --git a/src/jalview/ws/dbsources/Rfam.java b/src/jalview/ws/dbsources/Rfam.java index 1d9d99a..dba3e31 100644 --- a/src/jalview/ws/dbsources/Rfam.java +++ b/src/jalview/ws/dbsources/Rfam.java @@ -36,6 +36,11 @@ abstract public class Rfam extends Xfam private static final String DEFAULT_RFAM_BASEURL = "https://rfam.xfam.org"; + /* + * append to URLs to retrieve as a gzipped file + */ + public static final String GZIPPED = "?gz=1&download=1"; + @Override protected String getURLPrefix() { diff --git a/src/jalview/ws/dbsources/RfamFull.java b/src/jalview/ws/dbsources/RfamFull.java index d815336..396511c 100644 --- a/src/jalview/ws/dbsources/RfamFull.java +++ b/src/jalview/ws/dbsources/RfamFull.java @@ -36,7 +36,7 @@ public class RfamFull extends Rfam @Override public String getURLSuffix() { - return "/alignment/full"; + return "/alignment/full" + GZIPPED; } /* diff --git a/src/jalview/ws/dbsources/RfamSeed.java b/src/jalview/ws/dbsources/RfamSeed.java index a74e829..eaa574b 100644 --- a/src/jalview/ws/dbsources/RfamSeed.java +++ b/src/jalview/ws/dbsources/RfamSeed.java @@ -36,8 +36,7 @@ public class RfamSeed extends Rfam @Override public String getURLSuffix() { - // to download gzipped file add '?gzip=1' - return "/alignment/stockholm"; + return "/alignment/stockholm" + GZIPPED; } /* diff --git a/src/jalview/ws/dbsources/Xfam.java b/src/jalview/ws/dbsources/Xfam.java index b83f558..f0cb14b 100644 --- a/src/jalview/ws/dbsources/Xfam.java +++ b/src/jalview/ws/dbsources/Xfam.java @@ -36,7 +36,6 @@ import jalview.ws.seqfetcher.DbSourceProxyImpl; */ public abstract class Xfam extends DbSourceProxyImpl { - public Xfam() { super(); diff --git a/test/jalview/io/FileParseTest.java b/test/jalview/io/FileParseTest.java new file mode 100644 index 0000000..7eb2c9c --- /dev/null +++ b/test/jalview/io/FileParseTest.java @@ -0,0 +1,34 @@ +package jalview.io; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import org.testng.annotations.Test; + +public class FileParseTest +{ + @Test(groups = "Functional") + public void setIsGzipped() + { + assertFalse(FileParse.isGzipped(null)); + assertFalse(FileParse.isGzipped("foobar")); + assertFalse(FileParse.isGzipped(".gz.foobar")); + + assertTrue(FileParse.isGzipped("abc.gz")); + assertTrue(FileParse.isGzipped("abc.GZ")); + assertTrue(FileParse.isGzipped(".gz")); + assertFalse(FileParse.isGzipped("abc/gz")); + assertFalse(FileParse.isGzipped("gz")); + + assertTrue(FileParse.isGzipped("http:/xy.com/abc/gzipped")); + assertTrue(FileParse.isGzipped("abc/gzipped")); + assertTrue(FileParse.isGzipped("abc/GZIPPED")); + assertTrue(FileParse.isGzipped("/gzipped")); + assertFalse(FileParse.isGzipped("gzipped")); + + assertTrue(FileParse.isGzipped("http:/xy.com/abc?gz=1")); + assertTrue(FileParse.isGzipped("http:/xy.com/abc?GZ=1")); + // currently only recognised if the last token on the URL + assertFalse(FileParse.isGzipped("http:/xy.com/abc?gz=1&content-type=text/xml")); + } +} -- 1.7.10.2