From 9e338a58713e1f3a451aa34c5e6c488bdd6d8cab Mon Sep 17 00:00:00 2001 From: gmungoc Date: Thu, 2 Jul 2020 17:24:28 +0100 Subject: [PATCH] JAL-2656 JAL-3615 check response headers to test for gzipped url reply --- src/jalview/io/FileParse.java | 106 ++++++++++++++++++------------------ src/jalview/ws/dbsources/Pfam.java | 2 +- src/jalview/ws/dbsources/Rfam.java | 2 +- test/jalview/io/FileParseTest.java | 34 ------------ 4 files changed, 54 insertions(+), 90 deletions(-) delete mode 100644 test/jalview/io/FileParseTest.java diff --git a/src/jalview/io/FileParse.java b/src/jalview/io/FileParse.java index 68b5d28..167314e 100755 --- a/src/jalview/io/FileParse.java +++ b/src/jalview/io/FileParse.java @@ -35,14 +35,12 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.util.zip.GZIPInputStream; -import jalview.ws.dbsources.Pfam; -import jalview.ws.dbsources.Rfam; - /** * implements a random access wrapper around a particular datasource, for * passing to identifyFile and AlignFile objects. @@ -53,8 +51,6 @@ public class FileParse protected static final String TAB = "\t"; - private static final String GZ_EXT = ".gz"; - /** * text specifying source of data. usually filename or url. */ @@ -189,11 +185,11 @@ public class FileParse } if (!error) { - if (fileStr.toLowerCase().endsWith(GZ_EXT)) + if (fileStr.toLowerCase().endsWith(".gz")) { try { - dataIn = tryAsGzipSource(new FileInputStream(fileStr)); + dataIn = getGzipReader(new FileInputStream(fileStr)); dataName = fileStr; return error; } catch (Exception x) @@ -211,7 +207,16 @@ public class FileParse return error; } - private BufferedReader tryAsGzipSource(InputStream inputStream) + /** + * Returns a {@code BufferedReader} which wraps the input stream with a + * GZIPInputStream. Throws a {@code ZipException} if a GZIP format error + * occurs or the compression method used is unsupported. + * + * @param inputStream + * @return + * @throws Exception + */ + private BufferedReader getGzipReader(InputStream inputStream) throws Exception { BufferedReader inData = new BufferedReader( @@ -222,65 +227,58 @@ public class FileParse return inData; } - private boolean checkURLSource(String fileStr) + /** + * Tries to read from the given URL. If successful, saves a reader to the + * response in field {@code dataIn}, otherwise (on exception, or HTTP response + * status not 200), throws an exception. + *

+ * If the response status includes + * + * 

+   * Content-Type : application/x-gzip
+   *
+ * + * then tries to read as gzipped content. + * + * @param urlStr + * @throws IOException + * @throws MalformedURLException + */ + private void checkURLSource(String urlStr) throws IOException, MalformedURLException { errormessage = "URL NOT FOUND"; - URL url = new URL(fileStr); - // - // GZIPInputStream code borrowed from Aquaria (soon to be open sourced) via - // Kenny Sabir + URL url = new URL(urlStr); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + int rc = conn.getResponseCode(); + if (rc != HttpURLConnection.HTTP_OK) + { + throw new IOException( + "Response status from " + urlStr + " was " + rc); + } + String encoding = conn.getContentEncoding(); + String contentType = conn.getContentType(); + boolean isgzipped = "application/x-gzip".equalsIgnoreCase(contentType) + || "gzip".equals(encoding); Exception e = null; - if (isGzipped(fileStr)) + InputStream inputStream = conn.getInputStream(); + if (isgzipped) { try { - InputStream inputStream = url.openStream(); - dataIn = tryAsGzipSource(inputStream); - dataName = fileStr; - return false; - } catch (Exception ex) - { - e = ex; - } - } - - try - { - dataIn = new BufferedReader(new InputStreamReader(url.openStream())); - } catch (IOException q) - { - if (e != null) + dataIn = getGzipReader(inputStream); + dataName = urlStr; + } catch (Exception e1) { throw new IOException(MessageManager .getString("exception.failed_to_resolve_gzip_stream"), e); } - throw q; + return; } - // record URL as name of datasource. - dataName = fileStr; - return false; - } - /** - * Answers true if the filename (or URL) has a format which Jalview recognises - * as denoting gzipped content. - *

- * Currently this means having a ".gz" extension, or ending in "/gzipped" or - * "?gz=1" (used to retrieve gzipped from Pfam and Rfam respectively). - * - * @param filename - * @return - */ - protected static boolean isGzipped(String filename) - { - if (filename == null) - { - return false; - } - String lower = filename.toLowerCase(); - return lower.endsWith(GZ_EXT) || lower.endsWith(Pfam.GZIPPED) - || lower.endsWith(Rfam.GZIPPED); + dataIn = new BufferedReader(new InputStreamReader(inputStream)); + dataName = urlStr; + return; } /** diff --git a/src/jalview/ws/dbsources/Pfam.java b/src/jalview/ws/dbsources/Pfam.java index de19078..47e66ac 100644 --- a/src/jalview/ws/dbsources/Pfam.java +++ b/src/jalview/ws/dbsources/Pfam.java @@ -38,7 +38,7 @@ abstract public class Pfam extends Xfam /* * append to URLs to retrieve as a gzipped file */ - public static final String GZIPPED = "/gzipped"; + protected static final String GZIPPED = "/gzipped"; static final String PFAM_BASEURL_KEY = "PFAM_BASEURL"; diff --git a/src/jalview/ws/dbsources/Rfam.java b/src/jalview/ws/dbsources/Rfam.java index dba3e31..c9ee7fc 100644 --- a/src/jalview/ws/dbsources/Rfam.java +++ b/src/jalview/ws/dbsources/Rfam.java @@ -39,7 +39,7 @@ abstract public class Rfam extends Xfam /* * append to URLs to retrieve as a gzipped file */ - public static final String GZIPPED = "?gz=1&download=1"; + protected static final String GZIPPED = "?gzip=1&download=1"; @Override protected String getURLPrefix() diff --git a/test/jalview/io/FileParseTest.java b/test/jalview/io/FileParseTest.java deleted file mode 100644 index 7eb2c9c..0000000 --- a/test/jalview/io/FileParseTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package jalview.io; - -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import org.testng.annotations.Test; - -public class FileParseTest -{ - @Test(groups = "Functional") - public void setIsGzipped() - { - assertFalse(FileParse.isGzipped(null)); - assertFalse(FileParse.isGzipped("foobar")); - assertFalse(FileParse.isGzipped(".gz.foobar")); - - assertTrue(FileParse.isGzipped("abc.gz")); - assertTrue(FileParse.isGzipped("abc.GZ")); - assertTrue(FileParse.isGzipped(".gz")); - assertFalse(FileParse.isGzipped("abc/gz")); - assertFalse(FileParse.isGzipped("gz")); - - assertTrue(FileParse.isGzipped("http:/xy.com/abc/gzipped")); - assertTrue(FileParse.isGzipped("abc/gzipped")); - assertTrue(FileParse.isGzipped("abc/GZIPPED")); - assertTrue(FileParse.isGzipped("/gzipped")); - assertFalse(FileParse.isGzipped("gzipped")); - - assertTrue(FileParse.isGzipped("http:/xy.com/abc?gz=1")); - assertTrue(FileParse.isGzipped("http:/xy.com/abc?GZ=1")); - // currently only recognised if the last token on the URL - assertFalse(FileParse.isGzipped("http:/xy.com/abc?gz=1&content-type=text/xml")); - } -} -- 1.7.10.2