+ /**
+ * Recognise the 2-byte magic header indicating a gzipped stream
+ *
+ * see
+ * https://recalll.co/ask/v/topic/java-How-to-check-if-InputStream-is-Gzipped/555aadd62bd27354438b90f6
+ *
+ * @param input
+ * - input stream that supports mark and contains at least two bytes
+ *
+ * @return false if mark not supported or no magic header found
+ *
+ * @throws IOException
+ */
+ public static boolean isGzipStream(InputStream input) throws IOException
+ {
+ if (!input.markSupported())
+ {
+ Cache.log.error(
+ "FileParse.izGzipStream: input stream must support mark/reset");
+ return false;
+ }
+ input.mark(4);
+
+ // get first 2 bytes or return false
+ byte[] bytes = new byte[2];
+ int read = input.read(bytes);
+ input.reset();
+ if (read != bytes.length)
+ {
+ return false;
+ }
+
+ int header = (bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);
+ return (GZIPInputStream.GZIP_MAGIC == header);
+ }
+
+ /**
+ * Returns a Reader for the given input after wrapping it in a buffered input
+ * stream, and then checking if it needs to be wrapped by a GZipInputStream
+ *
+ * @param input
+ * @return
+ */
+ private BufferedReader checkForGzipStream(InputStream input) throws Exception {
+
+ // NB: stackoverflow https://stackoverflow.com/questions/4818468/how-to-check-if-inputstream-is-gzipped
+ // could use a PushBackInputStream rather than a BufferedInputStream
+
+ if (!input.markSupported()) {
+ input = new BufferedInputStream(input,16);
+ }
+ if (isGzipStream(input)) {
+ return getGzipReader(input);
+ }
+ // return a buffered reader for the stream.
+ InputStreamReader isReader= new InputStreamReader(input);
+ BufferedReader toReadFrom=new BufferedReader(isReader);
+ return toReadFrom;
+ }
+ /**
+ * Returns a {@code BufferedReader} which wraps the input stream with a
+ * GZIPInputStream. Throws a {@code ZipException} if a GZIP format error
+ * occurs or the compression method used is unsupported.
+ *
+ * @param inputStream
+ * @return
+ * @throws Exception
+ */
+ private BufferedReader getGzipReader(InputStream inputStream)
+ throws Exception
+ {
+ BufferedReader inData = new BufferedReader(
+ new InputStreamReader(new GZIPInputStream(inputStream)));
+ inData.mark(2048);
+ inData.read();
+ inData.reset();
+ return inData;
+ }
+
+ /**
+ * Tries to read from the given URL. If successful, saves a reader to the
+ * response in field {@code dataIn}, otherwise (on exception, or HTTP response
+ * status not 200), throws an exception.
+ * <p>
+ * If the response status includes
+ *
+ * <pre>
+ * Content-Type : application/x-gzip
+ * </pre>
+ *
+ * then tries to read as gzipped content.
+ *
+ * @param urlStr
+ * @throws IOException
+ * @throws MalformedURLException
+ */
+ private void checkURLSource(String urlStr)
+ throws IOException, MalformedURLException