+ /**
+ * Returns a Reader for the given input after wrapping it in a buffered input
+ * stream, and then checking if it needs to be wrapped by a GZipInputStream
+ *
+ * @param input
+ * @return
+ */
+ private BufferedReader checkForGzipStream(InputStream input) throws Exception {
+
+ // NB: stackoverflow https://stackoverflow.com/questions/4818468/how-to-check-if-inputstream-is-gzipped
+ // could use a PushBackInputStream rather than a BufferedInputStream
+
+ BufferedInputStream bufinput;
+ if (!input.markSupported()) {
+ bufinput= new BufferedInputStream(input,16);
+ input = bufinput;
+ }
+ input.mark(4);
+ byte[] bytes=input.readNBytes(2);
+ input.reset();
+ if (bytes.length==2 && isGzipStream(bytes)) {
+ return getGzipReader(input);
+ }
+ // return a buffered reader for the stream.
+ InputStreamReader isReader= new InputStreamReader(input);
+ BufferedReader toReadFrom=new BufferedReader(isReader);
+ return toReadFrom;
+ }
+ /**
+ * Returns a {@code BufferedReader} which wraps the input stream with a
+ * GZIPInputStream. Throws a {@code ZipException} if a GZIP format error
+ * occurs or the compression method used is unsupported.
+ *
+ * @param inputStream
+ * @return
+ * @throws Exception
+ */
+ private BufferedReader getGzipReader(InputStream inputStream)
+ throws Exception
+ {
+ BufferedReader inData = new BufferedReader(
+ new InputStreamReader(new GZIPInputStream(inputStream)));
+ inData.mark(2048);
+ inData.read();
+ inData.reset();
+ return inData;
+ }
+
+ /**
+ * Tries to read from the given URL. If successful, saves a reader to the
+ * response in field {@code dataIn}, otherwise (on exception, or HTTP response
+ * status not 200), throws an exception.
+ * <p>
+ * If the response status includes
+ *
+ * <pre>
+ * Content-Type : application/x-gzip
+ * </pre>
+ *
+ * then tries to read as gzipped content.
+ *
+ * @param urlStr
+ * @throws IOException
+ * @throws MalformedURLException
+ */
+ private void checkURLSource(String urlStr)
+ throws IOException, MalformedURLException