2 * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
3 * Copyright (C) $$Year-Rel$$ The Jalview Authors
5 * This file is part of Jalview.
7 * Jalview is free software: you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation, either version 3
10 * of the License, or (at your option) any later version.
12 * Jalview is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty
14 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
15 * PURPOSE. See the GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Jalview. If not, see <http://www.gnu.org/licenses/>.
19 * The Jalview Authors are detailed in the 'AUTHORS' file.
23 import java.io.BufferedInputStream;
24 import java.io.BufferedReader;
25 import java.io.ByteArrayInputStream;
27 import java.io.FileInputStream;
28 import java.io.FileReader;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.InputStreamReader;
32 import java.io.Reader;
33 import java.io.StringReader;
34 import java.net.HttpURLConnection;
35 import java.net.MalformedURLException;
37 import java.net.URLConnection;
38 import java.util.zip.GZIPInputStream;
40 import jalview.api.AlignExportSettingsI;
41 import jalview.api.AlignViewportI;
42 import jalview.api.AlignmentViewPanel;
43 import jalview.api.FeatureSettingsModelI;
44 import jalview.bin.Cache;
45 import jalview.util.MessageManager;
46 import jalview.util.Platform;
49 * implements a random access wrapper around a particular datasource, for
50 * passing to identifyFile and AlignFile objects.
52 public class FileParse
54 protected static final String SPACE = " ";
56 protected static final String TAB = "\t";
59 * text specifying source of data. usually filename or url.
61 private String dataName = "unknown source";
63 public File inFile = null;
65 private byte[] bytes; // from JavaScript
67 public byte[] getBytes()
73 * a viewport associated with the current file operation. May be null. May
74 * move to different object.
76 private AlignViewportI viewport;
79 * specific settings for exporting data from the current context
81 private AlignExportSettingsI exportSettings;
84 * sequence counter for FileParse object created from same data source
89 * separator for extracting specific 'frame' of a datasource for formats that
90 * support multiple records (e.g. BLC, Stockholm, etc)
92 protected char suffixSeparator = '#';
95 * character used to write newlines
97 protected String newline = System.getProperty("line.separator");
99 public void setNewlineString(String nl)
104 public String getNewlineString()
110 * '#' separated string tagged on to end of filename or url that was clipped
111 * off to resolve to valid filename
113 protected String suffix = null;
115 protected DataSourceType dataSourceType = null;
117 protected BufferedReader dataIn = null;
119 protected String errormessage = "UNINITIALISED SOURCE";
121 protected boolean error = true;
123 protected String warningMessage = null;
126 * size of readahead buffer used for when initial stream position is marked.
128 final int READAHEAD_LIMIT = 2048;
135 * Create a new FileParse instance reading from the same datasource starting
136 * at the current position. WARNING! Subsequent reads from either object will
137 * affect the read position of the other, but not the error state.
141 public FileParse(FileParse from) throws IOException
145 throw new Error(MessageManager
146 .getString("error.implementation_error_null_fileparse"));
152 index = ++from.index;
153 inFile = from.inFile;
154 suffixSeparator = from.suffixSeparator;
155 suffix = from.suffix;
156 errormessage = from.errormessage; // inherit potential error messages
157 error = false; // reset any error condition.
158 dataSourceType = from.dataSourceType;
159 dataIn = from.dataIn;
164 dataName = from.dataName;
168 * Attempt to open a file as a datasource. Sets error and errormessage if
169 * fileStr was invalid.
172 * @return this.error (true if the source was invalid)
174 private boolean checkFileSource(String fileStr) throws IOException
177 this.inFile = new File(fileStr);
178 // check to see if it's a Jar file in disguise.
179 if (!inFile.exists())
181 errormessage = "FILE NOT FOUND";
184 if (!inFile.canRead())
186 errormessage = "FILE CANNOT BE OPENED FOR READING";
189 if (inFile.isDirectory())
191 // this is really a 'complex' filetype - but we don't handle directory
193 errormessage = "FILE IS A DIRECTORY";
200 dataIn = checkForGzipStream(new FileInputStream(fileStr));
202 } catch (Exception x)
204 warningMessage = "Failed to resolve " + fileStr
205 + " as a data source. (" + x.getMessage() + ")";
206 // x.printStackTrace();
215 * Recognise the 2-byte magic header for gzip streams
217 * https://recalll.co/ask/v/topic/java-How-to-check-if-InputStream-is-Gzipped/555aadd62bd27354438b90f6
220 * - at least two bytes
222 * @throws IOException
224 public static boolean isGzipStream(InputStream input) throws IOException
226 if (!input.markSupported())
229 "FileParse.izGzipStream: input stream must support mark/reset");
234 // get first 2 bytes or return false
235 byte[] bytes = new byte[2];
236 int read = input.read(bytes);
238 if (read != bytes.length)
243 int header = (bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);
244 return (GZIPInputStream.GZIP_MAGIC == header);
248 * Returns a Reader for the given input after wrapping it in a buffered input
249 * stream, and then checking if it needs to be wrapped by a GZipInputStream
254 private BufferedReader checkForGzipStream(InputStream input)
258 // https://stackoverflow.com/questions/4818468/how-to-check-if-inputstream-is-gzipped
259 // could use a PushBackInputStream rather than a BufferedInputStream
260 if (!input.markSupported())
262 input = new BufferedInputStream(input, 16);
264 if (isGzipStream(input))
266 return getGzipReader(input);
268 // return a buffered reader for the stream.
269 InputStreamReader isReader = new InputStreamReader(input);
270 BufferedReader toReadFrom = new BufferedReader(isReader);
275 * Returns a {@code BufferedReader} which wraps the input stream with a
276 * GZIPInputStream. Throws a {@code ZipException} if a GZIP format error
277 * occurs or the compression method used is unsupported.
283 private BufferedReader getGzipReader(InputStream inputStream)
286 BufferedReader inData = new BufferedReader(
287 new InputStreamReader(new GZIPInputStream(inputStream)));
289 System.out.println("ABOUT TO inData.read()");
291 System.out.println("ABOUT TO inData.reset()");
297 * Tries to read from the given URL. If successful, saves a reader to the
298 * response in field {@code dataIn}, otherwise (on exception, or HTTP response
299 * status not 200), throws an exception.
301 * If the response status includes
304 * Content-Type : application/x-gzip
307 * then tries to read as gzipped content.
310 * @throws IOException
311 * @throws MalformedURLException
313 private void checkURLSource(String urlStr)
314 throws IOException, MalformedURLException
316 errormessage = "URL NOT FOUND";
317 URL url = new URL(urlStr);
318 URLConnection _conn = url.openConnection();
319 if (_conn instanceof HttpURLConnection)
321 HttpURLConnection conn = (HttpURLConnection) _conn;
322 int rc = conn.getResponseCode();
323 if (rc != HttpURLConnection.HTTP_OK)
325 throw new IOException(
326 "Response status from " + urlStr + " was " + rc);
333 dataIn = checkForGzipStream(_conn.getInputStream());
335 } catch (IOException ex)
337 throw new IOException("Failed to handle non-HTTP URI stream", ex);
338 } catch (Exception ex)
340 throw new IOException(
341 "Failed to determine type of input stream for given URI",
346 String encoding = _conn.getContentEncoding();
347 String contentType = _conn.getContentType();
348 boolean isgzipped = "application/x-gzip".equalsIgnoreCase(contentType)
349 || "gzip".equals(encoding);
351 InputStream inputStream = _conn.getInputStream();
356 dataIn = getGzipReader(inputStream);
358 } catch (Exception e1)
360 throw new IOException(MessageManager
361 .getString("exception.failed_to_resolve_gzip_stream"), e);
366 dataIn = new BufferedReader(new InputStreamReader(inputStream));
372 * sets the suffix string (if any) and returns remainder (if suffix was
376 * @return truncated fileStr or null
378 private String extractSuffix(String fileStr)
380 // first check that there wasn't a suffix string tagged on.
381 int sfpos = fileStr.lastIndexOf(suffixSeparator);
382 if (sfpos > -1 && sfpos < fileStr.length() - 1)
384 suffix = fileStr.substring(sfpos + 1);
385 // System.err.println("DEBUG: Found Suffix:"+suffix);
386 return fileStr.substring(0, sfpos);
392 * not for general use, creates a fileParse object for an existing reader with
393 * configurable values for the origin and the type of the source
395 public FileParse(BufferedReader source, String originString,
396 DataSourceType sourceType)
398 dataSourceType = sourceType;
401 dataName = originString;
405 if (dataIn.markSupported())
407 dataIn.mark(READAHEAD_LIMIT);
409 } catch (IOException q)
416 * Create a datasource for input to Jalview. See AppletFormatAdapter for the
417 * types of sources that are handled.
420 * - datasource locator/content as File or String
422 * - protocol of source
423 * @throws MalformedURLException
424 * @throws IOException
426 public FileParse(Object file, DataSourceType sourceType)
427 throws MalformedURLException, IOException
429 if (file instanceof File)
431 parse((File) file, ((File) file).getPath(), sourceType, true);
435 parse(null, file.toString(), sourceType, false);
439 private void parse(File file, String fileStr, DataSourceType sourceType,
440 boolean isFileObject) throws IOException
442 bytes = Platform.getFileBytes(file);
443 dataSourceType = sourceType;
446 if (sourceType == DataSourceType.FILE)
451 // this will be from JavaScript
453 // dataIn = new BufferedReader(new InputStreamReader(new
454 // ByteArrayInputStream(bytes)));
457 dataIn = checkForGzipStream(new ByteArrayInputStream(bytes));
458 } catch (Exception e)
464 else if (checkFileSource(fileStr))
466 String suffixLess = extractSuffix(fileStr);
467 if (suffixLess != null)
469 if (checkFileSource(suffixLess))
471 throw new IOException(MessageManager.formatMessage(
472 "exception.problem_opening_file_also_tried",
474 { inFile.getName(), suffixLess, errormessage }));
479 throw new IOException(MessageManager.formatMessage(
480 "exception.problem_opening_file", new String[]
481 { inFile.getName(), errormessage }));
485 else if (sourceType == DataSourceType.RELATIVE_URL)
487 // BH 2018 hack for no support for access-origin
488 bytes = Platform.getFileAsBytes(fileStr);
489 dataIn = new BufferedReader(
490 new InputStreamReader(new ByteArrayInputStream(bytes)));
494 else if (sourceType == DataSourceType.URL)
500 checkURLSource(fileStr);
501 if (suffixSeparator == '#')
503 extractSuffix(fileStr); // URL lref is stored for later reference.
505 } catch (IOException e)
507 String suffixLess = extractSuffix(fileStr);
508 if (suffixLess == null)
516 checkURLSource(suffixLess);
517 } catch (IOException e2)
519 errormessage = "BAD URL WITH OR WITHOUT SUFFIX";
520 throw (e); // just pass back original - everything was wrong.
524 } catch (Exception e)
526 errormessage = "CANNOT ACCESS DATA AT URL '" + fileStr + "' ("
527 + e.getMessage() + ")";
531 else if (sourceType == DataSourceType.PASTE)
533 errormessage = "PASTE INACCESSIBLE!";
534 dataIn = new BufferedReader(new StringReader(fileStr));
537 else if (sourceType == DataSourceType.CLASSLOADER)
539 errormessage = "RESOURCE CANNOT BE LOCATED";
540 InputStream is = getClass().getResourceAsStream("/" + fileStr);
543 String suffixLess = extractSuffix(fileStr);
544 if (suffixLess != null)
546 is = getClass().getResourceAsStream("/" + suffixLess);
551 dataIn = new BufferedReader(new InputStreamReader(is));
561 errormessage = "PROBABLE IMPLEMENTATION ERROR : Datasource Type given as '"
562 + (sourceType != null ? sourceType : "null") + "'";
565 if (dataIn == null || error)
567 // pass up the reason why we have no source to read from
568 throw new IOException(MessageManager.formatMessage(
569 "exception.failed_to_read_data_from_source", new String[]
573 dataIn.mark(READAHEAD_LIMIT);
577 * mark the current position in the source as start for the purposes of it
578 * being analysed by IdentifyFile().identify
580 * @throws IOException
582 public void mark() throws IOException
586 dataIn.mark(READAHEAD_LIMIT);
590 throw new IOException(
591 MessageManager.getString("exception.no_init_source_stream"));
595 public String nextLine() throws IOException
599 return dataIn.readLine();
601 throw new IOException(MessageManager
602 .formatMessage("exception.invalid_source_stream", new String[]
608 * @return true if this FileParse is configured for Export only
610 public boolean isExporting()
612 return !error && dataIn == null;
617 * @return true if the data source is valid
619 public boolean isValid()
625 * closes the datasource and tidies up. source will be left in an error state
627 public void close() throws IOException
629 errormessage = "EXCEPTION ON CLOSE";
633 errormessage = "SOURCE IS CLOSED";
637 * Rewinds the datasource to the marked point if possible
642 public void reset(int bytesRead) throws IOException
644 if (bytesRead >= READAHEAD_LIMIT)
646 System.err.println(String.format(
647 "File reset error: read %d bytes but reset limit is %d",
648 bytesRead, READAHEAD_LIMIT));
650 if (dataIn != null && !error)
656 throw new IOException(MessageManager.getString(
657 "error.implementation_error_reset_called_for_invalid_source"));
663 * @return true if there is a warning for the user
665 public boolean hasWarningMessage()
667 return (warningMessage != null && warningMessage.length() > 0);
672 * @return empty string or warning message about file that was just parsed.
674 public String getWarningMessage()
676 return warningMessage;
679 public String getInFile()
683 return inFile.getAbsolutePath() + " (" + index + ")";
687 return "From Paste + (" + index + ")";
692 * @return the dataName
694 public String getDataName()
700 * set the (human readable) name or URI for this datasource
704 protected void setDataName(String dataname)
710 * get the underlying bufferedReader for this data source.
712 * @return null if no reader available
713 * @throws IOException
715 public Reader getReader()
717 if (dataIn != null) // Probably don't need to test for readiness &&
725 public AlignViewportI getViewport()
730 public void setViewport(AlignViewportI viewport)
732 this.viewport = viewport;
736 * @return the currently configured exportSettings for writing data.
738 public AlignExportSettingsI getExportSettings()
740 return exportSettings;
744 * Set configuration for export of data.
746 * @param exportSettings
747 * the exportSettings to set
749 public void setExportSettings(AlignExportSettingsI exportSettings)
751 this.exportSettings = exportSettings;
755 * method overridden by complex file exporter/importers which support
756 * exporting visualisation and layout settings for a view
760 public void configureForView(AlignmentViewPanel avpanel)
764 setViewport(avpanel.getAlignViewport());
766 // could also set export/import settings
770 * Returns the preferred feature colour configuration if there is one, else
775 public FeatureSettingsModelI getFeatureColourScheme()
780 public DataSourceType getDataSourceType()
782 return dataSourceType;
786 * Returns a buffered reader for the input object. Returns null, or throws
787 * IOException, on failure.
790 * a File, or a String which is a name of a file
793 * @throws IOException
795 public BufferedReader getBufferedReader(Object file,
796 DataSourceType sourceType) throws IOException
798 BufferedReader in = null;
804 if (file instanceof String)
806 return new BufferedReader(new FileReader((String) file));
808 bytes = Platform.getFileBytes((File) file);
811 return new BufferedReader(
812 new InputStreamReader(new ByteArrayInputStream(bytes)));
814 return new BufferedReader(new FileReader((File) file));
816 URL url = new URL(file.toString());
817 in = new BufferedReader(new InputStreamReader(url.openStream()));
819 case RELATIVE_URL: // JalviewJS only
820 bytes = Platform.getFileAsBytes(file.toString());
823 in = new BufferedReader(
824 new InputStreamReader(new ByteArrayInputStream(bytes)));
828 in = new BufferedReader(new StringReader(file.toString()));
831 InputStream is = getClass().getResourceAsStream("/" + file);
834 in = new BufferedReader(new InputStreamReader(is));