2 * Jalview - A Sequence Alignment Editor and Viewer (Version 2.8.0b1)
3 * Copyright (C) 2014 The Jalview Authors
5 * This file is part of Jalview.
7 * Jalview is free software: you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
11 * Jalview is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
14 * PURPOSE. See the GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along with Jalview. If not, see <http://www.gnu.org/licenses/>.
17 * The Jalview Authors are detailed in the 'AUTHORS' file.
21 import jalview.datamodel.AlignmentAnnotation;
22 import jalview.datamodel.AlignmentI;
23 import jalview.datamodel.Annotation;
24 import jalview.datamodel.Sequence;
25 import jalview.datamodel.SequenceFeature;
26 import jalview.datamodel.SequenceI;
27 import jalview.schemes.UserColourScheme;
28 import jalview.util.Comparison;
30 import java.awt.Color;
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.HashMap;
34 import java.util.Iterator;
35 import java.util.LinkedHashMap;
36 import java.util.List;
38 import java.util.Map.Entry;
40 import java.util.Vector;
43 * A parser for input or output of MEGA format files. <br>
45 * Tamura K, Stecher G, Peterson D, Filipski A, and Kumar S (2013) MEGA6:
46 * Molecular Evolutionary Genetics Analysis Version 6.0. Molecular Biology and
47 * Evolution 30: 2725-2729. <br>
50 * MEGA file format is supported as described in
51 * http://www.megasoftware.net/manual.pdf <br>
54 * <li>any comments (delimited by [ ]) are ignored and not preserved</li>
57 * @see http://www.megasoftware.net/
59 public class MegaFile extends AlignFile
61 private static final String MEGA = "MEGA";
63 private static final String MEGA_ANNOTATION_LABEL = MEGA + " Label";
65 private static final String MEGA_ANNOTATION_GENE = MEGA + " Gene";
67 private static final String MEGA_ANNOTATION_DOMAIN = MEGA + " Domain";
69 private static final char UNDERSCORE = '_';
71 private static final String WHITESPACE = "\\s+";
73 private static final int DEFAULT_LINE_LENGTH = 60;
75 private static final String INDENT = " ";
77 private static final String N_SITES = "NSites";
79 private static final String N_SEQS = "NSeqs";
81 private static final String MISSING = "Missing";
83 private static final String IDENTICAL = "Identical";
85 private static final String INDEL = "Indel";
87 private static final String CODETABLE = "CodeTable";
89 private static final String PROTEIN = "Protein";
91 private static final String NUCLEOTIDE = "Nucleotide";
93 private static final String DATATYPE = "DataType";
95 private static final char COMMENT_START = '[';
97 private static final char COMMENT_END = ']';
99 private static final String HASHSIGN = "#";
101 private static final String SEMICOLON = ";";
103 private static final String BANG = "!";
105 private static final String EQUALS = "=";
107 private static final String MEGA_ID = HASHSIGN + MEGA;
109 private static final String TITLE = "Title";
111 private static final String FORMAT = "Format";
113 private static final String DESCRIPTION = "Description";
115 private static final String GENE = "Gene";
117 private static final String DOMAIN = "Domain";
119 private static final String PROPERTY = "Property";
121 private static final String CODONSTART = "CodonStart";
123 private static final String DOMAINEND = "domainend";
125 private static final String LABEL = "Label";
128 * names of properties to save to the alignment (may affect eventual output
131 static final String PROP_TITLE = "MEGA_TITLE";
133 static final String PROP_INTERLEAVED = "MEGA_INTERLEAVED";
135 static final String PROP_DESCRIPTION = "MEGA_DESCRIPTION";
137 static final String PROP_CODETABLE = "MEGA_CODETABLE";
139 static final String PROP_IDENTITY = "MEGA_IDENTITY";
141 static final String PROP_MISSING = "MEGA_MISSING";
143 static final String PROP_DATATYPE = "MEGA_DATATYPE";
145 // number of bases per line of file (value is inferred)
146 static final String PROP_LINELENGTH = "MEGA_LINELENGTH";
148 // TODO: need a controlled name for Gene as a feature if we want to be able to
149 // output the MEGA file with !Gene headers
150 // WTF do we do if the sequences get realigned?
152 // initial size for sequence data buffer
153 private static final int SEQBUFFERSIZE = 256;
155 private static final String SPACE = " ";
157 private static final String TAB = "\t";
159 private static final char DEFAULT_GAP = '-';
162 * number of sequence positions output per line
164 private int positionsPerLine;
166 private String title;
168 // gap character may be explicitly declared, default is -
169 private char gapCharacter = DEFAULT_GAP;
171 // identity character if declared
172 private char identityCharacter = 0;
174 // this can be True, False or null (meaning not asserted in file)
175 private Boolean nucleotide;
177 // set once we have seen one block of interleaved data
178 private boolean firstDataBlockRead = false;
180 // this can be True, False or null (meaning we don't know yet)
181 private Boolean interleaved;
183 // write end of line positions as a comment
184 private boolean writePositionNumbers = true;
186 // id of sequence being processed
187 private String currentSequenceId;
190 * Temporary store of {sequenceId, positionData} while parsing interleaved
191 * sequences; sequences are maintained in the order in which they are added
192 * i.e. read in the file
194 Map<String, StringBuilder> seqData;
196 // number of residues read (so far) per sequence
197 Map<String, Integer> residuesRead;
199 // current Gene if any we are parsing
200 private String currentGene;
202 // start position in alignment (base 0) of current Gene
203 private int currentGeneStartCol;
205 // start residue (base 1) per sequence of current Gene
206 private Map<String, Integer> geneStart;
208 // current Domain if any we are parsing
209 private String currentDomain;
211 // start position in alignment (base 0) of current Domain
212 private int currentDomainStartCol;
214 // start residue (base 1) per sequence of current Domain
215 private Map<String, Integer> domainStart;
217 // map of SequenceFeature's by sequence id
218 private Map<String, List<SequenceFeature>> sequenceFeatures;
220 // each !Label line character becomes an Annotation (except underscores)
221 List<Annotation> labelAnnotations;
223 // records any declared Gene positions (including null values)
224 List<Annotation> geneAnnotations;
226 // records any declared Domain positions (including null values)
227 List<Annotation> domainAnnotations;
233 public MegaFile(String inFile, String type) throws IOException
238 public MegaFile(FileParse source) throws IOException
244 * Parse the input stream.
247 public void parse() throws IOException
249 gapCharacter = DEFAULT_GAP;
250 sequenceFeatures = new HashMap<String, List<SequenceFeature>>();
251 geneStart = new HashMap<String, Integer>();
252 domainStart = new HashMap<String, Integer>();
253 residuesRead = new HashMap<String, Integer>();
254 labelAnnotations = new ArrayList<Annotation>();
255 geneAnnotations = new ArrayList<Annotation>();
256 domainAnnotations = new ArrayList<Annotation>();
257 currentDomainStartCol = -1;
258 currentGeneStartCol = -1;
261 * Read and process MEGA and Title/Format/Description headers if present.
262 * Returns the first data line following the headers.
264 String dataLine = parseHeaderLines();
267 * order-preserving map to hold sequences by id as they are built up during
270 seqData = new LinkedHashMap<String, StringBuilder>();
273 * The id of the sequence being read (for non-interleaved)
275 currentSequenceId = "";
277 while (dataLine != null)
279 dataLine = dataLine.trim();
280 if (dataLine.length() > 0)
282 dataLine = dataLine.replace(TAB, SPACE);
283 String upperCased = dataLine.toUpperCase();
284 if (upperCased.startsWith(BANG + GENE.toUpperCase())
285 || upperCased.startsWith(BANG + DOMAIN.toUpperCase()))
287 parseGeneOrDomain(dataLine);
289 else if (upperCased.startsWith(BANG + LABEL.toUpperCase()))
291 parseLabel(dataLine);
295 currentSequenceId = parseDataLine(dataLine);
298 else if (!seqData.isEmpty())
301 * Blank line after processing some data...
305 dataLine = nextNonCommentLine();
309 * close off any features currently being parsed
311 createFeature(GENE, currentGene, geneStart);
312 createFeature(DOMAIN, currentDomain, domainStart);
314 extendAnnotation(geneAnnotations, currentGene, currentGeneStartCol);
315 extendAnnotation(domainAnnotations, currentDomain,
316 currentDomainStartCol);
318 // remember the (longest) line length read in, so we can output the same
319 setAlignmentProperty(PROP_LINELENGTH, String.valueOf(positionsPerLine));
321 deriveSequencesAndFeatures();
327 * Create AlignmentAnnotation for Label, Gene and Domain (provided at least
328 * one non-null annotation is present)
330 protected void deriveAnnotations()
332 deriveAnnotation(this.labelAnnotations, MEGA_ANNOTATION_LABEL);
333 deriveAnnotation(this.geneAnnotations, MEGA_ANNOTATION_GENE);
334 deriveAnnotation(this.domainAnnotations, MEGA_ANNOTATION_DOMAIN);
338 * Create and ad an AlignmentAnnotation (provided at least one non-null
339 * annotation is present)
344 protected void deriveAnnotation(List<Annotation> anns, String label)
346 if (anns.size() > 0 && hasNonNullEntry(anns))
348 Annotation[] annotationArray = anns.toArray(new Annotation[anns
350 AlignmentAnnotation aa = new AlignmentAnnotation(label, "",
352 this.annotations.add(aa);
356 protected static boolean hasNonNullEntry(List<? extends Object> l)
369 * Parse a !Label line. This contains a single character per position (column)
370 * of the alignment block above. An underscore character represents no label.
371 * Labels are assembled into an AlignmentAnnotation object.
374 * @return true if any non-null annotation was created
375 * @throws FileFormatException
377 protected boolean parseLabel(String dataLine) throws FileFormatException
379 // strip off leading !Label and following spaces
380 dataLine = dataLine.substring(LABEL.length() + 1).trim();
382 // remove internal spacing and any leading tab
383 String labels = dataLine.replace(SPACE, "");
384 if (labels.endsWith(SEMICOLON))
386 labels = labels.substring(0, labels.length() - 1);
390 System.err.println("Warning: '" + dataLine
391 + "' should end with semi-colon");
393 boolean added = false;
394 for (char c : labels.toCharArray())
398 this.labelAnnotations.add(null);
402 this.labelAnnotations.add(new Annotation(String.valueOf(c), "",
411 * Post-processing after reading one block of interleaved data
413 protected void endOfDataBlock()
415 this.firstDataBlockRead = true;
417 padAnnotations(labelAnnotations);
421 * Append null annotations to keep the annotations list the same length as the
422 * sequences. This ensures that when the list is converted to an array it is
423 * correctly aligned with the alignment columns. It is needed when there are
424 * gaps in declared 'annotations' in a MEGA file, such as lines with no !Label
425 * statement, or regions between marked genes or domains.
429 protected void padAnnotations(List<Annotation> anns)
431 addNullAnnotations(anns, getAlignmentWidth());
435 * Append null annotations for positions up to (and excluding) the given end
441 protected void addNullAnnotations(List<Annotation> anns, int upTo)
443 int annotationsToAdd = upTo - anns.size();
444 for (int i = 0; i < annotationsToAdd; i++)
451 * Parse a !Gene or !Domain command line. MEGA accepts
453 * <li>!Gene=name;</li>
454 * <li>!Gene=name Property=Coding/Noncoding CodonStart=1/2/3;</li>
455 * <li>!Gene=genename Domain=domainname Property= etc</li>
456 * <li>!Domain=domainname Gene=genename Property= etc</li>
457 * <li>!Domain=domainname Property= etc</li>
458 * <li>!domain=domainname property=domainend</li>
460 * Properly, a Gene should be composed of Domain segments, but MEGA accepts
461 * without. Note that keywords don't seem to be case sensitive.
464 * @throws FileFormatException
466 protected void parseGeneOrDomain(String dataLine)
467 throws FileFormatException
469 String domain = null;
471 String property = null;
472 String codonStart = null;
473 String errorMsg = "Unrecognized format: " + dataLine;
475 if (!dataLine.startsWith(BANG) || !dataLine.endsWith(SEMICOLON))
477 throw new FileFormatException(errorMsg);
479 String trimmed = dataLine.substring(1, dataLine.length() - 1).trim();
480 String[] tokens = trimmed.split(WHITESPACE);
481 for (String token : tokens)
483 String[] keyValue = token.split("=");
484 if (keyValue.length != 2)
486 throw new FileFormatException(errorMsg);
488 String key = keyValue[0];
489 if (GENE.equalsIgnoreCase(key))
493 else if (DOMAIN.equalsIgnoreCase(key))
495 domain = keyValue[1];
497 else if (PROPERTY.equalsIgnoreCase(key))
499 property = keyValue[1];
501 else if (CODONSTART.equalsIgnoreCase(key))
503 codonStart = keyValue[1];
507 System.err.println("Unrecognised token: '" + key + "; in "
512 processGeneOrDomain(gene, domain, property, codonStart);
516 * Process a statement containing one or both of Gene and Domain, and
517 * optionally Property or CodonStart commands.
520 * the Gene name if specified, else null
522 * the Domain name if specified, else null
524 * the Property value if specified, else null
526 * the CodonStart value if specified, else null
528 protected void processGeneOrDomain(String gene, String domain,
529 String property, String codonStart)
532 * the order of processing below ensures that we correctly handle a domain
533 * in the context of an enclosing gene
535 processDomainEnd(domain, gene, property);
537 processGeneEnd(gene);
539 processGeneStart(gene);
541 processDomainStart(domain, property);
543 // TODO save codonStart if we plan to involve it in 'translate as cDNA'
547 * If we have declared a domain, and it is not continuing, start a sequence
553 protected void processDomainStart(String domain, String property)
555 if (DOMAINEND.equalsIgnoreCase(property))
557 currentDomain = null;
561 if (domain != null && !domain.equals(currentDomain))
563 String verboseDomain = makeVerboseDomainName(domain, property);
564 startSequenceFeature(domainStart);
565 currentDomainStartCol = getAlignmentWidth();
567 currentDomain = verboseDomain;
572 * Returns the width of alignment parsed so far. Note we assume (as does MEGA)
573 * that all sequences are the same length, so we can just take the length of
578 protected int getAlignmentWidth()
580 return seqData.isEmpty() ? 0 : seqData.values()
581 .iterator().next().length();
585 * If we have declared a gene, and it is not continuing, start a sequence
590 protected void processGeneStart(String gene)
592 if (gene != null && !gene.equals(currentGene))
594 startSequenceFeature(geneStart);
595 currentGeneStartCol = getAlignmentWidth();
601 * If we have been processing a domain, and it is not being continued, then
602 * make a sequence feature for the domain just ended. Criteria for the domain
603 * not being continued are either an explicit new domain or gene name, or a
604 * 'Property=domainend' statement
609 * @return true if a feature is created, else false
611 protected boolean processDomainEnd(String domain, String gene,
614 boolean newGene = (gene != null && !gene.equals(currentGene));
616 String verboseDomain = makeVerboseDomainName(domain, property);
618 if (this.currentDomain != null)
620 boolean newDomain = !this.currentDomain.equals(verboseDomain);
621 boolean domainEnded = DOMAINEND.equalsIgnoreCase(property);
622 if (newDomain || newGene || domainEnded)
624 createFeature(DOMAIN, currentDomain, domainStart);
625 // and/or... create annnotations for domain
626 extendAnnotation(domainAnnotations, currentDomain,
627 currentDomainStartCol);
628 currentDomain = null;
629 currentDomainStartCol = -1;
637 * If we have been processing a gene, and it is not being continued, then make
638 * a sequence feature for the gene just ended
641 * @return true if a feature is created, else false
643 protected boolean processGeneEnd(String gene)
645 boolean created = false;
647 * If we were processing a gene and now have either another, or none, create
648 * a sequence feature for that gene
650 if (this.currentGene != null && !this.currentGene.equals(gene))
652 createFeature(GENE, currentGene, geneStart);
653 // and/or... add annotations for Gene
654 extendAnnotation(geneAnnotations, currentGene, currentGeneStartCol);
656 currentGeneStartCol = -1;
664 * Helper method to add Annotation elements, with the given description and
665 * starting at the given start column, up to the end of the sequence length
666 * parsed so far. Null elements are inserted for any skipped columns since the
667 * last annotation (if any), i.e. positions with no annotation of this type.
672 * the start column of the annotations to add, or -1 if nothing to
675 protected void extendAnnotation(List<Annotation> anns,
676 String description, int startColumn)
678 int alignmentWidth = getAlignmentWidth();
679 addNullAnnotations(anns, startColumn == -1 ? alignmentWidth
682 int numberToAdd = alignmentWidth - anns.size();
685 Color col = description == null ? Color.black : UserColourScheme
686 .createColourFromName(description);
687 for (int i = 0; i < numberToAdd; i++)
689 anns.add(new Annotation("X", description, ' ', 0f, col));
695 * Makes an expanded descriptive name for Domain if possible e.g.
696 * "Intron1 (Adh Coding)". Currently incorporates the current gene name (if
697 * any) and the Coding/Noncoding property value (if given).
703 protected String makeVerboseDomainName(String domain, String property)
705 String verboseDomain = domain;
709 if ("Exon".equalsIgnoreCase(property)
710 || "Coding".equalsIgnoreCase(property))
714 else if ("Intron".equalsIgnoreCase(property)
715 || "Noncoding".equalsIgnoreCase(property))
717 coding = " Noncoding";
719 verboseDomain = domain
720 + (currentGene == null ? "" : " (" + currentGene + coding
723 return verboseDomain;
727 * Start processing a new feature
729 * @param startPositions
731 protected void startSequenceFeature(Map<String, Integer> startPositions)
734 * If the feature declaration precedes all sequences, we will know in
735 * createFeature that it started with residue 1; otherwise note now where it
736 * starts in each sequence
738 if (!residuesRead.isEmpty())
740 for (Entry<String, Integer> entry : residuesRead.entrySet())
742 String seqId = entry.getKey();
743 Integer nextResidue = entry.getValue() + 1;
744 startPositions.put(seqId, nextResidue);
750 * Add a SequenceFeature to each sequence, using the given start/end values
754 * @param featureValue
755 * @param featureStartResidues
757 protected void createFeature(String featureType, String featureValue,
758 Map<String, Integer> featureStartResidues)
760 if (featureValue == null)
765 Iterator<String> seqids = this.seqData.keySet().iterator();
766 while (seqids.hasNext())
768 String seqid = seqids.next();
769 Integer startAt = featureStartResidues.get(seqid);
770 int sfstart = startAt == null ? 1 : startAt.intValue();
771 int sfend = residuesRead.get(seqid);
772 if (sfend >= sfstart)
775 * don't add feature if entirely gapped in the sequence
777 // TODO: type="Gene" (but then all coloured the same) or
779 SequenceFeature sf = new SequenceFeature(featureValue, featureType,
780 sfstart, sfend, 0f, null);
781 sequenceFeatures.get(seqid).add(sf);
787 * Returns the next line that is not a comment, or null at end of file.
788 * Comments in MEGA are within [ ] brackets, and may be nested.
791 * @throws FileFormatException
793 protected String nextNonCommentLine() throws FileFormatException
795 return nextNonCommentLine(0);
799 * Returns the next non-comment line (or part line), or null at end of file.
800 * Comments in MEGA are within [ ] brackets, and may be nested. They may occur
801 * anywhere within a line (for example at the end with position numbers); this
802 * method returns the line with any comments removed.
805 * current depth of nesting of comments while parsing
807 * @throws FileFormatException
809 protected String nextNonCommentLine(final int depth)
810 throws FileFormatException
816 } catch (IOException e)
818 throw new FileFormatException(e.getMessage());
824 System.err.println("Warning: unterminated comment in data file");
830 * If we are in a (possibly nested) comment after parsing this line, keep
831 * reading recursively until the comment has unwound
833 int newDepth = commentDepth(data, depth);
836 return nextNonCommentLine(newDepth);
841 * not in a comment by end of this line; return what is left
843 String nonCommentPart = getNonCommentContent(data, depth);
844 return nonCommentPart;
849 * Returns what is left of the input data after removing any comments, whether
850 * 'in progress' from preceding lines, or embedded in the current line
855 * nested depth of comments pending termination
857 * @throws FileFormatException
859 protected static String getNonCommentContent(String data, int depth)
860 throws FileFormatException
862 int len = data.length();
863 StringBuilder result = new StringBuilder(len);
864 for (int i = 0; i < len; i++)
866 char c = data.charAt(i);
891 return result.toString();
895 * Calculates new depth of comment after parsing an input line i.e. the excess
896 * of opening '[' over closing ']' characters. Any excess ']' are ignored (not
897 * treated as comment delimiters).
902 * current comment nested depth before parsing the line
903 * @return new depth after parsing the line
905 protected static int commentDepth(CharSequence data, int depth)
907 int newDepth = depth;
908 int len = data.length();
909 for (int i = 0; i < len; i++)
911 char c = data.charAt(i);
912 if (c == COMMENT_START)
916 else if (c == COMMENT_END && newDepth > 0)
925 * Convert the parsed sequence strings to objects and store them in the model.
927 protected void deriveSequencesAndFeatures()
929 Set<Entry<String, StringBuilder>> datasets = seqData.entrySet();
931 for (Entry<String, StringBuilder> dataset : datasets)
933 String sequenceId = dataset.getKey();
934 StringBuilder characters = dataset.getValue();
935 SequenceI s = new Sequence(sequenceId, new String(characters));
936 this.seqs.addElement(s);
939 * and add any derived sequence features to the sequence
941 for (SequenceFeature sf : sequenceFeatures.get(sequenceId))
943 s.addSequenceFeature(sf);
949 * Process one line of sequence data. If it has no sequence identifier, append
950 * to the current id's sequence. Else parse out the sequence id and append the
951 * data (if any) to that id's sequence. Returns the sequence id (implicit or
952 * explicit) for this line.
956 * @throws FileFormatException
958 protected String parseDataLine(String dataLine)
959 throws FileFormatException
961 String seqId = getSequenceId(dataLine);
965 * Just character data
967 parseNoninterleavedDataLine(dataLine);
968 return currentSequenceId;
970 else if ((HASHSIGN + seqId).trim().equals(dataLine.trim()))
973 * Sequence id only - header line for noninterleaved data
980 * Sequence id followed by data
982 parseInterleavedDataLine(dataLine, seqId);
988 * Add a line of sequence data to the buffer for the given sequence id. Start
989 * a new one if we haven't seen it before.
992 * @throws FileFormatException
994 protected void parseNoninterleavedDataLine(String dataLine)
995 throws FileFormatException
997 if (currentSequenceId == null)
1000 * Oops. Data but no sequence id context.
1002 throw new FileFormatException("No sequence id context at: "
1006 assertInterleaved(false, dataLine);
1008 dataLine = addSequenceData(currentSequenceId, dataLine);
1010 setPositionsPerLine(Math.max(positionsPerLine, dataLine.length()));
1014 * Get the sequence data for this sequence id, starting a new one if
1020 protected StringBuilder getSequenceDataBuffer(String currentId)
1022 StringBuilder sb = seqData.get(currentId);
1025 // first data met for this sequence id, start a new buffer
1026 sb = new StringBuilder(SEQBUFFERSIZE);
1027 seqData.put(currentId, sb);
1029 // and a placeholder for any SequenceFeature found
1030 sequenceFeatures.put(currentId, new ArrayList<SequenceFeature>());
1036 * Parse one line of interleaved data e.g.
1039 * #TheSeqId CGATCGCATGCA
1044 * @throws FileFormatException
1046 protected void parseInterleavedDataLine(String dataLine, String seqId)
1047 throws FileFormatException
1050 * New sequence found in second or later data block - error.
1052 if (this.firstDataBlockRead && !seqData.containsKey(seqId))
1054 throw new FileFormatException(
1055 "Parse error: misplaced new sequence starting at " + dataLine);
1058 String data = dataLine.substring(seqId.length() + 1).trim();
1061 * Do nothing if this line is _only_ a sequence id with no data following.
1063 if (data != null && data.length() > 0)
1065 data = addSequenceData(seqId, data);
1066 setPositionsPerLine(Math.max(positionsPerLine, data.length()));
1067 assertInterleaved(true, dataLine);
1072 * Remove spaces, and replace identity symbol, before appending the sequence
1073 * data to the buffer for the sequence id. Returns the reformatted added data.
1074 * Also updates a count of residues read for the sequence.
1080 protected String addSequenceData(String seqId, String data)
1082 StringBuilder sb = getSequenceDataBuffer(seqId);
1083 int len = sb.length();
1084 String formatted = data.replace(SPACE, "");
1087 * If sequence contains '.' or other identity symbol; replace these with the
1088 * same position from the first (reference) sequence
1091 StringBuilder referenceSequence = seqData.values().iterator().next();
1092 StringBuilder sb1 = new StringBuilder(formatted.length());
1093 for (int i = 0; i < formatted.length(); i++)
1095 char nextChar = formatted.charAt(i);
1096 if (nextChar == gapCharacter)
1098 sb1.append(Comparison.isGap(nextChar) ? nextChar : DEFAULT_GAP);
1103 if (nextChar == identityCharacter
1104 && len + i < referenceSequence.length())
1106 sb1.append(referenceSequence.charAt(len + i));
1110 sb1.append(nextChar);
1114 formatted = sb1.toString();
1120 * increment residue count for the sequence
1124 Integer residueCount = residuesRead.get(seqId);
1125 residuesRead.put(seqId, nonGapped
1126 + (residueCount == null ? 0 : residueCount));
1133 * If the line begins with (e.g.) "#abcde " then returns "abcde" as the
1134 * identifier. Else returns null.
1139 public static String getSequenceId(String dataLine)
1141 // TODO refactor to a StringUtils type class
1142 if (dataLine != null)
1144 if (dataLine.startsWith(HASHSIGN))
1146 int spacePos = dataLine.indexOf(" ");
1147 return (spacePos == -1 ? dataLine.substring(1) : dataLine
1148 .substring(1, spacePos));
1155 * Read the #MEGA and Title/Format/Description header lines (if present).
1157 * Save as alignment properties in case useful.
1159 * @return the next non-blank line following the header lines.
1160 * @throws FileFormatException
1162 protected String parseHeaderLines() throws FileFormatException
1164 String inputLine = null;
1165 while ((inputLine = nextNonCommentLine()) != null)
1167 inputLine = inputLine.trim();
1172 if (inputLine.length() == 0)
1177 if (inputLine.toUpperCase().startsWith(MEGA_ID))
1182 if (isTitle(inputLine))
1184 this.title = getValue(inputLine);
1185 setAlignmentProperty(PROP_TITLE, title);
1187 else if (inputLine.startsWith(BANG + DESCRIPTION))
1189 parseDescription(inputLine);
1192 else if (inputLine.startsWith(BANG + FORMAT))
1194 parseFormat(inputLine);
1196 else if (!inputLine.toUpperCase().startsWith(MEGA_ID))
1200 * Return the first 'data line' i.e. one that is not blank, #MEGA or
1210 * Parse a !Format statement. This may be multiline, and is ended by a
1214 * @throws FileFormatException
1216 protected void parseFormat(String inputLine) throws FileFormatException
1218 while (inputLine != null)
1220 parseFormatLine(inputLine);
1221 if (inputLine.endsWith(SEMICOLON))
1225 inputLine = nextNonCommentLine();
1230 * Parse one line of a !Format statement. This may contain one or more
1231 * keyword=value pairs.
1234 * @throws FileFormatException
1236 protected void parseFormatLine(String inputLine)
1237 throws FileFormatException
1239 if (inputLine.startsWith(BANG + FORMAT))
1241 inputLine = inputLine.substring((BANG + FORMAT).length());
1243 if (inputLine.endsWith(SEMICOLON))
1245 inputLine = inputLine.substring(0, inputLine.length() - 1);
1247 if (inputLine.length() == 0)
1251 String[] tokens = inputLine.trim().split(WHITESPACE);
1252 for (String token : tokens)
1254 parseFormatKeyword(token);
1259 * Parse a Keyword=Value token. Possible keywords are
1261 * <li>DataType= DNA, RNA, Nucleotide, Protein</li>
1262 * <li>DataFormat= Interleaved, ?</li>
1263 * <li>NSeqs= number of sequences (synonym NTaxa)</li>
1264 * <li>NSites= number of bases / residues</li>
1265 * <li>Property= Exon (or Coding), Intron (or Noncoding), End (of domain)</li>
1266 * <li>Indel= gap character</li>
1267 * <li>Identical= identity character (to first sequence) (synonym MatchChar)</li>
1268 * <li>Missing= missing data character</li>
1269 * <li>CodeTable= Standard, other (MEGA supports various)</li>
1273 * @throws FileFormatException
1274 * if an unrecognised keyword or value is encountered
1276 protected void parseFormatKeyword(String token)
1277 throws FileFormatException
1279 String msg = "Unrecognised Format command: " + token;
1280 String[] bits = token.split(EQUALS);
1281 if (bits.length != 2)
1283 throw new FileFormatException(msg);
1285 String keyword = bits[0];
1286 String value = bits[1];
1289 * Jalview will work out whether nucleotide or not anyway
1291 if (keyword.equalsIgnoreCase(DATATYPE))
1293 if (value.equalsIgnoreCase("DNA") || value.equalsIgnoreCase("RNA")
1294 || value.equalsIgnoreCase("Nucleotide"))
1296 this.nucleotide = true;
1297 // alignment computes whether or not it is nucleotide when created
1299 else if (value.equalsIgnoreCase(PROTEIN))
1301 this.nucleotide = false;
1305 throw new FileFormatException(msg);
1307 setAlignmentProperty(PROP_DATATYPE, value);
1311 * accept non-Standard code table but save in case we want to disable
1312 * 'translate as cDNA'
1314 else if (keyword.equalsIgnoreCase(CODETABLE))
1316 setAlignmentProperty(PROP_CODETABLE, value);
1320 * save gap char to set later on alignment once created
1322 else if (keyword.equalsIgnoreCase(INDEL))
1324 this.gapCharacter = value.charAt(0);
1325 if (!Comparison.isGap(gapCharacter))
1327 System.err.println("Jalview doesn't support '" + gapCharacter
1328 + "' for gaps, will be converted to '" + DEFAULT_GAP + "'");
1332 else if (keyword.equalsIgnoreCase(IDENTICAL)
1333 || keyword.equalsIgnoreCase("MatchChar"))
1335 setAlignmentProperty(PROP_IDENTITY, value);
1336 this.identityCharacter = value.charAt(0);
1337 if (!".".equals(value))
1339 System.err.println("Warning: " + token
1340 + " not supported, Jalview uses '.' for identity");
1344 else if (keyword.equalsIgnoreCase(MISSING))
1346 setAlignmentProperty(PROP_MISSING, value);
1347 System.err.println("Warning: " + token + " not supported");
1350 else if (keyword.equalsIgnoreCase(PROPERTY))
1352 // TODO: can Property appear in a Format command?
1353 // suspect this is a mistake in the manual
1356 else if (!keyword.equalsIgnoreCase(N_SEQS)
1357 && !keyword.equalsIgnoreCase("NTaxa")
1358 && !keyword.equalsIgnoreCase(N_SITES))
1360 System.err.println("Warning: " + msg);
1365 * Returns the trimmed data on the line following either whitespace or '=',
1366 * with any trailing semi-colon removed<br>
1369 * <li>Hello World</li>
1370 * <li>!Hello: \tWorld;</li>
1371 * <li>!Hello=World</li>
1373 * should all return "World"
1378 protected static String getValue(String inputLine)
1380 if (inputLine == null)
1384 String value = null;
1385 String s = inputLine.replaceAll("\t", " ").trim();
1388 * KEYWORD = VALUE should return VALUE
1390 int equalsPos = s.indexOf("=");
1393 value = s.substring(equalsPos + 1);
1397 int spacePos = s.indexOf(' ');
1398 value = spacePos == -1 ? "" : s.substring(spacePos + 1);
1400 value = value.trim();
1401 if (value.endsWith(SEMICOLON))
1403 value = value.substring(0, value.length() - 1).trim();
1409 * Returns true if the input line starts with "TITLE" or "!TITLE" (not case
1410 * sensitive). The latter is the official format, some older data file
1411 * examples have it without the !.
1416 protected static boolean isTitle(String inputLine)
1418 if (inputLine == null)
1422 String upper = inputLine.toUpperCase();
1423 return (upper.startsWith(TITLE.toUpperCase()) || upper.startsWith(BANG
1424 + TITLE.toUpperCase()));
1428 * Reads lines until terminated by semicolon, appending each to the
1429 * Description property value.
1431 * @throws FileFormatException
1433 protected void parseDescription(String firstDescriptionLine)
1434 throws FileFormatException
1436 StringBuilder desc = new StringBuilder(256);
1437 desc.append(getValue(firstDescriptionLine));
1438 if (!firstDescriptionLine.endsWith(SEMICOLON))
1440 String line = nextNonCommentLine();
1441 while (line != null)
1443 if (line.endsWith(SEMICOLON))
1445 desc.append(line.substring(0, line.length() - 1));
1448 else if (line.length() > 0)
1450 desc.append(line).append(newline);
1452 line = nextNonCommentLine();
1455 setAlignmentProperty(PROP_DESCRIPTION, desc.toString());
1459 * Returns the alignment sequences in Mega format.
1462 public String print()
1464 return MEGA_ID + newline + print(getSeqsAsArray());
1468 * Write out the alignment sequences in Mega format - interleaved unless
1469 * explicitly noninterleaved.
1471 protected String print(SequenceI[] s)
1474 if (this.interleaved != null && !this.interleaved)
1476 result = printNonInterleaved(s);
1480 result = printInterleaved(s);
1486 * Print to string in Interleaved format - blocks of next N characters of each
1491 protected String printInterleaved(SequenceI[] s)
1493 int maxIdLength = getMaxIdLength(s);
1494 int maxSequenceLength = getMaxSequenceLength(s);
1495 int numLines = maxSequenceLength / positionsPerLine + 3; // approx
1497 int numDataBlocks = (maxSequenceLength - 1) / positionsPerLine + 1;
1498 int spaceEvery = this.nucleotide != null && this.nucleotide ? 3 : 10;
1499 int chunksPerLine = (positionsPerLine + spaceEvery - 1) / spaceEvery;
1502 * Roughly size a buffer to hold the whole output
1504 StringBuilder sb = new StringBuilder(numLines
1505 * (maxIdLength + positionsPerLine + chunksPerLine + 10));
1508 * Output as: #Seqid CGT AGC ACT ... or blocks of 10 for peptide
1511 for (int i = 0; i < numDataBlocks; i++)
1514 boolean first = true;
1516 for (SequenceI seq : s)
1519 String seqId = String.format("#%-" + maxIdLength + "s",
1523 * output next line for this sequence
1526 int lastPos = seqFrom + positionsPerLine; // exclusive
1527 for (int j = 0; j < chunksPerLine; j++)
1529 char[] subSequence = seq.getSequence(seqFrom,
1530 Math.min(lastPos, seqFrom + spaceEvery));
1531 if (subSequence.length > 0)
1533 sb.append(SPACE).append(subSequence);
1535 seqFrom += subSequence.length;
1538 // all sequences should be the same length in MEGA
1539 advancedBy += subSequence.length;
1542 // write last position as a comment
1543 if (writePositionNumbers)
1545 sb.append(SPACE).append(COMMENT_START).append(from + advancedBy)
1546 .append(COMMENT_END);
1551 sb.append(printLabel(from, advancedBy, maxIdLength));
1555 return new String(sb);
1559 * Outputs to string the MEGA header and any other known and relevant
1560 * alignment properties
1564 protected String printHeaders(AlignmentI al)
1566 StringBuilder sb = new StringBuilder(128);
1567 sb.append(MEGA_ID).append(newline);
1568 String propertyValue = (String) al.getProperty(PROP_TITLE);
1569 if (propertyValue != null)
1571 sb.append(BANG).append(TITLE).append(SPACE).append(propertyValue)
1572 .append(SEMICOLON).append(newline);
1574 propertyValue = (String) al.getProperty(PROP_DESCRIPTION);
1575 if (propertyValue != null)
1577 sb.append(BANG).append(DESCRIPTION).append(newline)
1578 .append(propertyValue).append(SEMICOLON)
1583 * !Format DataType CodeTable
1585 sb.append(BANG).append(FORMAT).append(newline);
1586 String dataType = (String) al.getProperty(PROP_DATATYPE);
1587 if (dataType == null)
1589 dataType = al.isNucleotide() ? NUCLEOTIDE : PROTEIN;
1591 sb.append(INDENT).append(DATATYPE).append(EQUALS).append(dataType);
1592 String codeTable = (String) al.getProperty(PROP_CODETABLE);
1593 sb.append(SPACE).append(CODETABLE).append(EQUALS)
1594 .append(codeTable == null ? "Standard" : codeTable)
1598 * !Format NSeqs NSites (the length of sequences - they should all be the
1599 * same - including gaps)
1601 sb.append(INDENT).append(N_SEQS).append(EQUALS).append(al.getHeight());
1602 sb.append(SPACE).append(N_SITES).append(EQUALS)
1603 .append(String.valueOf(al.getWidth()));
1607 * !Format Indel Identical Missing
1610 sb.append(INDEL).append(EQUALS).append(al.getGapCharacter());
1611 String identity = (String) al.getProperty(PROP_IDENTITY);
1612 if (identity != null)
1614 sb.append(SPACE).append(IDENTICAL).append(EQUALS).append(identity);
1616 String missing = (String) al.getProperty(PROP_MISSING);
1617 if (missing != null)
1619 sb.append(SPACE).append(MISSING).append(EQUALS).append(missing);
1621 sb.append(SEMICOLON).append(newline);
1623 return sb.toString();
1627 * Get the longest sequence id (to allow aligned printout).
1632 protected static int getMaxIdLength(SequenceI[] s)
1634 // TODO pull up for reuse
1636 for (SequenceI seq : s)
1638 int len = seq.getName().length();
1639 if (len > maxLength)
1648 * Get the longest sequence length
1653 protected static int getMaxSequenceLength(SequenceI[] s)
1655 // TODO pull up for reuse
1657 for (SequenceI seq : s)
1659 int len = seq.getLength();
1660 if (len > maxLength)
1669 * Print to string in noninterleaved format - all of each sequence in turn, in
1670 * blocks of 50 characters.
1675 protected String printNonInterleaved(SequenceI[] s)
1677 int maxSequenceLength = getMaxSequenceLength(s);
1679 int numLines = maxSequenceLength / positionsPerLine + 2 + s.length;
1682 * Roughly size a buffer to hold the whole output
1684 StringBuilder sb = new StringBuilder(numLines * positionsPerLine);
1686 for (SequenceI seq : s)
1688 printSequence(sb, seq);
1691 return new String(sb);
1695 * Append a formatted complete sequence to the string buffer
1700 protected void printSequence(StringBuilder sb, SequenceI seq)
1702 int spaceEvery = this.nucleotide != null && this.nucleotide ? 3 : 10;
1703 // round down to output a whole number of spaced blocks
1704 int chunksPerLine = positionsPerLine / spaceEvery;
1707 sb.append(HASHSIGN + seq.getName()).append(newline);
1709 while (startPos < seq.getLength())
1712 * print next line for this sequence
1714 boolean firstChunk = true;
1715 int lastPos = startPos + positionsPerLine; // exclusive
1716 for (int j = 0; j < chunksPerLine; j++)
1718 char[] subSequence = seq.getSequence(startPos,
1719 Math.min(lastPos, startPos + spaceEvery));
1720 if (subSequence.length > 0)
1726 sb.append(subSequence);
1729 startPos += subSequence.length;
1731 // line end position (base 1) as a comment
1732 sb.append(SPACE).append(COMMENT_START).append(startPos)
1733 .append(COMMENT_END);
1739 * Returns a formatted string like <br>
1740 * !Label aa_b_ ab_b_ <br>
1741 * where underscore represents no annotation, any other character a MEGA label
1743 * Returns an empty string if there is no non-null annotation in the given
1747 * start column of the alignment (base 0)
1749 * number of positions to output
1751 * padded width of !Label statement to output
1754 protected String printLabel(int fromPos, int positions, int labelWidth)
1756 int spaceEvery = this.nucleotide != null && this.nucleotide ? 3 : 10;
1758 AlignmentAnnotation ann = findAnnotation(MEGA_ANNOTATION_LABEL);
1764 StringBuilder sb = new StringBuilder(positions + 20);
1765 sb.append(String.format("%-" + labelWidth + "s ", BANG + LABEL));
1766 Annotation[] anns = annotations.get(0).annotations;
1767 int blockCharCount = 0;
1768 boolean annotationFound = false;
1770 for (int i = fromPos; i < fromPos + positions; i++)
1772 String label = String.valueOf(UNDERSCORE);
1773 if (i < anns.length && anns[i] != null)
1775 label = anns[i].displayCharacter;
1778 if (label.charAt(0) != UNDERSCORE)
1780 annotationFound = true;
1782 // add a space after each block except the last
1783 if (++blockCharCount % spaceEvery == 0
1784 && (i < fromPos + positions - 1))
1789 sb.append(SEMICOLON).append(newline);
1791 return annotationFound ? sb.toString() : none;
1795 * Returns the first stored annotation found with the given label, or null
1797 * @param annotationLabel
1800 protected AlignmentAnnotation findAnnotation(String annotationLabel)
1802 if (annotations != null)
1804 for (AlignmentAnnotation ann : annotations)
1806 if (annotationLabel.equals(ann.label))
1816 * Flag this file as interleaved or not, based on data format. Throws an
1817 * exception if has previously been determined to be otherwise.
1821 * @throws FileFormatException
1823 protected void assertInterleaved(boolean isIt, String dataLine)
1824 throws FileFormatException
1826 if (this.interleaved != null && isIt != this.interleaved.booleanValue())
1828 throw new FileFormatException("Parse error: interleaved was " + !isIt
1829 + " but now seems to be " + isIt + ", at line: " + dataLine);
1831 this.interleaved = new Boolean(isIt);
1832 setAlignmentProperty(PROP_INTERLEAVED, interleaved.toString());
1835 public boolean isInterleaved()
1837 return this.interleaved == null ? false : this.interleaved
1842 * Adds saved parsed values either as alignment properties, or (in some cases)
1843 * as specific member fields of the alignment
1846 public void addProperties(AlignmentI al)
1848 super.addProperties(al);
1851 * record gap character specified, but convert to '-' if not one we support
1853 al.setGapCharacter(Comparison.isGap(gapCharacter) ? gapCharacter
1857 * warn if e.g. DataType=DNA but data is protein (or vice versa)
1859 if (this.nucleotide != null && this.nucleotide != al.isNucleotide()) {
1860 System.err.println("Warning: " + this.title + " declared "
1861 + (nucleotide ? "" : " not ") + "nucleotide but it is"
1862 + (nucleotide ? " not" : ""));
1867 * Print the given alignment in MEGA format. If the alignment was created by
1868 * parsing a MEGA file, it should have properties set (e.g. Title) which can
1869 * surface in the output.
1872 public String print(AlignmentI al)
1874 this.nucleotide = al.isNucleotide();
1877 * if the alignment has a "MEGA" annotation, we'll output its values as
1878 * !Label statements; MEGA only supports one of these
1880 AlignmentAnnotation[] anns = al.getAlignmentAnnotation();
1883 for (AlignmentAnnotation ann : anns)
1885 if (MEGA_ANNOTATION_LABEL.equals(ann.label))
1887 this.annotations = new Vector<AlignmentAnnotation>();
1888 annotations.add(ann);
1894 String lineLength = (String) al.getProperty(PROP_LINELENGTH);
1895 this.positionsPerLine = lineLength == null ? DEFAULT_LINE_LENGTH : Integer
1896 .parseInt(lineLength);
1899 * round down to a multiple of 3 positions per line for nucleotide
1903 positionsPerLine = positionsPerLine - (positionsPerLine % 3);
1906 String interleave = (String) al.getProperty(PROP_INTERLEAVED);
1907 if (interleave != null)
1909 this.interleaved = Boolean.valueOf(interleave);
1912 String headers = printHeaders(al);
1913 return headers + print(al.getSequencesArray());
1917 * Returns the number of sequence positions output per line
1921 public int getPositionsPerLine()
1923 return positionsPerLine;
1927 * Sets the number of sequence positions output per line. Note these will be
1928 * formatted in blocks of 3 (nucleotide) or 10 (peptide).
1932 public void setPositionsPerLine(int p)
1934 this.positionsPerLine = p;