2 * @(#)SequenceUtil.java 1.0 September 2009 Copyright (c) 2009 Peter Troshin
\r
3 * Jalview Web Services version: 2.0 This library is free software; you can
\r
4 * redistribute it and/or modify it under the terms of the Apache License
\r
5 * version 2 as published by the Apache Software Foundation This library is
\r
6 * distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
\r
7 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
\r
8 * PARTICULAR PURPOSE. See the Apache License for more details. A copy of the
\r
9 * license is in apache_license.txt. It is also available here: see:
\r
10 * http://www.apache.org/licenses/LICENSE-2.0.txt Any republication or derived
\r
11 * work distributed in source code form must include this copyright and license
\r
15 package compbio.data.sequence;
\r
17 import java.io.BufferedReader;
\r
18 import java.io.BufferedWriter;
\r
19 import java.io.Closeable;
\r
20 import java.io.File;
\r
21 import java.io.FileInputStream;
\r
22 import java.io.IOException;
\r
23 import java.io.InputStream;
\r
24 import java.io.InputStreamReader;
\r
25 import java.io.OutputStream;
\r
26 import java.io.OutputStreamWriter;
\r
27 import java.util.ArrayList;
\r
28 import java.util.HashMap;
\r
29 import java.util.HashSet;
\r
30 import java.util.List;
\r
31 import java.util.Map;
\r
32 import java.util.Scanner;
\r
33 import java.util.SortedSet;
\r
34 import java.util.TreeSet;
\r
35 import java.util.logging.Level;
\r
36 import java.util.regex.Matcher;
\r
37 import java.util.regex.Pattern;
\r
39 import compbio.util.Util;
\r
42 * Utility class for operations on sequences
\r
44 * @author Petr Troshin
\r
47 public final class SequenceUtil {
\r
50 * A whitespace character: [\t\n\x0B\f\r]
\r
52 public static final Pattern WHITE_SPACE = Pattern.compile("\\s");
\r
57 public static final Pattern DIGIT = Pattern.compile("\\d");
\r
62 public static final Pattern NONWORD = Pattern.compile("\\W");
\r
67 public static final Pattern AA = Pattern.compile("[ARNDCQEGHILKMFPSTWYV]+",
\r
68 Pattern.CASE_INSENSITIVE);
\r
71 * inversion of AA pattern
\r
73 public static final Pattern NON_AA = Pattern.compile(
\r
74 "[^ARNDCQEGHILKMFPSTWYV]+", Pattern.CASE_INSENSITIVE);
\r
77 * Same as AA pattern but with two additional letters - XU
\r
79 public static final Pattern AMBIGUOUS_AA = Pattern.compile(
\r
80 "[ARNDCQEGHILKMFPSTWYVXU]+", Pattern.CASE_INSENSITIVE);
\r
83 * Nucleotides a, t, g, c, u
\r
85 public static final Pattern NUCLEOTIDE = Pattern.compile("[AGTCU]+",
\r
86 Pattern.CASE_INSENSITIVE);
\r
89 * Ambiguous nucleotide
\r
91 public static final Pattern AMBIGUOUS_NUCLEOTIDE = Pattern.compile(
\r
92 "[AGTCRYMKSWHBVDNU]+", Pattern.CASE_INSENSITIVE); // see IUPAC
\r
96 public static final Pattern NON_NUCLEOTIDE = Pattern.compile("[^AGTCU]+",
\r
97 Pattern.CASE_INSENSITIVE);
\r
99 private SequenceUtil() {
\r
100 } // utility class, no instantiation
\r
103 * public static void write_PirSeq(OutputStream os, FastaSequence seq)
\r
104 * throws IOException { BufferedWriter pir_out = new BufferedWriter(new
\r
105 * OutputStreamWriter(os)); pir_out.write(">P1;" + seq.getId() +
\r
106 * SysPrefs.newlinechar); pir_out.write(seq.getSequence() +
\r
107 * SysPrefs.newlinechar); pir_out.close(); } public static void
\r
108 * write_FastaSeq(OutputStream os, FastaSequence seq) throws IOException {
\r
109 * BufferedWriter fasta_out = new BufferedWriter( new
\r
110 * OutputStreamWriter(os)); fasta_out.write(">" + seq.getId() +
\r
111 * SysPrefs.newlinechar); fasta_out.write(seq.getSequence() +
\r
112 * SysPrefs.newlinechar); fasta_out.close(); }
\r
116 * @return true is the sequence contains only letters a,c, t, g, u
\r
118 public static boolean isNucleotideSequence(final FastaSequence s) {
\r
119 return SequenceUtil.isNonAmbNucleotideSequence(s.getSequence());
\r
123 * Ambiguous DNA chars : AGTCRYMKSWHBVDN // differs from protein in only one
\r
126 public static boolean isNonAmbNucleotideSequence(String sequence) {
\r
127 sequence = SequenceUtil.cleanSequence(sequence);
\r
128 if (SequenceUtil.DIGIT.matcher(sequence).find()) {
\r
131 if (SequenceUtil.NON_NUCLEOTIDE.matcher(sequence).find()) {
\r
134 * System.out.format("I found the text starting at " +
\r
135 * "index %d and ending at index %d.%n", nonDNAmatcher .start(),
\r
136 * nonDNAmatcher.end());
\r
139 final Matcher DNAmatcher = SequenceUtil.NUCLEOTIDE.matcher(sequence);
\r
140 return DNAmatcher.find();
\r
144 * Removes all whitespace chars in the sequence string
\r
147 * @return cleaned up sequence
\r
149 public static String cleanSequence(String sequence) {
\r
150 assert sequence != null;
\r
151 final Matcher m = SequenceUtil.WHITE_SPACE.matcher(sequence);
\r
152 sequence = m.replaceAll("").toUpperCase();
\r
157 * Removes all special characters and digits as well as whitespace chars
\r
158 * from the sequence
\r
161 * @return cleaned up sequence
\r
163 public static String deepCleanSequence(String sequence) {
\r
164 sequence = SequenceUtil.cleanSequence(sequence);
\r
165 sequence = SequenceUtil.DIGIT.matcher(sequence).replaceAll("");
\r
166 sequence = SequenceUtil.NONWORD.matcher(sequence).replaceAll("");
\r
167 final Pattern othernonSeqChars = Pattern.compile("[_-]+");
\r
168 sequence = othernonSeqChars.matcher(sequence).replaceAll("");
\r
174 * @return true is the sequence is a protein sequence, false overwise
\r
176 public static boolean isProteinSequence(String sequence) {
\r
177 sequence = SequenceUtil.cleanSequence(sequence);
\r
178 if (SequenceUtil.isNonAmbNucleotideSequence(sequence)) {
\r
181 if (SequenceUtil.DIGIT.matcher(sequence).find()) {
\r
184 if (SequenceUtil.NON_AA.matcher(sequence).find()) {
\r
187 final Matcher protmatcher = SequenceUtil.AA.matcher(sequence);
\r
188 return protmatcher.find();
\r
192 * Check whether the sequence confirms to amboguous protein sequence
\r
195 * @return return true only if the sequence if ambiguous protein sequence
\r
196 * Return false otherwise. e.g. if the sequence is non-ambiguous
\r
199 public static boolean isAmbiguosProtein(String sequence) {
\r
200 sequence = SequenceUtil.cleanSequence(sequence);
\r
201 if (SequenceUtil.isNonAmbNucleotideSequence(sequence)) {
\r
204 if (SequenceUtil.DIGIT.matcher(sequence).find()) {
\r
207 if (SequenceUtil.NON_AA.matcher(sequence).find()) {
\r
210 if (SequenceUtil.AA.matcher(sequence).find()) {
\r
213 final Matcher amb_prot = SequenceUtil.AMBIGUOUS_AA.matcher(sequence);
\r
214 return amb_prot.find();
\r
218 * Writes list of FastaSequeces into the outstream formatting the sequence
\r
219 * so that it contains width chars on each line
\r
224 * - the maximum number of characters to write in one line
\r
225 * @throws IOException
\r
227 public static void writeFasta(final OutputStream outstream,
\r
228 final List<FastaSequence> sequences, final int width)
\r
229 throws IOException {
\r
230 writeFastaKeepTheStream(outstream, sequences, width);
\r
234 public static void writeFastaKeepTheStream(final OutputStream outstream,
\r
235 final List<FastaSequence> sequences, final int width)
\r
236 throws IOException {
\r
237 final OutputStreamWriter writer = new OutputStreamWriter(outstream);
\r
238 final BufferedWriter fastawriter = new BufferedWriter(writer);
\r
239 for (final FastaSequence fs : sequences) {
\r
240 fastawriter.write(">" + fs.getId() + "\n");
\r
241 fastawriter.write(fs.getFormatedSequence(width));
\r
242 fastawriter.write("\n");
\r
244 fastawriter.flush();
\r
249 * Reads fasta sequences from inStream into the list of FastaSequence
\r
254 * @return list of FastaSequence objects
\r
255 * @throws IOException
\r
257 public static List<FastaSequence> readFasta(final InputStream inStream)
\r
258 throws IOException {
\r
259 final List<FastaSequence> seqs = new ArrayList<FastaSequence>();
\r
261 final BufferedReader infasta = new BufferedReader(
\r
262 new InputStreamReader(inStream, "UTF8"), 16000);
\r
263 final Pattern pattern = Pattern.compile("//s+");
\r
266 String sname = "", seqstr = null;
\r
268 line = infasta.readLine();
\r
269 if ((line == null) || line.startsWith(">")) {
\r
270 if (seqstr != null) {
\r
271 seqs.add(new FastaSequence(sname.substring(1), seqstr));
\r
273 sname = line; // remove >
\r
276 final String subseq = pattern.matcher(line).replaceAll("");
\r
279 } while (line != null);
\r
286 * Writes FastaSequence in the file, each sequence will take one line only
\r
290 * @throws IOException
\r
292 public static void writeFasta(final OutputStream os,
\r
293 final List<FastaSequence> sequences) throws IOException {
\r
294 final OutputStreamWriter outWriter = new OutputStreamWriter(os);
\r
295 final BufferedWriter fasta_out = new BufferedWriter(outWriter);
\r
296 for (final FastaSequence fs : sequences) {
\r
297 fasta_out.write(fs.getOnelineFasta());
\r
303 public static Map<String, Score> readJRonn(final File result)
\r
304 throws IOException, UnknownFileFormatException {
\r
305 InputStream input = new FileInputStream(result);
\r
306 Map<String, Score> sequences = readJRonn(input);
\r
312 * Reader for JRonn horizontal file format
\r
315 * >Foobar M G D T T A G 0.48 0.42
\r
316 * 0.42 0.48 0.52 0.53 0.54
\r
319 * Where all values are tab delimited
\r
322 * the InputStream connected to the JRonn output file
\r
323 * @return List of {@link AnnotatedSequence} objects
\r
324 * @throws IOException
\r
325 * is thrown if the inStream has problems accessing the data
\r
326 * @throws UnknownFileFormatException
\r
327 * is thrown if the inStream represents an unknown source of
\r
328 * data, i.e. not a JRonn output
\r
330 public static Map<String, Score> readJRonn(final InputStream inStream)
\r
331 throws IOException, UnknownFileFormatException {
\r
332 final Map<String, Score> seqs = new HashMap<String, Score>();
\r
334 final BufferedReader infasta = new BufferedReader(
\r
335 new InputStreamReader(inStream, "UTF8"), 16000);
\r
340 line = infasta.readLine();
\r
341 if (line == null || line.isEmpty()) {
\r
342 // skip empty lines
\r
345 if (line.startsWith(">")) {
\r
347 sname = line.trim().substring(1);
\r
348 // read sequence line
\r
349 line = infasta.readLine();
\r
350 final String sequence = line.replace("\t", "");
\r
351 // read annotation line
\r
352 line = infasta.readLine();
\r
353 String[] annotValues = line.split("\t");
\r
354 float[] annotation = convertToNumber(annotValues);
\r
355 if (annotation.length != sequence.length()) {
\r
356 throw new UnknownFileFormatException(
\r
357 "File does not look like Jronn horizontally formatted output file!\n"
\r
358 + JRONN_WRONG_FORMAT_MESSAGE);
\r
360 seqs.put(sname, new Score(DisorderMethod.JRonn, annotation));
\r
362 } while (line != null);
\r
368 private static float[] convertToNumber(String[] annotValues)
\r
369 throws UnknownFileFormatException {
\r
370 float[] annotation = new float[annotValues.length];
\r
372 for (int i = 0; i < annotation.length; i++) {
\r
373 annotation[i] = Float.parseFloat(annotValues[i]);
\r
375 } catch (NumberFormatException e) {
\r
376 throw new UnknownFileFormatException(JRONN_WRONG_FORMAT_MESSAGE,
\r
382 private static final String JRONN_WRONG_FORMAT_MESSAGE = "Jronn file must be in the following format:\n"
\r
383 + ">sequence_name\n "
\r
385 + "0.43 0.22 0.65\n"
\r
386 + "Where first line is the sequence name,\n"
\r
387 + "second line is the tab delimited sequence,\n"
\r
388 + "third line contains tab delimited disorder prediction values.\n"
\r
389 + "No lines are allowed between these three. Additionally, the number of "
\r
390 + "sequence residues must be equal to the number of the disorder values.";
\r
393 * Closes the Closable and logs the exception if any
\r
398 public final static void closeSilently(java.util.logging.Logger log,
\r
399 Closeable stream) {
\r
400 if (stream != null) {
\r
403 } catch (IOException e) {
\r
404 log.log(Level.WARNING, e.getLocalizedMessage(), e.getCause());
\r
411 > Foobar_dundeefriends
\r
413 * # COILS 34-41, 50-58, 83-91, 118-127, 160-169, 191-220, 243-252, 287-343
\r
417 * # HOTLOOPS 190-204
\r
419 * # RESIDUE COILS REM465 HOTLOOPS
\r
421 * M 0.86010 0.88512 0.37094
\r
423 * T 0.79983 0.85864 0.44331
\r
425 * >Next Sequence name
\r
430 * @throws IOException
\r
431 * @throws UnknownFileFormatException
\r
433 public static HashMap<FastaSequence, HashSet<Score>> readDisembl(
\r
434 final InputStream input) throws IOException,
\r
435 UnknownFileFormatException {
\r
436 Scanner scan = new Scanner(input);
\r
437 scan.useDelimiter(">");
\r
438 if (!scan.hasNext()) {
\r
439 throw new UnknownFileFormatException(
\r
440 "In Disembl score format each sequence score is expected "
\r
441 + "to start from the line: >Sequence name "
\r
442 + " No such line was found!");
\r
445 HashMap<FastaSequence, HashSet<Score>> results = new HashMap<FastaSequence, HashSet<Score>>();
\r
446 int seqCounter = 0;
\r
447 while (scan.hasNext()) {
\r
449 String singleSeq = scan.next();
\r
450 Scanner scansingle = new Scanner(singleSeq);
\r
451 if (!scansingle.hasNextLine()) {
\r
452 throw new RuntimeException(
\r
453 "The input looks like an incomplete disembl file - cannot parse!");
\r
456 StringBuffer seqbuffer = new StringBuffer();
\r
457 ArrayList<Float> coils = new ArrayList<Float>();
\r
458 ArrayList<Float> rem = new ArrayList<Float>();
\r
459 ArrayList<Float> hotloops = new ArrayList<Float>();
\r
461 String sequenceName = scansingle.nextLine().trim();
\r
462 SortedSet<Range> coilsR = parseRanges(DisemblResult.COILS,
\r
463 scansingle.nextLine());
\r
464 SortedSet<Range> rem465R = parseRanges(DisemblResult.REM465,
\r
465 scansingle.nextLine());
\r
466 SortedSet<Range> loopsR = parseRanges(DisemblResult.HOTLOOPS,
\r
467 scansingle.nextLine());
\r
469 String title = scansingle.nextLine();
\r
470 assert title.startsWith("# RESIDUE COILS REM465 HOTLOOPS") : ">Sequence_name must follow column title: # RESIDUE COILS REM465 HOTLOOPS!";
\r
472 while (scansingle.hasNext()) {
\r
473 seqbuffer.append(scansingle.next());
\r
474 coils.add(scansingle.nextFloat());
\r
475 rem.add(scansingle.nextFloat());
\r
476 hotloops.add(scansingle.nextFloat());
\r
478 FastaSequence fs = new FastaSequence(sequenceName,
\r
479 seqbuffer.toString());
\r
480 HashSet<Score> scores = new HashSet<Score>();
\r
481 scores.add(new Score(DisemblResult.COILS, coils, coilsR));
\r
482 scores.add(new Score(DisemblResult.HOTLOOPS, hotloops, rem465R));
\r
483 scores.add(new Score(DisemblResult.REM465, rem, loopsR));
\r
484 results.put(fs, scores);
\r
486 scansingle.close();
\r
496 * # COILS 34-41, 50-58, 83-91, 118-127, 160-169, 191-220, 243-252, 287-343,
\r
497 * 350-391, 429-485, 497-506, 539-547
\r
501 * # HOTLOOPS 190-204
\r
506 private static SortedSet<Range> parseRanges(Enum resultType, String lines) {
\r
507 SortedSet<Range> ranges = new TreeSet<Range>();
\r
509 Scanner scan = new Scanner(lines);
\r
511 assert scan.hasNext();
\r
512 String del = scan.next();
\r
513 assert "#".equals(del); // pass delimiter #
\r
514 String type = scan.next(); // pass enum name e.g. COILS
\r
515 assert resultType.toString().equalsIgnoreCase(type) : "Unknown result type: "
\r
516 + resultType.toString();
\r
518 // beginning of the ranges
\r
519 scan.useDelimiter(",");
\r
520 while (scan.hasNext()) {
\r
521 String range = scan.next();
\r
522 if (!Util.isEmpty(range)) {
\r
523 ranges.add(new Range(range.split("-")));
\r
529 public static HashMap<String, HashSet<Score>> removeSequences(
\r
530 HashMap<FastaSequence, HashSet<Score>> disemblResults) {
\r
531 HashMap<String, HashSet<Score>> seqNameScores = new HashMap<String, HashSet<Score>>();
\r
532 for (Map.Entry<FastaSequence, HashSet<Score>> dres : disemblResults
\r
534 seqNameScores.put(dres.getKey().getId(), dres.getValue());
\r
536 return seqNameScores;
\r
541 > Foobar_dundeefriends
\r
543 * # COILS 34-41, 50-58, 83-91, 118-127, 160-169, 191-220, 243-252, 287-343
\r
547 * # HOTLOOPS 190-204
\r
549 * # RESIDUE COILS REM465 HOTLOOPS
\r
551 * M 0.86010 0.88512 0.37094
\r
553 * T 0.79983 0.85864 0.44331
\r
555 * >Next Sequence name
\r
560 * @throws IOException
\r
561 * @throws UnknownFileFormatException
\r
563 public static HashMap<FastaSequence, HashSet<Score>> readGlobPlot(
\r
564 final InputStream input) throws IOException,
\r
565 UnknownFileFormatException {
\r
566 Scanner scan = new Scanner(input);
\r
567 scan.useDelimiter(">");
\r
568 if (!scan.hasNext()) {
\r
569 throw new UnknownFileFormatException(
\r
570 "In GlobPlot score format each sequence score is expected "
\r
571 + "to start from the line: >Sequence name "
\r
572 + " No such line was found!");
\r
575 HashMap<FastaSequence, HashSet<Score>> results = new HashMap<FastaSequence, HashSet<Score>>();
\r
576 int seqCounter = 0;
\r
577 while (scan.hasNext()) {
\r
579 String singleSeq = scan.next();
\r
580 Scanner scansingle = new Scanner(singleSeq);
\r
581 if (!scansingle.hasNextLine()) {
\r
582 throw new RuntimeException(
\r
583 "The input looks like an incomplete GlobPlot file - cannot parse!");
\r
586 StringBuffer seqbuffer = new StringBuffer();
\r
587 ArrayList<Float> dydxScore = new ArrayList<Float>();
\r
588 ArrayList<Float> rawScore = new ArrayList<Float>();
\r
589 ArrayList<Float> smoothedScore = new ArrayList<Float>();
\r
591 String sequenceName = scansingle.nextLine().trim();
\r
592 SortedSet<Range> domsR = parseRanges(GlobProtResult.GlobDoms,
\r
593 scansingle.nextLine());
\r
594 SortedSet<Range> disorderR = parseRanges(GlobProtResult.Disorder,
\r
595 scansingle.nextLine());
\r
597 String title = scansingle.nextLine();
\r
598 assert title.startsWith("# RESIDUE DYDX") : ">Sequence_name must follow column title: # RESIDUE DYDX RAW SMOOTHED!";
\r
600 while (scansingle.hasNext()) {
\r
601 seqbuffer.append(scansingle.next());
\r
602 dydxScore.add(scansingle.nextFloat());
\r
603 rawScore.add(scansingle.nextFloat());
\r
604 smoothedScore.add(scansingle.nextFloat());
\r
606 FastaSequence fs = new FastaSequence(sequenceName,
\r
607 seqbuffer.toString());
\r
608 HashSet<Score> scores = new HashSet<Score>();
\r
609 scores.add(new Score(GlobProtResult.Disorder, disorderR));
\r
610 scores.add(new Score(GlobProtResult.GlobDoms, domsR));
\r
611 scores.add(new Score(GlobProtResult.Dydx, dydxScore));
\r
612 scores.add(new Score(GlobProtResult.RawScore, rawScore));
\r
613 scores.add(new Score(GlobProtResult.SmoothedScore, smoothedScore));
\r
614 results.put(fs, scores);
\r
616 scansingle.close();
\r
623 * Read AACon result with no alignment files. This method leaves incoming
\r
624 * the InputStream results open!
\r
627 * output file of AAConservation
\r
628 * @return Map with keys {@link ConservationMethod} -> float[]
\r
630 public static HashSet<Score> readAAConResults(InputStream results) {
\r
631 if (results == null) {
\r
632 throw new NullPointerException(
\r
633 "InputStream with results must be provided");
\r
635 HashSet<Score> annotations = new HashSet<Score>();
\r
636 Scanner sc = new Scanner(results);
\r
637 sc.useDelimiter("#");
\r
638 while (sc.hasNext()) {
\r
639 String line = sc.next();
\r
640 int spacePos = line.indexOf(" ");
\r
641 assert spacePos > 0 : "Space is expected as delimited between method "
\r
642 + "name and values!";
\r
643 String methodLine = line.substring(0, spacePos);
\r
644 ConservationMethod method = ConservationMethod
\r
645 .getMethod(methodLine);
\r
646 assert method != null : "Method " + methodLine
\r
647 + " is not recognized! ";
\r
648 Scanner valuesScanner = new Scanner(line.substring(spacePos));
\r
649 ArrayList<Float> values = new ArrayList<Float>();
\r
650 while (valuesScanner.hasNextDouble()) {
\r
651 Double value = valuesScanner.nextDouble();
\r
652 values.add(value.floatValue());
\r
654 annotations.add(new Score(method, values));
\r
656 return annotations;
\r
660 * Reads and parses Fasta or Clustal formatted file into a list of
\r
661 * FastaSequence objects
\r
663 * @param inFilePath
\r
664 * the path to the input file
\r
665 * @throws IOException
\r
666 * if the file denoted by inFilePath cannot be read
\r
667 * @throws UnknownFileFormatException
\r
668 * if the inFilePath points to the file which format cannot be
\r
670 * @return the List of FastaSequence objects
\r
673 public static List<FastaSequence> openInputStream(String inFilePath)
\r
674 throws IOException, UnknownFileFormatException {
\r
676 // This stream gets closed in isValidClustalFile method
\r
677 InputStream inStrForValidation = new FileInputStream(inFilePath);
\r
678 // This stream is closed in the calling methods
\r
679 InputStream inStr = new FileInputStream(inFilePath);
\r
680 List<FastaSequence> fastaSeqs = null;
\r
681 if (ClustalAlignmentUtil.isValidClustalFile(inStrForValidation)) {
\r
682 Alignment al = ClustalAlignmentUtil.readClustalFile(inStr);
\r
683 // alignment cannot be null see
\r
684 // ClustalAlignmentUtil.readClustalFile(inStr);
\r
685 fastaSeqs = al.getSequences();
\r
687 fastaSeqs = SequenceUtil.readFasta(inStr);
\r
694 enum DisemblResult {
\r
695 /** These contains ranges and scores */
\r
696 COILS, REM465, HOTLOOPS
\r
698 enum GlobProtResult {
\r
699 /** This a range with no scores */
\r
701 /** This a range with no scores */
\r
703 /** This a score with no range */
\r
705 /** This a score with no range */
\r
707 /** This a score with no range */
\r