2 // forester -- software libraries and applications
3 // for genomics and evolutionary biology research.
5 // Copyright (C) 2010 Christian M Zmasek
6 // Copyright (C) 2010 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // Contact: phylosoft @ gmail . com
24 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
26 package org.forester.ws.seqdb;
28 import java.io.BufferedReader;
29 import java.io.FileNotFoundException;
30 import java.io.IOException;
31 import java.io.InputStreamReader;
32 import java.io.UnsupportedEncodingException;
34 import java.net.URLConnection;
35 import java.net.URLEncoder;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.SortedSet;
39 import java.util.TreeSet;
41 import org.forester.go.GoTerm;
42 import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException;
43 import org.forester.phylogeny.Phylogeny;
44 import org.forester.phylogeny.PhylogenyNode;
45 import org.forester.phylogeny.data.Accession;
46 import org.forester.phylogeny.data.Accession.Source;
47 import org.forester.phylogeny.data.Annotation;
48 import org.forester.phylogeny.data.Identifier;
49 import org.forester.phylogeny.data.Sequence;
50 import org.forester.phylogeny.data.Taxonomy;
51 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
52 import org.forester.sequence.MolecularSequence.TYPE;
53 import org.forester.util.ForesterUtil;
54 import org.forester.util.SequenceAccessionTools;
56 public final class SequenceDbWsTools {
58 public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/";
59 public final static int DEFAULT_LINES_TO_RETURN = 4000;
60 public final static String EMBL_DBS_REFSEQ_N = "refseqn";
61 public final static String EMBL_DBS_REFSEQ_P = "refseqp";
62 public final static String EMBL_GENBANK = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch?db=GENBANK&style=raw&id=";
63 public final static String EMBL_REFSEQ = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch?db=REFSEQ&style=raw&id=";
64 public final static String EMBL_EMBL = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch?db=EMBL&style=raw&id=";
65 private final static boolean DEBUG = true;
66 private final static String URL_ENC = "UTF-8";
67 private final static int SLEEP = 200;
68 private static final boolean ALLOW_TO_OVERWRITE_MOL_SEQ = false;
70 public static List<UniProtTaxonomy> getTaxonomiesFromCommonNameStrict( final String cn,
71 final int max_taxonomies_return )
73 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromCommonName( cn, max_taxonomies_return );
74 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
75 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
76 for( final UniProtTaxonomy taxonomy : taxonomies ) {
77 if ( taxonomy.getCommonName().equalsIgnoreCase( cn ) ) {
78 filtered_taxonomies.add( taxonomy );
81 return filtered_taxonomies;
86 public static List<UniProtTaxonomy> getTaxonomiesFromId( final String id, final int max_taxonomies_return )
88 final List<String> result = getTaxonomyStringFromId( id, max_taxonomies_return );
89 if ( result.size() > 0 ) {
90 return parseUniProtTaxonomy( result );
96 * Does not return "sub-types".
97 * For example, for "Mus musculus" only returns "Mus musculus"
98 * and not "Mus musculus", "Mus musculus bactrianus", ...
101 public static List<UniProtTaxonomy> getTaxonomiesFromScientificNameStrict( final String sn,
102 final int max_taxonomies_return )
104 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromScientificName( sn, max_taxonomies_return );
105 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
106 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
107 for( final UniProtTaxonomy taxonomy : taxonomies ) {
108 if ( taxonomy.getScientificName().equalsIgnoreCase( sn ) ) {
109 filtered_taxonomies.add( taxonomy );
112 return filtered_taxonomies;
117 public static List<UniProtTaxonomy> getTaxonomiesFromTaxonomyCode( final String code,
118 final int max_taxonomies_return )
120 final String my_code = new String( code );
121 final List<String> result = getTaxonomyStringFromTaxonomyCode( my_code, max_taxonomies_return );
122 if ( result.size() > 0 ) {
123 return parseUniProtTaxonomy( result );
128 public static SequenceDatabaseEntry obtainEmblEntry( final Accession acc ) throws IOException {
129 return obtainEmblEntry( acc, DEFAULT_LINES_TO_RETURN );
132 public static SequenceDatabaseEntry obtainEmblEntry( final Accession acc, final int max_lines_to_return )
134 final List<String> lines = queryEmblDb( acc, max_lines_to_return );
135 return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines );
138 public static SequenceDatabaseEntry obtainEntry( final String acc_str ) throws IOException {
139 if ( ForesterUtil.isEmpty( acc_str ) ) {
140 throw new IllegalArgumentException( "cannot not extract sequence db accessor from null or empty string" );
142 final Accession acc = SequenceAccessionTools.parseAccessorFromString( acc_str );
144 throw new IllegalArgumentException( "could not extract acceptable sequence db accessor from \"" + acc_str
147 if ( acc.getSource().equals( Source.REFSEQ.toString() ) || acc.getSource().equals( Source.EMBL.toString() )
148 || acc.getSource().equals( Source.NCBI.toString() ) ) {
149 return obtainEmblEntry( acc, DEFAULT_LINES_TO_RETURN );
151 else if ( acc.getSource().equals( Source.UNIPROT.toString() ) ) {
152 return obtainUniProtEntry( acc.getValue(), DEFAULT_LINES_TO_RETURN );
155 throw new IllegalArgumentException( "don't know how to handle request for source \"" + acc.getSource()
160 public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Accession acc ) throws IOException {
161 return obtainRefSeqEntryFromEmbl( acc, DEFAULT_LINES_TO_RETURN );
164 public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Accession acc, final int max_lines_to_return )
166 final List<String> lines = queryEmblDbForRefSeqEntry( acc, max_lines_to_return );
167 return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines );
170 public final static Accession obtainSeqAccession( final PhylogenyNode node ) {
171 Accession acc = SequenceAccessionTools.obtainFromSeqAccession( node );
172 if ( !isAccessionAcceptable( acc ) ) {
173 acc = SequenceAccessionTools.obtainAccessorFromDataFields( node );
178 public final static void obtainSeqInformation( final boolean allow_to_set_taxonomic_data,
179 final int lines_to_return,
180 final SortedSet<String> not_found,
181 final PhylogenyNode node ) throws IOException {
182 final Accession acc = obtainSeqAccession( node );
183 if ( !isAccessionAcceptable( acc ) ) {
184 if ( node.isExternal() || !node.isEmpty() ) {
185 not_found.add( node.toString() );
189 addDataFromDbToNode( allow_to_set_taxonomic_data, lines_to_return, not_found, node, acc );
193 public final static void obtainSeqInformation( final boolean allow_to_set_taxonomic_data,
194 final SortedSet<String> not_found,
195 final PhylogenyNode node ) throws IOException {
196 obtainSeqInformation( allow_to_set_taxonomic_data, DEFAULT_LINES_TO_RETURN, not_found, node );
199 public final static SortedSet<String> obtainSeqInformation( final Phylogeny phy,
200 final boolean ext_nodes_only,
201 final boolean allow_to_set_taxonomic_data,
202 final int lines_to_return ) throws IOException {
203 final SortedSet<String> not_found = new TreeSet<String>();
204 for( final PhylogenyNodeIterator iter = phy.iteratorPostorder(); iter.hasNext(); ) {
205 final PhylogenyNode node = iter.next();
206 if ( node.isExternal() || !ext_nodes_only ) {
207 obtainSeqInformation( allow_to_set_taxonomic_data, lines_to_return, not_found, node );
213 public final static void obtainSeqInformation( final PhylogenyNode node ) throws IOException {
214 obtainSeqInformation( true, DEFAULT_LINES_TO_RETURN, new TreeSet<String>(), node );
217 public static SequenceDatabaseEntry obtainUniProtEntry( final String query ) throws IOException {
218 return obtainUniProtEntry( query, DEFAULT_LINES_TO_RETURN );
221 public static SequenceDatabaseEntry obtainUniProtEntry( final String query, final int max_lines_to_return )
223 final List<String> lines = queryUniprot( "uniprot/" + query + ".txt", max_lines_to_return );
224 return UniProtEntry.createInstanceFromPlainText( lines );
227 public static List<String> queryDb( final String query, int max_lines_to_return, final String base_url )
229 if ( ForesterUtil.isEmpty( query ) ) {
230 throw new IllegalArgumentException( "illegal attempt to use empty query " );
232 if ( max_lines_to_return < 1 ) {
233 max_lines_to_return = 1;
235 final URL url = new URL( base_url + query );
237 System.out.println( "url: " + url.toString() );
239 final URLConnection urlc = url.openConnection();
240 final BufferedReader in = new BufferedReader( new InputStreamReader( urlc.getInputStream() ) );
242 final List<String> result = new ArrayList<String>();
243 while ( ( line = in.readLine() ) != null ) {
245 System.out.println( line );
248 if ( result.size() > max_lines_to_return ) {
254 // To prevent accessing online dbs in too quick succession.
255 Thread.sleep( SLEEP );
257 catch ( final InterruptedException e ) {
263 public static List<String> queryEmblDb( final Accession acc, final int max_lines_to_return ) throws IOException {
264 final StringBuilder url_sb = new StringBuilder();
265 // url_sb.append( BASE_EMBL_DB_URL );
266 System.out.println( "source: " + acc.getSource() );
267 if ( acc.getSource().equals( Source.NCBI.toString() ) ) {
268 url_sb.append( EMBL_GENBANK );
269 //url_sb.append( '/' );
271 else if ( acc.getSource().equals( Source.REFSEQ.toString() ) ) {
272 url_sb.append( EMBL_REFSEQ );
274 else if ( acc.getSource().equals( Source.EMBL.toString() ) ) {
275 url_sb.append( EMBL_EMBL );
278 throw new IllegalArgumentException( "unable to handle source: " + acc.getSource() );
280 return queryDb( acc.getValue(), max_lines_to_return, url_sb.toString() );
283 public static List<String> queryEmblDbForRefSeqEntry( final Accession id, final int max_lines_to_return )
285 final StringBuilder url_sb = new StringBuilder();
286 url_sb.append( EMBL_REFSEQ );
287 return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() );
290 public static List<String> queryUniprot( final String query, final int max_lines_to_return ) throws IOException {
291 return queryDb( query, max_lines_to_return, BASE_UNIPROT_URL );
294 final static String extractFrom( final String target, final String a ) {
295 final int i_a = target.indexOf( a );
296 return target.substring( i_a + a.length() ).trim();
299 final static String extractFromTo( final String target, final String a, final String b ) {
300 final int i_a = target.indexOf( a );
301 final int i_b = target.indexOf( b );
302 if ( ( i_a < 0 ) || ( i_b < i_a ) ) {
305 return target.substring( i_a + a.length(), i_b ).trim();
308 final static String extractTo( final String target, final String b ) {
309 final int i_b = target.indexOf( b );
310 return target.substring( 0, i_b ).trim();
313 private static void addDataFromDbToNode( final boolean allow_to_set_taxonomic_data,
314 final int lines_to_return,
315 final SortedSet<String> not_found,
316 final PhylogenyNode node,
317 final Accession acc ) throws IOException {
318 SequenceDatabaseEntry db_entry = null;
319 final String query = acc.getValue();
320 if ( acc.getSource().equals( Source.UNIPROT.toString() ) ) {
322 System.out.println( "uniprot: " + query );
325 db_entry = obtainUniProtEntry( query, lines_to_return );
327 catch ( final FileNotFoundException e ) {
328 // Eat this, and move to next.
331 else if ( acc.getSource().equals( Source.REFSEQ.toString() ) ) {
333 System.out.println( "refseq: " + query );
336 db_entry = obtainRefSeqEntryFromEmbl( new Accession( query ), lines_to_return );
338 catch ( final FileNotFoundException e ) {
339 // Eat this, and move to next.
342 else if ( acc.getSource().equals( Source.EMBL.toString() ) || acc.getSource().equals( Source.NCBI.toString() )
343 || acc.getSource().equals( Source.EMBL.toString() ) ) {
345 System.out.println( acc.toString() );
348 db_entry = obtainEmblEntry( acc, lines_to_return );
350 catch ( final FileNotFoundException e ) {
351 // Eat this, and move to next.
354 else if ( acc.getSource().equals( Source.GI.toString() ) ) {
356 System.out.println( "gi: " + query );
359 db_entry = obtainRefSeqEntryFromEmbl( new Accession( query ), lines_to_return );
361 catch ( final FileNotFoundException e ) {
362 // Eat this, and move to next.
365 if ( ( db_entry != null ) && !db_entry.isEmpty() ) {
366 final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence() : new Sequence();
367 if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) {
368 seq.setAccession( new Accession( db_entry.getAccession(), acc.getSource() ) );
370 if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) {
371 seq.setName( db_entry.getSequenceName() );
373 if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) {
374 seq.setGeneName( db_entry.getGeneName() );
376 if ( !ForesterUtil.isEmpty( db_entry.getSequenceSymbol() ) ) {
378 seq.setSymbol( db_entry.getSequenceSymbol() );
380 catch ( final PhyloXmlDataFormatException e ) {
381 // Eat this exception.
384 if ( ( db_entry.getMolecularSequence() != null )
385 && !ForesterUtil.isEmpty( db_entry.getMolecularSequence().getMolecularSequenceAsString() )
386 && ( ALLOW_TO_OVERWRITE_MOL_SEQ || seq.getMolecularSequence().isEmpty() ) ) {
387 seq.setMolecularSequence( db_entry.getMolecularSequence().getMolecularSequenceAsString() );
388 seq.setMolecularSequenceAligned( false );
389 if ( db_entry.getMolecularSequence().getType() == TYPE.AA ) {
390 seq.setType( "protein" );
392 else if ( db_entry.getMolecularSequence().getType() == TYPE.DNA ) {
393 seq.setType( "dna" );
395 else if ( db_entry.getMolecularSequence().getType() == TYPE.RNA ) {
396 seq.setType( "rna" );
399 if ( ( db_entry.getGoTerms() != null ) && !db_entry.getGoTerms().isEmpty() ) {
400 for( final GoTerm go : db_entry.getGoTerms() ) {
401 final Annotation ann = new Annotation( go.getGoId().getId() );
402 ann.setDesc( go.getName() );
403 seq.addAnnotation( ann );
406 if ( ( db_entry.getCrossReferences() != null ) && !db_entry.getCrossReferences().isEmpty() ) {
407 for( final Accession x : db_entry.getCrossReferences() ) {
408 seq.addCrossReference( x );
411 if ( !ForesterUtil.isEmpty( db_entry.getChromosome() ) && !ForesterUtil.isEmpty( db_entry.getMap() ) ) {
412 seq.setLocation( "chr " + db_entry.getChromosome() + ", " + db_entry.getMap() );
414 else if ( !ForesterUtil.isEmpty( db_entry.getChromosome() ) ) {
415 seq.setLocation( "chr " + db_entry.getChromosome() );
417 else if ( !ForesterUtil.isEmpty( db_entry.getMap() ) ) {
418 seq.setLocation( db_entry.getMap() );
420 final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy() : new Taxonomy();
421 if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) {
422 tax.setScientificName( db_entry.getTaxonomyScientificName() );
424 if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) {
425 tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) );
427 node.getNodeData().setTaxonomy( tax );
428 node.getNodeData().setSequence( seq );
431 if ( node.isExternal() || !node.isEmpty() ) {
432 not_found.add( node.toString() );
436 Thread.sleep( SLEEP );
438 catch ( final InterruptedException ie ) {
442 private static String encode( final String str ) throws UnsupportedEncodingException {
443 return URLEncoder.encode( str.trim(), URL_ENC );
446 private static List<UniProtTaxonomy> getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return )
448 final List<String> result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return );
449 if ( result.size() > 0 ) {
450 return parseUniProtTaxonomy( result );
455 private static List<UniProtTaxonomy> getTaxonomiesFromScientificName( final String sn,
456 final int max_taxonomies_return )
458 final List<String> result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return );
459 if ( result.size() > 0 ) {
460 return parseUniProtTaxonomy( result );
465 private static List<String> getTaxonomyStringFromCommonName( final String cn, final int max_lines_to_return )
467 return queryUniprot( "taxonomy/?query=common%3a%22" + encode( cn ) + "%22&format=tab", max_lines_to_return );
470 private static List<String> getTaxonomyStringFromId( final String id, final int max_lines_to_return )
472 return queryUniprot( "taxonomy/?query=id%3a%22" + encode( id ) + "%22&format=tab", max_lines_to_return );
475 private static List<String> getTaxonomyStringFromScientificName( final String sn, final int max_lines_to_return )
477 return queryUniprot( "taxonomy/?query=scientific%3a%22" + encode( sn ) + "%22&format=tab", max_lines_to_return );
480 private static List<String> getTaxonomyStringFromTaxonomyCode( final String code, final int max_lines_to_return )
482 return queryUniprot( "taxonomy/?query=mnemonic%3a%22" + encode( code ) + "%22&format=tab", max_lines_to_return );
485 private final static boolean isAccessionAcceptable( final Accession acc ) {
486 return ( !( ( acc == null ) || ForesterUtil.isEmpty( acc.getSource() ) || ForesterUtil.isEmpty( acc.getValue() ) || ( ( acc
487 .getSource().equals( Source.UNIPROT.toString() ) )
488 && ( acc.getSource().toString().equals( Source.EMBL.toString() ) ) && ( acc.getSource().toString()
489 .equals( Source.REFSEQ.toString() ) ) ) ) );
492 private static List<UniProtTaxonomy> parseUniProtTaxonomy( final List<String> result ) throws IOException {
493 final List<UniProtTaxonomy> taxonomies = new ArrayList<UniProtTaxonomy>();
494 for( final String line : result ) {
495 if ( ForesterUtil.isEmpty( line ) ) {
496 // Ignore empty lines.
498 else if ( line.startsWith( "Taxon" ) ) {
499 final String[] items = line.split( "\t" );
500 if ( !( items[ 1 ].equalsIgnoreCase( "Mnemonic" ) && items[ 2 ].equalsIgnoreCase( "Scientific name" )
501 && items[ 3 ].equalsIgnoreCase( "Common name" ) && items[ 4 ].equalsIgnoreCase( "Synonym" )
502 && items[ 5 ].equalsIgnoreCase( "Other Names" ) && items[ 6 ].equalsIgnoreCase( "Reviewed" )
503 && items[ 7 ].equalsIgnoreCase( "Rank" ) && items[ 8 ].equalsIgnoreCase( "Lineage" ) ) ) {
504 throw new IOException( "Unreconized UniProt Taxonomy format: " + line );
508 if ( line.split( "\t" ).length > 4 ) {
509 taxonomies.add( new UniProtTaxonomy( line ) );