2 // forester -- software libraries and applications
3 // for genomics and evolutionary biology research.
5 // Copyright (C) 2010 Christian M Zmasek
6 // Copyright (C) 2010 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // Contact: phylosoft @ gmail . com
24 // WWW: www.phylosoft.org/forester
26 package org.forester.ws.seqdb;
28 import java.io.BufferedReader;
29 import java.io.IOException;
30 import java.io.InputStreamReader;
31 import java.io.UnsupportedEncodingException;
33 import java.net.URLConnection;
34 import java.net.URLEncoder;
35 import java.util.ArrayList;
36 import java.util.List;
37 import java.util.SortedSet;
38 import java.util.TreeSet;
39 import java.util.regex.Matcher;
40 import java.util.regex.Pattern;
42 import org.forester.phylogeny.Phylogeny;
43 import org.forester.phylogeny.PhylogenyNode;
44 import org.forester.phylogeny.data.Accession;
45 import org.forester.phylogeny.data.Identifier;
46 import org.forester.phylogeny.data.Sequence;
47 import org.forester.phylogeny.data.Taxonomy;
48 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
49 import org.forester.util.ForesterUtil;
50 import org.forester.util.SequenceIdParser;
52 public final class SequenceDbWsTools {
54 private static final boolean ALLOW_TAXONOMY_CODE_HACKS = true; //TODO turn off for final realease!
55 public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/";
56 public final static String BASE_EMBL_DB_URL = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/";
57 public final static String EMBL_DBS_EMBL = "embl";
58 public final static String EMBL_DBS_REFSEQ_P = "refseqp";
59 public final static String EMBL_DBS_REFSEQ_N = "refseqn";
60 private final static String URL_ENC = "UTF-8";
61 // uniprot/expasy accession number format (6 chars):
62 // letter digit letter-or-digit letter-or-digit letter-or-digit digit
63 // ?: => no back-reference
64 // \A => begin of String
65 // \Z => end of String
66 private final static Pattern UNIPROT_AC_PATTERN = Pattern
67 .compile( "(?:\\A|.*[^a-zA-Z0-9])([A-Z]\\d[A-Z0-9]{3}\\d)(?:[^a-zA-Z0-9]|\\Z)" );
68 private final static boolean DEBUG = false;
70 public static List<UniProtTaxonomy> getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return )
72 final List<String> result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return );
73 if ( result.size() > 0 ) {
74 return parseUniProtTaxonomy( result );
79 public static List<UniProtTaxonomy> getTaxonomiesFromCommonNameStrict( final String cn,
80 final int max_taxonomies_return )
82 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromCommonName( cn, max_taxonomies_return );
83 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
84 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
85 for( final UniProtTaxonomy taxonomy : taxonomies ) {
86 if ( taxonomy.getCommonName().equalsIgnoreCase( cn ) ) {
87 filtered_taxonomies.add( taxonomy );
90 return filtered_taxonomies;
95 public static List<UniProtTaxonomy> getTaxonomiesFromId( final String id, final int max_taxonomies_return )
97 final List<String> result = getTaxonomyStringFromId( id, max_taxonomies_return );
98 if ( result.size() > 0 ) {
99 return parseUniProtTaxonomy( result );
104 public static List<UniProtTaxonomy> getTaxonomiesFromScientificName( final String sn,
105 final int max_taxonomies_return )
107 // Hack! Craniata? ..
108 if ( sn.equals( "Drosophila" ) ) {
109 return uniProtTaxonomyToList( UniProtTaxonomy.DROSOPHILA_GENUS );
111 else if ( sn.equals( "Xenopus" ) ) {
112 return uniProtTaxonomyToList( UniProtTaxonomy.XENOPUS_GENUS );
114 // else if ( sn.equals( "Nucleariidae and Fonticula group" ) ) {
115 // return hack( UniProtTaxonomy.NUCLEARIIDAE_AND_FONTICULA );
117 final List<String> result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return );
118 if ( result.size() > 0 ) {
119 return parseUniProtTaxonomy( result );
125 * Does not return "sub-types".
126 * For example, for "Mus musculus" only returns "Mus musculus"
127 * and not "Mus musculus", "Mus musculus bactrianus", ...
130 public static List<UniProtTaxonomy> getTaxonomiesFromScientificNameStrict( final String sn,
131 final int max_taxonomies_return )
133 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromScientificName( sn, max_taxonomies_return );
134 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
135 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
136 for( final UniProtTaxonomy taxonomy : taxonomies ) {
137 if ( taxonomy.getScientificName().equalsIgnoreCase( sn ) ) {
138 filtered_taxonomies.add( taxonomy );
141 return filtered_taxonomies;
146 public static List<UniProtTaxonomy> getTaxonomiesFromTaxonomyCode( final String code,
147 final int max_taxonomies_return )
149 final String my_code = new String( code );
150 if ( ALLOW_TAXONOMY_CODE_HACKS ) {
151 final List<UniProtTaxonomy> l = resolveFakeTaxonomyCodes( max_taxonomies_return, my_code );
156 final List<String> result = getTaxonomyStringFromTaxonomyCode( my_code, max_taxonomies_return );
157 if ( result.size() > 0 ) {
158 return parseUniProtTaxonomy( result );
163 public static SequenceDatabaseEntry obtainEmblEntry( final Identifier id, final int max_lines_to_return )
165 final List<String> lines = queryEmblDb( id, max_lines_to_return );
166 return EbiDbEntry.createInstanceFromPlainText( lines );
169 public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Identifier id, final int max_lines_to_return )
171 final List<String> lines = queryEmblDb( id, max_lines_to_return );
172 return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines );
175 public static SortedSet<String> obtainSeqInformation( final Phylogeny phy,
176 final boolean ext_nodes_only,
177 final boolean allow_to_set_taxonomic_data,
178 final int lines_to_return ) throws IOException {
179 final SortedSet<String> not_found = new TreeSet<String>();
180 for( final PhylogenyNodeIterator iter = phy.iteratorPostorder(); iter.hasNext(); ) {
181 final PhylogenyNode node = iter.next();
182 if ( ext_nodes_only && node.isInternal() ) {
186 Identifier id = null;
188 if ( node.getNodeData().isHasSequence() && ( node.getNodeData().getSequence().getAccession() != null )
189 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
190 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
191 && node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "uniprot" ) ) {
192 query = node.getNodeData().getSequence().getAccession().getValue();
195 else if ( node.getNodeData().isHasSequence()
196 && ( node.getNodeData().getSequence().getAccession() != null )
197 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
198 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
199 && ( node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "embl" ) || node
200 .getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "ebi" ) ) ) {
201 query = node.getNodeData().getSequence().getAccession().getValue();
204 else if ( !ForesterUtil.isEmpty( node.getName() ) ) {
205 if ( ( query = parseUniProtAccessor( node.getName() ) ) != null ) {
208 else if ( ( id = SequenceIdParser.parse( node.getName() ) ) != null ) {
209 if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
212 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
217 if ( db == Db.NONE ) {
218 not_found.add( node.getName() );
220 SequenceDatabaseEntry db_entry = null;
221 if ( !ForesterUtil.isEmpty( query ) ) {
222 if ( db == Db.UNIPROT ) {
224 System.out.println( "uniprot: " + query );
226 db_entry = obtainUniProtEntry( query, lines_to_return );
228 if ( ( db == Db.EMBL ) || ( ( db == Db.UNIPROT ) && ( db_entry == null ) ) ) {
230 System.out.println( "embl: " + query );
232 db_entry = obtainEmblEntry( new Identifier( query ), lines_to_return );
233 if ( ( db == Db.UNIPROT ) && ( db_entry != null ) ) {
238 else if ( ( db == Db.REFSEQ ) && ( id != null ) ) {
239 db_entry = obtainRefSeqEntryFromEmbl( id, lines_to_return );
241 else if ( ( db == Db.NCBI ) && ( id != null ) ) {
242 db_entry = obtainEmblEntry( id, lines_to_return );
243 if ( ( db_entry != null ) && !db_entry.isEmpty() ) {
244 final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence()
246 if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) {
248 if ( db == Db.EMBL ) {
251 else if ( db == Db.UNIPROT ) {
254 else if ( db == Db.NCBI ) {
257 else if ( db == Db.REFSEQ ) {
260 seq.setAccession( new Accession( db_entry.getAccession(), type ) );
262 if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) {
263 seq.setName( db_entry.getSequenceName() );
265 if ( !ForesterUtil.isEmpty( db_entry.getSequenceSymbol() ) ) {
266 seq.setSymbol( db_entry.getSequenceSymbol() );
268 final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy()
270 if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) {
271 tax.setScientificName( db_entry.getTaxonomyScientificName() );
273 if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) {
274 tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) );
276 node.getNodeData().setTaxonomy( tax );
277 node.getNodeData().setSequence( seq );
279 else if ( db != Db.NONE ) {
280 not_found.add( node.getName() );
283 Thread.sleep( 10 );// Sleep for 10 ms
285 catch ( final InterruptedException ie ) {
292 public static SequenceDatabaseEntry obtainUniProtEntry( final String query, final int max_lines_to_return )
294 final List<String> lines = queryUniprot( "uniprot/" + query + ".txt", max_lines_to_return );
295 return UniProtEntry.createInstanceFromPlainText( lines );
299 * Returns null if no match.
305 static public String parseUniProtAccessor( final String query ) {
306 final Matcher m = UNIPROT_AC_PATTERN.matcher( query );
307 if ( m.lookingAt() ) {
315 public static List<String> queryDb( final String query, int max_lines_to_return, final String base_url )
317 if ( ForesterUtil.isEmpty( query ) ) {
318 throw new IllegalArgumentException( "illegal attempt to use empty query " );
320 if ( max_lines_to_return < 1 ) {
321 max_lines_to_return = 1;
323 final URL url = new URL( base_url + query );
325 System.out.println( "url: " + url.toString() );
327 final URLConnection urlc = url.openConnection();
328 final BufferedReader in = new BufferedReader( new InputStreamReader( urlc.getInputStream() ) );
330 final List<String> result = new ArrayList<String>();
331 while ( ( line = in.readLine() ) != null ) {
333 System.out.println( line );
336 if ( result.size() > max_lines_to_return ) {
342 // To prevent accessing online dbs in too quick succession.
345 catch ( final InterruptedException e ) {
351 public static List<String> queryEmblDb( final Identifier id, final int max_lines_to_return ) throws IOException {
352 final StringBuilder url_sb = new StringBuilder();
353 url_sb.append( BASE_EMBL_DB_URL );
354 if ( ForesterUtil.isEmpty( id.getProvider() ) || id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
355 url_sb.append( SequenceDbWsTools.EMBL_DBS_EMBL );
356 url_sb.append( '/' );
358 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
359 if ( id.getValue().toUpperCase().indexOf( 'P' ) == 1 ) {
360 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_P );
361 url_sb.append( '/' );
364 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_N );
365 url_sb.append( '/' );
368 return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() );
371 public static List<String> queryUniprot( final String query, final int max_lines_to_return ) throws IOException {
372 return queryDb( query, max_lines_to_return, BASE_UNIPROT_URL );
375 private static String encode( final String str ) throws UnsupportedEncodingException {
376 return URLEncoder.encode( str.trim(), URL_ENC );
379 private static List<String> getTaxonomyStringFromCommonName( final String cn, final int max_lines_to_return )
381 return queryUniprot( "taxonomy/?query=common%3a%22" + encode( cn ) + "%22&format=tab", max_lines_to_return );
384 private static List<String> getTaxonomyStringFromId( final String id, final int max_lines_to_return )
386 return queryUniprot( "taxonomy/?query=id%3a%22" + encode( id ) + "%22&format=tab", max_lines_to_return );
389 private static List<String> getTaxonomyStringFromScientificName( final String sn, final int max_lines_to_return )
391 return queryUniprot( "taxonomy/?query=scientific%3a%22" + encode( sn ) + "%22&format=tab", max_lines_to_return );
394 private static List<String> getTaxonomyStringFromTaxonomyCode( final String code, final int max_lines_to_return )
396 return queryUniprot( "taxonomy/?query=mnemonic%3a%22" + encode( code ) + "%22&format=tab", max_lines_to_return );
399 private static List<UniProtTaxonomy> parseUniProtTaxonomy( final List<String> result ) throws IOException {
400 final List<UniProtTaxonomy> taxonomies = new ArrayList<UniProtTaxonomy>();
401 for( final String line : result ) {
402 if ( ForesterUtil.isEmpty( line ) ) {
403 // Ignore empty lines.
405 else if ( line.startsWith( "Taxon" ) ) {
406 final String[] items = line.split( "\t" );
407 if ( !( items[ 1 ].equalsIgnoreCase( "Mnemonic" ) && items[ 2 ].equalsIgnoreCase( "Scientific name" )
408 && items[ 3 ].equalsIgnoreCase( "Common name" ) && items[ 4 ].equalsIgnoreCase( "Synonym" )
409 && items[ 5 ].equalsIgnoreCase( "Other Names" ) && items[ 6 ].equalsIgnoreCase( "Reviewed" )
410 && items[ 7 ].equalsIgnoreCase( "Rank" ) && items[ 8 ].equalsIgnoreCase( "Lineage" ) ) ) {
411 throw new IOException( "Unreconized UniProt Taxonomy format: " + line );
415 if ( line.split( "\t" ).length > 4 ) {
416 taxonomies.add( new UniProtTaxonomy( line ) );
423 private static List<UniProtTaxonomy> resolveFakeTaxonomyCodes( final int max_taxonomies_return, final String code )
425 if ( code.equals( "CAP" ) ) {
426 return getTaxonomiesFromId( "283909", max_taxonomies_return );
428 else if ( code.equals( "FUGRU" ) ) {
429 return getTaxonomiesFromId( "31033", max_taxonomies_return );
431 else if ( code.equals( "GIALA" ) ) {
432 return getTaxonomiesFromId( "5741", max_taxonomies_return );
434 else if ( code.equals( "TRIVE" ) ) {
435 return getTaxonomiesFromId( "413071", max_taxonomies_return );
437 else if ( code.equals( "CAPOWC" ) ) {
438 return getTaxonomiesFromId( "192875", max_taxonomies_return );
440 else if ( code.equals( "SPHARC" ) ) {
441 return getTaxonomiesFromId( "667725", max_taxonomies_return );
443 else if ( code.equals( "THETRA" ) ) {
444 return getTaxonomiesFromId( "529818", max_taxonomies_return );
446 else if ( code.equals( "CHLVUL" ) ) {
447 return getTaxonomiesFromId( "574566", max_taxonomies_return );
449 else if ( code.equals( "CITCLE" ) ) {
450 return getTaxonomiesFromId( "85681", max_taxonomies_return );
452 else if ( code.equals( "MYCPOP" ) ) {
453 return getTaxonomiesFromId( "85929", max_taxonomies_return );
455 else if ( code.equals( "AGABB" ) ) {
456 return getTaxonomiesFromId( "597362", max_taxonomies_return );
458 else if ( code.equals( "BAUCOM" ) ) {
459 return getTaxonomiesFromId( "430998", max_taxonomies_return );
461 else if ( code.equals( "DICSQU" ) ) {
462 return getTaxonomiesFromId( "114155", max_taxonomies_return );
464 else if ( code.equals( "FOMPIN" ) ) {
465 return getTaxonomiesFromId( "40483", max_taxonomies_return );
467 else if ( code.equals( "HYDMA" ) ) {
468 return getTaxonomiesFromId( "6085", max_taxonomies_return );
470 else if ( code.equals( "MYCFI" ) ) {
471 return getTaxonomiesFromId( "83344", max_taxonomies_return );
473 else if ( code.equals( "OIDMAI" ) ) {
474 return getTaxonomiesFromId( "78148", max_taxonomies_return );
476 else if ( code.equals( "OSTRC" ) ) {
477 return getTaxonomiesFromId( "385169", max_taxonomies_return );
479 else if ( code.equals( "POSPL" ) ) {
480 return getTaxonomiesFromId( "104341", max_taxonomies_return );
482 else if ( code.equals( "SAICOM" ) ) {
483 return getTaxonomiesFromId( "5606", max_taxonomies_return );
485 else if ( code.equals( "SERLA" ) ) {
486 return getTaxonomiesFromId( "85982", max_taxonomies_return );
488 else if ( code.equals( "SPORO" ) ) {
489 return getTaxonomiesFromId( "40563", max_taxonomies_return );
491 else if ( code.equals( "ACRALC" ) ) {
492 return getTaxonomiesFromId( "398408", max_taxonomies_return );
494 else if ( code.equals( "THITER" ) ) {
495 return getTaxonomiesFromId( "35720", max_taxonomies_return );
497 else if ( code.equals( "MYCTHE" ) ) {
498 return getTaxonomiesFromId( "78579", max_taxonomies_return );
500 else if ( code.equals( "CONPUT" ) ) {
501 return getTaxonomiesFromId( "80637", max_taxonomies_return );
503 else if ( code.equals( "WOLCOC" ) ) {
504 return getTaxonomiesFromId( "81056", max_taxonomies_return );
506 else if ( code.equals( "CLAGRA" ) ) {
507 return getTaxonomiesFromId( "27339", max_taxonomies_return );
509 else if ( code.equals( "XANPAR" ) ) {
510 return getTaxonomiesFromId( "107463", max_taxonomies_return );
512 else if ( code.equals( "HYDPIN" ) ) {
513 return getTaxonomiesFromId( "388859", max_taxonomies_return );
515 else if ( code.equals( "SERLAC" ) ) {
516 return getTaxonomiesFromId( "85982", max_taxonomies_return );
523 private static List<UniProtTaxonomy> uniProtTaxonomyToList( final UniProtTaxonomy tax ) {
524 final List<UniProtTaxonomy> l = new ArrayList<UniProtTaxonomy>();
530 UNIPROT, EMBL, NCBI, NONE, REFSEQ;