2 // forester -- software libraries and applications
3 // for genomics and evolutionary biology research.
5 // Copyright (C) 2010 Christian M Zmasek
6 // Copyright (C) 2010 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // Contact: phylosoft @ gmail . com
24 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
26 package org.forester.ws.seqdb;
28 import java.io.BufferedReader;
29 import java.io.IOException;
30 import java.io.InputStreamReader;
31 import java.io.UnsupportedEncodingException;
33 import java.net.URLConnection;
34 import java.net.URLEncoder;
35 import java.util.ArrayList;
36 import java.util.List;
37 import java.util.SortedSet;
38 import java.util.TreeSet;
40 import org.forester.go.GoTerm;
41 import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException;
42 import org.forester.phylogeny.Phylogeny;
43 import org.forester.phylogeny.PhylogenyNode;
44 import org.forester.phylogeny.data.Accession;
45 import org.forester.phylogeny.data.Annotation;
46 import org.forester.phylogeny.data.Identifier;
47 import org.forester.phylogeny.data.Sequence;
48 import org.forester.phylogeny.data.Taxonomy;
49 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
50 import org.forester.util.ForesterUtil;
51 import org.forester.util.SequenceIdParser;
53 public final class SequenceDbWsTools {
55 public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/";
56 public final static String BASE_EMBL_DB_URL = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/";
57 public final static String EMBL_DBS_EMBL = "embl";
58 public final static String EMBL_DBS_REFSEQ_P = "refseqp";
59 public final static String EMBL_DBS_REFSEQ_N = "refseqn";
60 private final static String URL_ENC = "UTF-8";
61 private final static boolean DEBUG = false;
63 private static List<UniProtTaxonomy> getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return )
65 final List<String> result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return );
66 if ( result.size() > 0 ) {
67 return parseUniProtTaxonomy( result );
72 public static List<UniProtTaxonomy> getTaxonomiesFromCommonNameStrict( final String cn,
73 final int max_taxonomies_return )
75 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromCommonName( cn, max_taxonomies_return );
76 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
77 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
78 for( final UniProtTaxonomy taxonomy : taxonomies ) {
79 if ( taxonomy.getCommonName().equalsIgnoreCase( cn ) ) {
80 filtered_taxonomies.add( taxonomy );
83 return filtered_taxonomies;
88 public static List<UniProtTaxonomy> getTaxonomiesFromId( final String id, final int max_taxonomies_return )
90 final List<String> result = getTaxonomyStringFromId( id, max_taxonomies_return );
91 if ( result.size() > 0 ) {
92 return parseUniProtTaxonomy( result );
97 private static List<UniProtTaxonomy> getTaxonomiesFromScientificName( final String sn,
98 final int max_taxonomies_return )
100 final List<String> result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return );
101 if ( result.size() > 0 ) {
102 return parseUniProtTaxonomy( result );
108 * Does not return "sub-types".
109 * For example, for "Mus musculus" only returns "Mus musculus"
110 * and not "Mus musculus", "Mus musculus bactrianus", ...
113 public static List<UniProtTaxonomy> getTaxonomiesFromScientificNameStrict( final String sn,
114 final int max_taxonomies_return )
116 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromScientificName( sn, max_taxonomies_return );
117 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
118 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
119 for( final UniProtTaxonomy taxonomy : taxonomies ) {
120 if ( taxonomy.getScientificName().equalsIgnoreCase( sn ) ) {
121 filtered_taxonomies.add( taxonomy );
124 return filtered_taxonomies;
129 public static List<UniProtTaxonomy> getTaxonomiesFromTaxonomyCode( final String code,
130 final int max_taxonomies_return )
132 final String my_code = new String( code );
133 final List<String> result = getTaxonomyStringFromTaxonomyCode( my_code, max_taxonomies_return );
134 if ( result.size() > 0 ) {
135 return parseUniProtTaxonomy( result );
140 public static SequenceDatabaseEntry obtainEmblEntry( final Identifier id, final int max_lines_to_return )
142 final List<String> lines = queryEmblDb( id, max_lines_to_return );
143 return EbiDbEntry.createInstanceFromPlainText( lines );
146 public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Identifier id, final int max_lines_to_return )
148 final List<String> lines = queryEmblDb( id, max_lines_to_return );
149 return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines );
152 public static SortedSet<String> obtainSeqInformation( final Phylogeny phy,
153 final boolean ext_nodes_only,
154 final boolean allow_to_set_taxonomic_data,
155 final int lines_to_return ) throws IOException {
156 final SortedSet<String> not_found = new TreeSet<String>();
157 for( final PhylogenyNodeIterator iter = phy.iteratorPostorder(); iter.hasNext(); ) {
158 final PhylogenyNode node = iter.next();
159 if ( ext_nodes_only && node.isInternal() ) {
163 Identifier id = null;
165 if ( node.getNodeData().isHasSequence()
166 && ( node.getNodeData().getSequence().getAccession() != null )
167 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
168 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
169 && ( node.getNodeData().getSequence().getAccession().getValue().toLowerCase()
170 .startsWith( "uniprot" )
171 || node.getNodeData().getSequence().getAccession().getValue().toLowerCase()
172 .startsWith( "swissprot" )
173 || node.getNodeData().getSequence().getAccession().getValue().toLowerCase()
174 .startsWith( "trembl" )
175 || node.getNodeData().getSequence().getAccession().getValue().toLowerCase()
176 .startsWith( "sp" ) || node.getNodeData().getSequence().getAccession().getValue()
177 .toLowerCase().startsWith( "uniprotkb" ) ) ) {
178 query = node.getNodeData().getSequence().getAccession().getValue();
181 else if ( node.getNodeData().isHasSequence()
182 && ( node.getNodeData().getSequence().getAccession() != null )
183 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
184 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
185 && ( node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "embl" ) || node
186 .getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "ebi" ) ) ) {
187 query = node.getNodeData().getSequence().getAccession().getValue();
190 else if ( node.getNodeData().isHasSequence()
191 && ( node.getNodeData().getSequence().getAccession() != null )
192 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
193 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
194 && ( node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "ncbi" ) || node
195 .getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "genbank" ) ) ) {
196 query = node.getNodeData().getSequence().getAccession().getValue();
199 else if ( node.getNodeData().isHasSequence() && ( node.getNodeData().getSequence().getAccession() != null )
200 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() )
201 && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() )
202 && node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "refseq" ) ) {
203 query = node.getNodeData().getSequence().getAccession().getValue();
207 if ( ( query = ForesterUtil.extractUniProtKbProteinSeqIdentifier( node ) ) != null ) {
210 else if ( node.getNodeData().isHasSequence() ) {
211 if ( ( id = SequenceIdParser.parse( node.getName() ) ) != null ) {
212 if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
215 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
219 else if ( ( id = SequenceIdParser.parse( node.getNodeData().getSequence().getName() ) ) != null ) {
220 if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
223 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
227 else if ( ( id = SequenceIdParser.parse( node.getNodeData().getSequence().getGeneName() ) ) != null ) {
228 if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
231 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
235 else if ( ( id = SequenceIdParser.parse( node.getNodeData().getSequence().getSymbol() ) ) != null ) {
236 if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
239 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
245 if ( db == Db.NONE ) {
246 not_found.add( node.toString() );
248 SequenceDatabaseEntry db_entry = null;
249 if ( !ForesterUtil.isEmpty( query ) ) {
250 if ( db == Db.UNIPROT ) {
252 System.out.println( "uniprot: " + query );
254 db_entry = obtainUniProtEntry( query, lines_to_return );
256 else if ( db == Db.EMBL ) {
258 System.out.println( "embl: " + query );
260 db_entry = obtainEmblEntry( new Identifier( query ), lines_to_return );
262 else if ( db == Db.REFSEQ ) {
264 System.out.println( "refseq: " + query );
266 db_entry = obtainRefSeqEntryFromEmbl( new Identifier( query ), lines_to_return );
268 // else if ( db == Db.NCBI ) {
270 // System.out.println( "ncbi: " + query );
272 // db_entry = obtainNcbiEntry( new Identifier( query ), lines_to_return );
275 else if ( ( db == Db.REFSEQ ) && ( id != null ) ) {
276 db_entry = obtainRefSeqEntryFromEmbl( id, lines_to_return );
278 //else if ( ( db == Db.NCBI ) && ( id != null ) ) {
279 // db_entry = obtainNcbiEntry( id, lines_to_return );
281 if ( ( db_entry != null ) && !db_entry.isEmpty() ) {
282 final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence()
284 if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) {
286 if ( db == Db.EMBL ) {
289 else if ( db == Db.UNIPROT ) {
292 // else if ( db == Db.NCBI ) {
295 else if ( db == Db.REFSEQ ) {
298 seq.setAccession( new Accession( db_entry.getAccession(), type ) );
300 if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) {
301 seq.setName( db_entry.getSequenceName() );
303 if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) {
304 seq.setGeneName( db_entry.getGeneName() );
306 if ( !ForesterUtil.isEmpty( db_entry.getSequenceSymbol() ) ) {
308 seq.setSymbol( db_entry.getSequenceSymbol() );
310 catch ( final PhyloXmlDataFormatException e ) {
311 // Eat this exception.
314 if ( ( db_entry.getGoTerms() != null ) && !db_entry.getGoTerms().isEmpty() ) {
315 for( final GoTerm go : db_entry.getGoTerms() ) {
316 final Annotation ann = new Annotation( go.getGoId().getId() );
317 ann.setDesc( go.getName() );
318 seq.addAnnotation( ann );
321 if ( ( db_entry.getCrossReferences() != null ) && !db_entry.getCrossReferences().isEmpty() ) {
322 for( final Accession x : db_entry.getCrossReferences() ) {
323 seq.addCrossReference( x );
326 final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy()
328 if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) {
329 tax.setScientificName( db_entry.getTaxonomyScientificName() );
331 if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) {
332 tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) );
334 node.getNodeData().setTaxonomy( tax );
335 node.getNodeData().setSequence( seq );
337 else if ( db != Db.NONE ) {
338 not_found.add( node.getName() );
341 Thread.sleep( 10 );// Sleep for 10 ms
343 catch ( final InterruptedException ie ) {
349 public static SequenceDatabaseEntry obtainUniProtEntry( final String query, final int max_lines_to_return )
351 final List<String> lines = queryUniprot( "uniprot/" + query + ".txt", max_lines_to_return );
352 return UniProtEntry.createInstanceFromPlainText( lines );
355 public static List<String> queryDb( final String query, int max_lines_to_return, final String base_url )
357 if ( ForesterUtil.isEmpty( query ) ) {
358 throw new IllegalArgumentException( "illegal attempt to use empty query " );
360 if ( max_lines_to_return < 1 ) {
361 max_lines_to_return = 1;
363 final URL url = new URL( base_url + query );
365 System.out.println( "url: " + url.toString() );
367 final URLConnection urlc = url.openConnection();
368 final BufferedReader in = new BufferedReader( new InputStreamReader( urlc.getInputStream() ) );
370 final List<String> result = new ArrayList<String>();
371 while ( ( line = in.readLine() ) != null ) {
373 System.out.println( line );
376 if ( result.size() > max_lines_to_return ) {
382 // To prevent accessing online dbs in too quick succession.
385 catch ( final InterruptedException e ) {
391 public static List<String> queryEmblDb( final Identifier id, final int max_lines_to_return ) throws IOException {
392 final StringBuilder url_sb = new StringBuilder();
393 url_sb.append( BASE_EMBL_DB_URL );
394 if ( ForesterUtil.isEmpty( id.getProvider() ) || id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) {
395 url_sb.append( SequenceDbWsTools.EMBL_DBS_EMBL );
396 url_sb.append( '/' );
398 else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) {
399 if ( id.getValue().toUpperCase().indexOf( 'P' ) == 1 ) {
400 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_P );
401 url_sb.append( '/' );
404 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_N );
405 url_sb.append( '/' );
408 return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() );
411 public static List<String> queryUniprot( final String query, final int max_lines_to_return ) throws IOException {
412 return queryDb( query, max_lines_to_return, BASE_UNIPROT_URL );
415 private static String encode( final String str ) throws UnsupportedEncodingException {
416 return URLEncoder.encode( str.trim(), URL_ENC );
419 private static List<String> getTaxonomyStringFromCommonName( final String cn, final int max_lines_to_return )
421 return queryUniprot( "taxonomy/?query=common%3a%22" + encode( cn ) + "%22&format=tab", max_lines_to_return );
424 private static List<String> getTaxonomyStringFromId( final String id, final int max_lines_to_return )
426 return queryUniprot( "taxonomy/?query=id%3a%22" + encode( id ) + "%22&format=tab", max_lines_to_return );
429 private static List<String> getTaxonomyStringFromScientificName( final String sn, final int max_lines_to_return )
431 return queryUniprot( "taxonomy/?query=scientific%3a%22" + encode( sn ) + "%22&format=tab", max_lines_to_return );
434 private static List<String> getTaxonomyStringFromTaxonomyCode( final String code, final int max_lines_to_return )
436 return queryUniprot( "taxonomy/?query=mnemonic%3a%22" + encode( code ) + "%22&format=tab", max_lines_to_return );
439 private static List<UniProtTaxonomy> parseUniProtTaxonomy( final List<String> result ) throws IOException {
440 final List<UniProtTaxonomy> taxonomies = new ArrayList<UniProtTaxonomy>();
441 for( final String line : result ) {
442 if ( ForesterUtil.isEmpty( line ) ) {
443 // Ignore empty lines.
445 else if ( line.startsWith( "Taxon" ) ) {
446 final String[] items = line.split( "\t" );
447 if ( !( items[ 1 ].equalsIgnoreCase( "Mnemonic" ) && items[ 2 ].equalsIgnoreCase( "Scientific name" )
448 && items[ 3 ].equalsIgnoreCase( "Common name" ) && items[ 4 ].equalsIgnoreCase( "Synonym" )
449 && items[ 5 ].equalsIgnoreCase( "Other Names" ) && items[ 6 ].equalsIgnoreCase( "Reviewed" )
450 && items[ 7 ].equalsIgnoreCase( "Rank" ) && items[ 8 ].equalsIgnoreCase( "Lineage" ) ) ) {
451 throw new IOException( "Unreconized UniProt Taxonomy format: " + line );
455 if ( line.split( "\t" ).length > 4 ) {
456 taxonomies.add( new UniProtTaxonomy( line ) );
464 UNIPROT, EMBL, NCBI, NONE, REFSEQ;