2 // forester -- software libraries and applications
3 // for genomics and evolutionary biology research.
5 // Copyright (C) 2010 Christian M Zmasek
6 // Copyright (C) 2010 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // Contact: phylosoft @ gmail . com
24 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
26 package org.forester.ws.seqdb;
28 import java.io.BufferedReader;
29 import java.io.FileNotFoundException;
30 import java.io.IOException;
31 import java.io.InputStreamReader;
32 import java.io.UnsupportedEncodingException;
34 import java.net.URLConnection;
35 import java.net.URLEncoder;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.SortedSet;
39 import java.util.TreeSet;
41 import org.forester.go.GoTerm;
42 import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException;
43 import org.forester.phylogeny.Phylogeny;
44 import org.forester.phylogeny.PhylogenyNode;
45 import org.forester.phylogeny.data.Accession;
46 import org.forester.phylogeny.data.Annotation;
47 import org.forester.phylogeny.data.Identifier;
48 import org.forester.phylogeny.data.Sequence;
49 import org.forester.phylogeny.data.Taxonomy;
50 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
51 import org.forester.util.ForesterUtil;
52 import org.forester.util.SequenceAccessionTools;
54 public final class SequenceDbWsTools {
56 public final static String BASE_EMBL_DB_URL = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/";
57 public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/";
58 public final static String EMBL_DBS_EMBL = "embl";
59 public final static String EMBL_DBS_REFSEQ_N = "refseqn";
60 public final static String EMBL_DBS_REFSEQ_P = "refseqp";
61 private final static boolean DEBUG = true;
62 private final static String URL_ENC = "UTF-8";
64 public static List<UniProtTaxonomy> getTaxonomiesFromCommonNameStrict( final String cn,
65 final int max_taxonomies_return )
67 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromCommonName( cn, max_taxonomies_return );
68 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
69 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
70 for( final UniProtTaxonomy taxonomy : taxonomies ) {
71 if ( taxonomy.getCommonName().equalsIgnoreCase( cn ) ) {
72 filtered_taxonomies.add( taxonomy );
75 return filtered_taxonomies;
80 public static List<UniProtTaxonomy> getTaxonomiesFromId( final String id, final int max_taxonomies_return )
82 final List<String> result = getTaxonomyStringFromId( id, max_taxonomies_return );
83 if ( result.size() > 0 ) {
84 return parseUniProtTaxonomy( result );
90 * Does not return "sub-types".
91 * For example, for "Mus musculus" only returns "Mus musculus"
92 * and not "Mus musculus", "Mus musculus bactrianus", ...
95 public static List<UniProtTaxonomy> getTaxonomiesFromScientificNameStrict( final String sn,
96 final int max_taxonomies_return )
98 final List<UniProtTaxonomy> taxonomies = getTaxonomiesFromScientificName( sn, max_taxonomies_return );
99 if ( ( taxonomies != null ) && ( taxonomies.size() > 0 ) ) {
100 final List<UniProtTaxonomy> filtered_taxonomies = new ArrayList<UniProtTaxonomy>();
101 for( final UniProtTaxonomy taxonomy : taxonomies ) {
102 if ( taxonomy.getScientificName().equalsIgnoreCase( sn ) ) {
103 filtered_taxonomies.add( taxonomy );
106 return filtered_taxonomies;
111 public static List<UniProtTaxonomy> getTaxonomiesFromTaxonomyCode( final String code,
112 final int max_taxonomies_return )
114 final String my_code = new String( code );
115 final List<String> result = getTaxonomyStringFromTaxonomyCode( my_code, max_taxonomies_return );
116 if ( result.size() > 0 ) {
117 return parseUniProtTaxonomy( result );
122 public static SequenceDatabaseEntry obtainEmblEntry( final Accession id, final int max_lines_to_return )
124 final List<String> lines = queryEmblDb( id, max_lines_to_return );
125 return EbiDbEntry.createInstanceFromPlainText( lines );
128 public final static Accession obtainFromSeqAccession( final PhylogenyNode node ) {
129 Accession acc = SequenceAccessionTools.obtainFromSeqAccession( node );
130 if ( !isAccessionAcceptable( acc ) ) {
131 acc = SequenceAccessionTools.obtainAccessorFromDataFields( node );
136 public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Accession id, final int max_lines_to_return )
138 final List<String> lines = queryEmblDb( id, max_lines_to_return );
139 return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines );
142 public final static void obtainSeqInformation( final boolean allow_to_set_taxonomic_data,
143 final int lines_to_return,
144 final SortedSet<String> not_found,
145 final PhylogenyNode node ) throws IOException {
146 final Accession acc = obtainFromSeqAccession( node );
147 if ( !isAccessionAcceptable( acc ) ) {
148 if ( node.isExternal() || !node.isEmpty() ) {
149 not_found.add( node.toString() );
153 addDataFromDbToNode( allow_to_set_taxonomic_data, lines_to_return, not_found, node, acc );
157 public final static SortedSet<String> obtainSeqInformation( final Phylogeny phy,
158 final boolean ext_nodes_only,
159 final boolean allow_to_set_taxonomic_data,
160 final int lines_to_return ) throws IOException {
161 final SortedSet<String> not_found = new TreeSet<String>();
162 for( final PhylogenyNodeIterator iter = phy.iteratorPostorder(); iter.hasNext(); ) {
163 final PhylogenyNode node = iter.next();
164 if ( node.isExternal() || !ext_nodes_only ) {
165 obtainSeqInformation( allow_to_set_taxonomic_data, lines_to_return, not_found, node );
171 public static SequenceDatabaseEntry obtainUniProtEntry( final String query, final int max_lines_to_return )
173 final List<String> lines = queryUniprot( "uniprot/" + query + ".txt", max_lines_to_return );
174 return UniProtEntry.createInstanceFromPlainText( lines );
177 public static List<String> queryDb( final String query, int max_lines_to_return, final String base_url )
179 if ( ForesterUtil.isEmpty( query ) ) {
180 throw new IllegalArgumentException( "illegal attempt to use empty query " );
182 if ( max_lines_to_return < 1 ) {
183 max_lines_to_return = 1;
185 final URL url = new URL( base_url + query );
187 System.out.println( "url: " + url.toString() );
189 final URLConnection urlc = url.openConnection();
190 final BufferedReader in = new BufferedReader( new InputStreamReader( urlc.getInputStream() ) );
192 final List<String> result = new ArrayList<String>();
193 while ( ( line = in.readLine() ) != null ) {
195 System.out.println( line );
198 if ( result.size() > max_lines_to_return ) {
204 // To prevent accessing online dbs in too quick succession.
207 catch ( final InterruptedException e ) {
213 public static List<String> queryEmblDb( final Accession id, final int max_lines_to_return ) throws IOException {
214 final StringBuilder url_sb = new StringBuilder();
215 url_sb.append( BASE_EMBL_DB_URL );
216 if ( ForesterUtil.isEmpty( id.getSource() ) || ( id.getSource() == Accession.NCBI ) ) {
217 url_sb.append( SequenceDbWsTools.EMBL_DBS_EMBL );
218 url_sb.append( '/' );
220 else if ( id.getSource() == Accession.REFSEQ ) {
221 if ( id.getValue().toUpperCase().indexOf( 'P' ) == 1 ) {
222 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_P );
223 url_sb.append( '/' );
226 url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_N );
227 url_sb.append( '/' );
230 return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() );
233 public static List<String> queryUniprot( final String query, final int max_lines_to_return ) throws IOException {
234 return queryDb( query, max_lines_to_return, BASE_UNIPROT_URL );
237 private static void addDataFromDbToNode( final boolean allow_to_set_taxonomic_data,
238 final int lines_to_return,
239 final SortedSet<String> not_found,
240 final PhylogenyNode node,
241 final Accession acc ) throws IOException {
242 SequenceDatabaseEntry db_entry = null;
243 final String query = acc.getValue();
244 if ( acc.getSource() == Accession.UNIPROT ) {
246 System.out.println( "uniprot: " + query );
249 db_entry = obtainUniProtEntry( query, lines_to_return );
251 catch ( final FileNotFoundException e ) {
252 // Eat this, and move to next.
255 else if ( acc.getSource() == Accession.EMBL ) {
257 System.out.println( "embl: " + query );
260 db_entry = obtainEmblEntry( new Accession( query ), lines_to_return );
262 catch ( final FileNotFoundException e ) {
263 // Eat this, and move to next.
266 else if ( acc.getSource() == Accession.REFSEQ ) {
268 System.out.println( "refseq: " + query );
271 db_entry = obtainRefSeqEntryFromEmbl( new Accession( query ), lines_to_return );
273 catch ( final FileNotFoundException e ) {
274 // Eat this, and move to next.
277 if ( ( db_entry != null ) && !db_entry.isEmpty() ) {
278 final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence() : new Sequence();
279 if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) {
280 seq.setAccession( new Accession( db_entry.getAccession(), acc.getSource() ) );
282 if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) {
283 seq.setName( db_entry.getSequenceName() );
285 if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) {
286 seq.setGeneName( db_entry.getGeneName() );
288 if ( !ForesterUtil.isEmpty( db_entry.getSequenceSymbol() ) ) {
290 seq.setSymbol( db_entry.getSequenceSymbol() );
292 catch ( final PhyloXmlDataFormatException e ) {
293 // Eat this exception.
296 if ( ( db_entry.getGoTerms() != null ) && !db_entry.getGoTerms().isEmpty() ) {
297 for( final GoTerm go : db_entry.getGoTerms() ) {
298 final Annotation ann = new Annotation( go.getGoId().getId() );
299 ann.setDesc( go.getName() );
300 seq.addAnnotation( ann );
303 if ( ( db_entry.getCrossReferences() != null ) && !db_entry.getCrossReferences().isEmpty() ) {
304 for( final Accession x : db_entry.getCrossReferences() ) {
305 seq.addCrossReference( x );
308 final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy() : new Taxonomy();
309 if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) {
310 tax.setScientificName( db_entry.getTaxonomyScientificName() );
312 if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) {
313 tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) );
315 node.getNodeData().setTaxonomy( tax );
316 node.getNodeData().setSequence( seq );
319 if ( node.isExternal() || !node.isEmpty() ) {
320 not_found.add( node.toString() );
324 Thread.sleep( 10 );// Sleep for 10 ms
326 catch ( final InterruptedException ie ) {
330 private static String encode( final String str ) throws UnsupportedEncodingException {
331 return URLEncoder.encode( str.trim(), URL_ENC );
334 private static List<UniProtTaxonomy> getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return )
336 final List<String> result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return );
337 if ( result.size() > 0 ) {
338 return parseUniProtTaxonomy( result );
343 private static List<UniProtTaxonomy> getTaxonomiesFromScientificName( final String sn,
344 final int max_taxonomies_return )
346 final List<String> result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return );
347 if ( result.size() > 0 ) {
348 return parseUniProtTaxonomy( result );
353 private static List<String> getTaxonomyStringFromCommonName( final String cn, final int max_lines_to_return )
355 return queryUniprot( "taxonomy/?query=common%3a%22" + encode( cn ) + "%22&format=tab", max_lines_to_return );
358 private static List<String> getTaxonomyStringFromId( final String id, final int max_lines_to_return )
360 return queryUniprot( "taxonomy/?query=id%3a%22" + encode( id ) + "%22&format=tab", max_lines_to_return );
363 private static List<String> getTaxonomyStringFromScientificName( final String sn, final int max_lines_to_return )
365 return queryUniprot( "taxonomy/?query=scientific%3a%22" + encode( sn ) + "%22&format=tab", max_lines_to_return );
368 private static List<String> getTaxonomyStringFromTaxonomyCode( final String code, final int max_lines_to_return )
370 return queryUniprot( "taxonomy/?query=mnemonic%3a%22" + encode( code ) + "%22&format=tab", max_lines_to_return );
373 private final static boolean isAccessionAcceptable( final Accession acc ) {
374 return ( !( ( acc == null ) || ForesterUtil.isEmpty( acc.getSource() ) || ForesterUtil.isEmpty( acc.getValue() ) || ( ( acc
375 .getSource() != Accession.UNIPROT ) && ( acc.getSource() != Accession.EMBL ) && ( acc.getSource() != Accession.REFSEQ ) ) ) );
378 private static List<UniProtTaxonomy> parseUniProtTaxonomy( final List<String> result ) throws IOException {
379 final List<UniProtTaxonomy> taxonomies = new ArrayList<UniProtTaxonomy>();
380 for( final String line : result ) {
381 if ( ForesterUtil.isEmpty( line ) ) {
382 // Ignore empty lines.
384 else if ( line.startsWith( "Taxon" ) ) {
385 final String[] items = line.split( "\t" );
386 if ( !( items[ 1 ].equalsIgnoreCase( "Mnemonic" ) && items[ 2 ].equalsIgnoreCase( "Scientific name" )
387 && items[ 3 ].equalsIgnoreCase( "Common name" ) && items[ 4 ].equalsIgnoreCase( "Synonym" )
388 && items[ 5 ].equalsIgnoreCase( "Other Names" ) && items[ 6 ].equalsIgnoreCase( "Reviewed" )
389 && items[ 7 ].equalsIgnoreCase( "Rank" ) && items[ 8 ].equalsIgnoreCase( "Lineage" ) ) ) {
390 throw new IOException( "Unreconized UniProt Taxonomy format: " + line );
394 if ( line.split( "\t" ).length > 4 ) {
395 taxonomies.add( new UniProtTaxonomy( line ) );