X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;ds=sidebyside;f=forester%2Fjava%2Fsrc%2Forg%2Fforester%2Fws%2Fseqdb%2FSequenceDbWsTools.java;h=cecef9f27a251e6251eee06bbedbde48206d6202;hb=482c9a54ff2b83c947c43449c7b0d86dc9c8dafd;hp=f370e3d9e4eb720d9fffc8bc3150fb401e327a93;hpb=a648fae3c8d0402dbdafa379ff3d42bbea96633d;p=jalview.git diff --git a/forester/java/src/org/forester/ws/seqdb/SequenceDbWsTools.java b/forester/java/src/org/forester/ws/seqdb/SequenceDbWsTools.java index f370e3d..cecef9f 100644 --- a/forester/java/src/org/forester/ws/seqdb/SequenceDbWsTools.java +++ b/forester/java/src/org/forester/ws/seqdb/SequenceDbWsTools.java @@ -26,6 +26,7 @@ package org.forester.ws.seqdb; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; @@ -42,31 +43,44 @@ import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException; import org.forester.phylogeny.Phylogeny; import org.forester.phylogeny.PhylogenyNode; import org.forester.phylogeny.data.Accession; +import org.forester.phylogeny.data.Accession.Source; import org.forester.phylogeny.data.Annotation; import org.forester.phylogeny.data.Identifier; import org.forester.phylogeny.data.Sequence; import org.forester.phylogeny.data.Taxonomy; import org.forester.phylogeny.iterators.PhylogenyNodeIterator; import org.forester.util.ForesterUtil; -import org.forester.util.SequenceIdParser; +import org.forester.util.SequenceAccessionTools; public final class SequenceDbWsTools { - public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/"; - public final static String BASE_EMBL_DB_URL = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/"; - public final static String EMBL_DBS_EMBL = "embl"; - public final static String EMBL_DBS_REFSEQ_P = "refseqp"; - public final static String EMBL_DBS_REFSEQ_N = "refseqn"; - private final static String URL_ENC = "UTF-8"; - private final static boolean DEBUG = false; + public final static String EMBL_REFSEQ = "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch?db=REFSEQ&style=raw&id="; + public final static String BASE_UNIPROT_URL = "http://www.uniprot.org/"; + public final static String EMBL_DBS_EMBL = "embl"; + public final static String EMBL_DBS_REFSEQ_N = "refseqn"; + public final static String EMBL_DBS_REFSEQ_P = "refseqp"; + private final static boolean DEBUG = true; + private final static String URL_ENC = "UTF-8"; + public final static int DEFAULT_LINES_TO_RETURN = 4000; - private static List getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return ) - throws IOException { - final List result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return ); - if ( result.size() > 0 ) { - return parseUniProtTaxonomy( result ); + final static String extractFrom( final String target, final String a ) { + final int i_a = target.indexOf( a ); + return target.substring( i_a + a.length() ).trim(); + } + + final static String extractFromTo( final String target, final String a, final String b ) { + final int i_a = target.indexOf( a ); + final int i_b = target.indexOf( b ); + if ( ( i_a < 0 ) || ( i_b < i_a ) ) { + throw new IllegalArgumentException( "attempt to extract from \"" + target + "\" between \"" + a + + "\" and \"" + b + "\"" ); } - return null; + return target.substring( i_a + a.length(), i_b ).trim(); + } + + final static String extractTo( final String target, final String b ) { + final int i_b = target.indexOf( b ); + return target.substring( 0, i_b ).trim(); } public static List getTaxonomiesFromCommonNameStrict( final String cn, @@ -94,16 +108,6 @@ public final class SequenceDbWsTools { return null; } - private static List getTaxonomiesFromScientificName( final String sn, - final int max_taxonomies_return ) - throws IOException { - final List result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return ); - if ( result.size() > 0 ) { - return parseUniProtTaxonomy( result ); - } - return null; - } - /** * Does not return "sub-types". * For example, for "Mus musculus" only returns "Mus musculus" @@ -137,145 +141,60 @@ public final class SequenceDbWsTools { return null; } - public static SequenceDatabaseEntry obtainEmblEntry( final Identifier id, final int max_lines_to_return ) + public static SequenceDatabaseEntry obtainEmblEntry( final Accession id, final int max_lines_to_return ) throws IOException { final List lines = queryEmblDb( id, max_lines_to_return ); return EbiDbEntry.createInstanceFromPlainText( lines ); } - public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Identifier id, final int max_lines_to_return ) + public final static Accession obtainSeqAccession( final PhylogenyNode node ) { + Accession acc = SequenceAccessionTools.obtainFromSeqAccession( node ); + if ( !isAccessionAcceptable( acc ) ) { + acc = SequenceAccessionTools.obtainAccessorFromDataFields( node ); + } + return acc; + } + + public static SequenceDatabaseEntry obtainRefSeqEntryFromEmbl( final Accession id, final int max_lines_to_return ) throws IOException { - final List lines = queryEmblDb( id, max_lines_to_return ); + final List lines = queryEmblDbForRefSeqEntry( id, max_lines_to_return ); return EbiDbEntry.createInstanceFromPlainTextForRefSeq( lines ); } - public static SortedSet obtainSeqInformation( final Phylogeny phy, - final boolean ext_nodes_only, - final boolean allow_to_set_taxonomic_data, - final int lines_to_return ) throws IOException { + public final static void obtainSeqInformation( final boolean allow_to_set_taxonomic_data, + final int lines_to_return, + final SortedSet not_found, + final PhylogenyNode node ) throws IOException { + final Accession acc = obtainSeqAccession( node ); + if ( !isAccessionAcceptable( acc ) ) { + if ( node.isExternal() || !node.isEmpty() ) { + not_found.add( node.toString() ); + } + } + else { + addDataFromDbToNode( allow_to_set_taxonomic_data, lines_to_return, not_found, node, acc ); + } + } + + public final static void obtainSeqInformation( final boolean allow_to_set_taxonomic_data, + final SortedSet not_found, + final PhylogenyNode node ) throws IOException { + obtainSeqInformation( allow_to_set_taxonomic_data, DEFAULT_LINES_TO_RETURN, not_found, node ); + } + + public final static void obtainSeqInformation( final PhylogenyNode node ) throws IOException { + obtainSeqInformation( true, DEFAULT_LINES_TO_RETURN, new TreeSet(), node ); + } + + public final static SortedSet obtainSeqInformation( final Phylogeny phy, + final boolean ext_nodes_only, + final boolean allow_to_set_taxonomic_data, + final int lines_to_return ) throws IOException { final SortedSet not_found = new TreeSet(); for( final PhylogenyNodeIterator iter = phy.iteratorPostorder(); iter.hasNext(); ) { final PhylogenyNode node = iter.next(); - if ( ext_nodes_only && node.isInternal() ) { - continue; - } - String query = null; - Identifier id = null; - Db db = Db.NONE; - if ( node.getNodeData().isHasSequence() && ( node.getNodeData().getSequence().getAccession() != null ) - && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() ) - && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() ) - && node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "uniprot" ) ) { - query = node.getNodeData().getSequence().getAccession().getValue(); - db = Db.UNIPROT; - } - else if ( node.getNodeData().isHasSequence() - && ( node.getNodeData().getSequence().getAccession() != null ) - && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getSource() ) - && !ForesterUtil.isEmpty( node.getNodeData().getSequence().getAccession().getValue() ) - && ( node.getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "embl" ) || node - .getNodeData().getSequence().getAccession().getValue().toLowerCase().startsWith( "ebi" ) ) ) { - query = node.getNodeData().getSequence().getAccession().getValue(); - db = Db.EMBL; - } - else if ( !ForesterUtil.isEmpty( node.getName() ) ) { - if ( ( query = ForesterUtil.extractUniProtKbProteinSeqIdentifier( node ) ) != null ) { - db = Db.UNIPROT; - } - else if ( ( id = SequenceIdParser.parse( node.getName() ) ) != null ) { - if ( id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) { - db = Db.NCBI; - } - else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) { - db = Db.REFSEQ; - } - } - } - if ( db == Db.NONE ) { - not_found.add( node.getName() ); - } - SequenceDatabaseEntry db_entry = null; - if ( !ForesterUtil.isEmpty( query ) ) { - if ( db == Db.UNIPROT ) { - if ( DEBUG ) { - System.out.println( "uniprot: " + query ); - } - db_entry = obtainUniProtEntry( query, lines_to_return ); - } - if ( ( db == Db.EMBL ) || ( ( db == Db.UNIPROT ) && ( db_entry == null ) ) ) { - if ( DEBUG ) { - System.out.println( "embl: " + query ); - } - db_entry = obtainEmblEntry( new Identifier( query ), lines_to_return ); - if ( ( db == Db.UNIPROT ) && ( db_entry != null ) ) { - db = Db.EMBL; - } - } - } - else if ( ( db == Db.REFSEQ ) && ( id != null ) ) { - db_entry = obtainRefSeqEntryFromEmbl( id, lines_to_return ); - } - else if ( ( db == Db.NCBI ) && ( id != null ) ) { - db_entry = obtainEmblEntry( id, lines_to_return ); //TODO ? - } - if ( ( db_entry != null ) && !db_entry.isEmpty() ) { - final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence() - : new Sequence(); - if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) { - String type = null; - if ( db == Db.EMBL ) { - type = "embl"; - } - else if ( db == Db.UNIPROT ) { - type = "uniprot"; - } - else if ( db == Db.NCBI ) { - type = "ncbi"; - } - else if ( db == Db.REFSEQ ) { - type = "refseq"; - } - seq.setAccession( new Accession( db_entry.getAccession(), type ) ); - } - if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) { - seq.setName( db_entry.getSequenceName() ); - } - if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) { - final String gn = db_entry.getGeneName().replace( ' ', '_' ); - try { - seq.setSymbol( gn ); - } - catch ( PhyloXmlDataFormatException e ) { - // Eat this exception. - } - } - if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) { - // seq.addAnnotation( new Annotation( "GN", db_entry.getGeneName() ) ); - } - if ( db_entry.getGoTerms() != null && !db_entry.getGoTerms().isEmpty() ) { - for( final GoTerm go : db_entry.getGoTerms() ) { - seq.addAnnotation( new Annotation( go.getGoId().getId(), go.getName() ) ); - } - } - - final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy() - : new Taxonomy(); - if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) { - tax.setScientificName( db_entry.getTaxonomyScientificName() ); - } - if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) { - tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) ); - } - node.getNodeData().setTaxonomy( tax ); - node.getNodeData().setSequence( seq ); - } - else if ( db != Db.NONE ) { - not_found.add( node.getName() ); - } - try { - Thread.sleep( 10 );// Sleep for 10 ms - } - catch ( final InterruptedException ie ) { + if ( node.isExternal() || !ext_nodes_only ) { + obtainSeqInformation( allow_to_set_taxonomic_data, lines_to_return, not_found, node ); } } return not_found; @@ -323,22 +242,30 @@ public final class SequenceDbWsTools { return result; } - public static List queryEmblDb( final Identifier id, final int max_lines_to_return ) throws IOException { + public static List queryEmblDbForRefSeqEntry( final Accession id, final int max_lines_to_return ) + throws IOException { final StringBuilder url_sb = new StringBuilder(); - url_sb.append( BASE_EMBL_DB_URL ); - if ( ForesterUtil.isEmpty( id.getProvider() ) || id.getProvider().equalsIgnoreCase( Identifier.NCBI ) ) { - url_sb.append( SequenceDbWsTools.EMBL_DBS_EMBL ); + url_sb.append( EMBL_REFSEQ ); + return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() ); + } + + public static List queryEmblDb( final Accession id, final int max_lines_to_return ) throws IOException { + final StringBuilder url_sb = new StringBuilder(); + // url_sb.append( BASE_EMBL_DB_URL ); + if ( ForesterUtil.isEmpty( id.getSource() ) || ( id.getSource().equals( Source.NCBI.toString() ) ) ) { + url_sb.append( EMBL_DBS_EMBL ); url_sb.append( '/' ); } - else if ( id.getProvider().equalsIgnoreCase( Identifier.REFSEQ ) ) { - if ( id.getValue().toUpperCase().indexOf( 'P' ) == 1 ) { - url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_P ); - url_sb.append( '/' ); - } - else { - url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_N ); - url_sb.append( '/' ); - } + else if ( id.getSource().equals( Source.REFSEQ.toString() ) ) { + url_sb.append( EMBL_REFSEQ ); + // if ( id.getValue().toUpperCase().indexOf( 'P' ) == 1 ) { + // url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_P ); + // url_sb.append( '/' ); + // } + // else { + // url_sb.append( SequenceDbWsTools.EMBL_DBS_REFSEQ_N ); + // url_sb.append( '/' ); + // } } return queryDb( id.getValue(), max_lines_to_return, url_sb.toString() ); } @@ -347,10 +274,122 @@ public final class SequenceDbWsTools { return queryDb( query, max_lines_to_return, BASE_UNIPROT_URL ); } + private static void addDataFromDbToNode( final boolean allow_to_set_taxonomic_data, + final int lines_to_return, + final SortedSet not_found, + final PhylogenyNode node, + final Accession acc ) throws IOException { + SequenceDatabaseEntry db_entry = null; + final String query = acc.getValue(); + if ( acc.getSource().equals( Source.UNIPROT.toString() ) ) { + if ( DEBUG ) { + System.out.println( "uniprot: " + query ); + } + try { + db_entry = obtainUniProtEntry( query, lines_to_return ); + } + catch ( final FileNotFoundException e ) { + // Eat this, and move to next. + } + } + else if ( acc.getSource().equals( Source.EMBL.toString() ) ) { + if ( DEBUG ) { + System.out.println( "embl: " + query ); + } + try { + db_entry = obtainEmblEntry( new Accession( query ), lines_to_return ); + } + catch ( final FileNotFoundException e ) { + // Eat this, and move to next. + } + } + else if ( acc.getSource().equals( Source.REFSEQ.toString() ) ) { + if ( DEBUG ) { + System.out.println( "refseq: " + query ); + } + try { + db_entry = obtainRefSeqEntryFromEmbl( new Accession( query ), lines_to_return ); + } + catch ( final FileNotFoundException e ) { + // Eat this, and move to next. + } + } + if ( ( db_entry != null ) && !db_entry.isEmpty() ) { + final Sequence seq = node.getNodeData().isHasSequence() ? node.getNodeData().getSequence() : new Sequence(); + if ( !ForesterUtil.isEmpty( db_entry.getAccession() ) ) { + seq.setAccession( new Accession( db_entry.getAccession(), acc.getSource() ) ); + } + if ( !ForesterUtil.isEmpty( db_entry.getSequenceName() ) ) { + seq.setName( db_entry.getSequenceName() ); + } + if ( !ForesterUtil.isEmpty( db_entry.getGeneName() ) ) { + seq.setGeneName( db_entry.getGeneName() ); + } + if ( !ForesterUtil.isEmpty( db_entry.getSequenceSymbol() ) ) { + try { + seq.setSymbol( db_entry.getSequenceSymbol() ); + } + catch ( final PhyloXmlDataFormatException e ) { + // Eat this exception. + } + } + if ( ( db_entry.getGoTerms() != null ) && !db_entry.getGoTerms().isEmpty() ) { + for( final GoTerm go : db_entry.getGoTerms() ) { + final Annotation ann = new Annotation( go.getGoId().getId() ); + ann.setDesc( go.getName() ); + seq.addAnnotation( ann ); + } + } + if ( ( db_entry.getCrossReferences() != null ) && !db_entry.getCrossReferences().isEmpty() ) { + for( final Accession x : db_entry.getCrossReferences() ) { + seq.addCrossReference( x ); + } + } + final Taxonomy tax = node.getNodeData().isHasTaxonomy() ? node.getNodeData().getTaxonomy() : new Taxonomy(); + if ( !ForesterUtil.isEmpty( db_entry.getTaxonomyScientificName() ) ) { + tax.setScientificName( db_entry.getTaxonomyScientificName() ); + } + if ( allow_to_set_taxonomic_data && !ForesterUtil.isEmpty( db_entry.getTaxonomyIdentifier() ) ) { + tax.setIdentifier( new Identifier( db_entry.getTaxonomyIdentifier(), "uniprot" ) ); + } + node.getNodeData().setTaxonomy( tax ); + node.getNodeData().setSequence( seq ); + } + else { + if ( node.isExternal() || !node.isEmpty() ) { + not_found.add( node.toString() ); + } + } + try { + Thread.sleep( 10 );// Sleep for 10 ms + } + catch ( final InterruptedException ie ) { + } + } + private static String encode( final String str ) throws UnsupportedEncodingException { return URLEncoder.encode( str.trim(), URL_ENC ); } + private static List getTaxonomiesFromCommonName( final String cn, final int max_taxonomies_return ) + throws IOException { + final List result = getTaxonomyStringFromCommonName( cn, max_taxonomies_return ); + if ( result.size() > 0 ) { + return parseUniProtTaxonomy( result ); + } + return null; + } + + private static List getTaxonomiesFromScientificName( final String sn, + final int max_taxonomies_return ) + throws IOException { + final List result = getTaxonomyStringFromScientificName( sn, max_taxonomies_return ); + if ( result.size() > 0 ) { + return parseUniProtTaxonomy( result ); + } + return null; + } + private static List getTaxonomyStringFromCommonName( final String cn, final int max_lines_to_return ) throws IOException { return queryUniprot( "taxonomy/?query=common%3a%22" + encode( cn ) + "%22&format=tab", max_lines_to_return ); @@ -371,6 +410,13 @@ public final class SequenceDbWsTools { return queryUniprot( "taxonomy/?query=mnemonic%3a%22" + encode( code ) + "%22&format=tab", max_lines_to_return ); } + private final static boolean isAccessionAcceptable( final Accession acc ) { + return ( !( ( acc == null ) || ForesterUtil.isEmpty( acc.getSource() ) || ForesterUtil.isEmpty( acc.getValue() ) || ( ( acc + .getSource().equals( Source.UNIPROT.toString() ) ) + && ( acc.getSource().toString().equals( Source.EMBL.toString() ) ) && ( acc.getSource().toString() + .equals( Source.REFSEQ.toString() ) ) ) ) ); + } + private static List parseUniProtTaxonomy( final List result ) throws IOException { final List taxonomies = new ArrayList(); for( final String line : result ) { @@ -394,8 +440,4 @@ public final class SequenceDbWsTools { } return taxonomies; } - - public enum Db { - UNIPROT, EMBL, NCBI, NONE, REFSEQ; - } }