From: cmzmasek@gmail.com Date: Thu, 24 Apr 2014 02:07:56 +0000 (+0000) Subject: inprogress X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=806774f7ed8880f0555b9433b5cd1799513993c9;p=jalview.git inprogress --- diff --git a/forester/java/src/org/forester/application/msa_compactor.java b/forester/java/src/org/forester/application/msa_compactor.java index 11b6ac8..2b3bad0 100644 --- a/forester/java/src/org/forester/application/msa_compactor.java +++ b/forester/java/src/org/forester/application/msa_compactor.java @@ -26,6 +26,7 @@ package org.forester.application; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -249,8 +250,9 @@ public class msa_compactor { System.out.println( "Output format : " + ( output_format == MSA_FORMAT.FASTA ? "fasta" : "phylip" ) ); } - System.out.println( "Step for output and re-aligning) : " + step ); - System.out.println( "Step for diagnostics reports : " + step_for_diagnostics ); + System.out.println( "Step for output and re-aligning) : " + ( step > 1 ? step : 1 ) ); + System.out.println( "Step for diagnostics reports : " + + ( step_for_diagnostics > 1 ? step_for_diagnostics : 1 ) ); System.out.println( "Report mean identity (\"MSA quality\") : " + report_aln_mean_identity ); if ( !norm ) { System.out.println( "Normalize : " + norm ); @@ -276,8 +278,13 @@ public class msa_compactor { } mc.setNorm( norm ); mc.setOutFileBase( out ); - mc.setStep( step ); - mc.removeWorstOffenders( worst_remove, true ); + if ( step > 1 ) { + mc.setStep( step ); + } + if ( step_for_diagnostics > 1 ) { + mc.setStepForDiagnostics( step_for_diagnostics ); + } + mc.removeWorstOffenders( worst_remove ); } else if ( av_gap > 0 ) { final MsaCompactor mc = new MsaCompactor( msa ); @@ -311,15 +318,28 @@ public class msa_compactor { mc.setPathToMafft( path_to_mafft ); } mc.setNorm( norm ); + mc.setReportAlnMeanIdentity( report_aln_mean_identity ); mc.setOutFileBase( out ); - mc.setStep( step ); - final List msa_props = mc.chart( step, realign, norm, true ); + if ( step > 1 ) { + mc.setStep( step ); + } + if ( step_for_diagnostics > 1 ) { + mc.setStepForDiagnostics( step_for_diagnostics ); + } + final List msa_props = mc.chart( step, realign, norm ); Chart.display( msa_props, initial_number_of_seqs ); } } + catch ( final IllegalArgumentException iae ) { + iae.printStackTrace(); //TODO remove me + ForesterUtil.fatalError( PRG_NAME, iae.getMessage() ); + } + catch ( final IOException ioe ) { + ioe.printStackTrace(); //TODO remove me + ForesterUtil.fatalError( PRG_NAME, ioe.getMessage() ); + } catch ( final Exception e ) { - e.printStackTrace(); - ForesterUtil.fatalError( PRG_NAME, e.getMessage() ); + ForesterUtil.unexpectedFatalError( PRG_NAME, e ); } } diff --git a/forester/java/src/org/forester/msa/DeleteableMsa.java b/forester/java/src/org/forester/msa/DeleteableMsa.java index da781b8..9f18924 100644 --- a/forester/java/src/org/forester/msa/DeleteableMsa.java +++ b/forester/java/src/org/forester/msa/DeleteableMsa.java @@ -64,7 +64,7 @@ public final class DeleteableMsa extends BasicMsa { final public void deleteGapOnlyColumns() { for( int col = getLength() - 1; col >= 0; --col ) { - if ( MsaMethods.calcGapSumPerColumn( this, col ) == getNumberOfSequences() ) { + if ( isAllGap( col ) ) { deleteColumn( col ); } } @@ -113,6 +113,16 @@ public final class DeleteableMsa extends BasicMsa { return new BasicSequence( getIdentifier( row ), getSequenceAsString( row ).toString(), getType() ); } + final public boolean isAllGap( final int col ) { + final int m_col = _mapped_col_positions[ col ]; + for( int j = 0; j < getNumberOfSequences(); ++j ) { + if ( super.getResidueAt( _mapped_row_positions[ j ], m_col ) != Sequence.GAP ) { + return false; + } + } + return true; + } + @Override final public void setIdentifier( final int row, final String id ) { checkRow( row ); diff --git a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java index 17e4397..8923aa7 100644 --- a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java +++ b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java @@ -63,18 +63,18 @@ public class MsaCompactor { private final String _maffts_opts = "--auto"; private int _min_length = -1; // - private DeleteableMsa _msa; + private DeleteableMsa _msa = null; private boolean _norm = true; - private File _out_file_base; + private File _out_file_base = null; private MSA_FORMAT _output_format = MSA_FORMAT.FASTA; - private String _path_to_mafft; + private String _path_to_mafft = null; // private boolean _realign = false; private final SortedSet _removed_seq_ids; private final File _removed_seqs_out_base = null; - private final boolean _report_aln_mean_identity = false; - private int _step = 1; - private int _step_for_diagnostics = 1; + private boolean _report_aln_mean_identity = false; + private int _step = -1; + private int _step_for_diagnostics = -1; static { NF_4.setRoundingMode( RoundingMode.HALF_UP ); NF_3.setRoundingMode( RoundingMode.HALF_UP ); @@ -85,53 +85,6 @@ public class MsaCompactor { _removed_seq_ids = new TreeSet(); } - public final List chart( final int step, - final boolean realign, - final boolean norm, - final boolean verbose ) throws IOException, InterruptedException { - final GapContribution stats[] = calcGapContribtionsStats( norm ); - final List to_remove_ids = new ArrayList(); - final List msa_props = new ArrayList(); - for( final GapContribution gap_gontribution : stats ) { - to_remove_ids.add( gap_gontribution.getId() ); - } - if ( verbose ) { - printTableHeader(); - } - int i = 0; - final int s = _msa.getNumberOfSequences(); - final int x = ForesterUtil.roundToInt( s / 20.0 ); - while ( _msa.getNumberOfSequences() > x ) { - final String id = to_remove_ids.get( i ); - //~_msa = MsaMethods.removeSequence( _msa, id ); - _msa.deleteRow( id ); - if ( ( s < 500 ) || ( ( step > 0 ) && ( ( ( i + 1 ) % step ) == 0 ) ) ) { - removeGapColumns(); - if ( realign && ( ( step > 0 ) && ( ( ( i + 1 ) % step ) == 0 ) ) ) { - realignWithMafft(); - msa_props.add( new MsaProperties( _msa ) ); - if ( verbose ) { - printMsaStats( id ); - } - if ( verbose ) { - System.out.print( "(realigned)" ); - } - } - else { - msa_props.add( new MsaProperties( _msa ) ); - if ( verbose ) { - printMsaStats( id ); - } - } - if ( verbose ) { - System.out.println(); - } - } - ++i; - } - return msa_props; - } - final public Msa getMsa() { return _msa; } @@ -157,10 +110,11 @@ public class MsaCompactor { removeGapColumns(); if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, verbose, id ); + printMsaStatsWriteOutfileAndRealign( _realign, id ); } else if ( verbose ) { - printMsaStats( id ); + final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + printMsaProperties( id, msa_prop ); } if ( verbose ) { System.out.println(); @@ -184,10 +138,11 @@ public class MsaCompactor { _msa.deleteRow( id ); removeGapColumns(); if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) || ( _msa.getLength() <= length ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, verbose, id ); + printMsaStatsWriteOutfileAndRealign( _realign, id ); } else if ( verbose ) { - printMsaStats( id ); + final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + printMsaProperties( id, msa_prop ); } if ( verbose ) { System.out.println(); @@ -196,31 +151,75 @@ public class MsaCompactor { } } - public final void removeWorstOffenders( final int to_remove, final boolean verbose ) throws IOException, - InterruptedException { + public final void removeWorstOffenders( final int to_remove ) throws IOException, InterruptedException { final GapContribution stats[] = calcGapContribtionsStats( _norm ); final List to_remove_ids = new ArrayList(); for( int j = 0; j < to_remove; ++j ) { to_remove_ids.add( stats[ j ].getId() ); _removed_seq_ids.add( stats[ j ].getId() ); } - if ( verbose ) { - printTableHeader(); - } + printTableHeader(); for( int i = 0; i < to_remove_ids.size(); ++i ) { final String id = to_remove_ids.get( i ); _msa.deleteRow( id ); removeGapColumns(); - if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) || ( i == ( to_remove_ids.size() - 1 ) ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, verbose, id ); + if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( i == ( to_remove_ids.size() - 1 ) ) ) { + printMsaStatsWriteOutfileAndRealign( _realign, id ); + System.out.println(); } - else if ( verbose ) { - printMsaStats( id ); + else if ( isPrintMsaStats( i ) ) { + final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + printMsaProperties( id, msa_prop ); + System.out.println(); } - if ( verbose ) { + } + } + + public final List chart( final int step, final boolean realign, final boolean norm ) + throws IOException, InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( norm ); + final List to_remove_ids = new ArrayList(); + final List msa_props = new ArrayList(); + for( final GapContribution gap_gontribution : stats ) { + to_remove_ids.add( gap_gontribution.getId() ); + } + printTableHeader(); + int i = 0; + final int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 ); + MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + msa_props.add( msa_prop ); + printMsaProperties( "", msa_prop ); + System.out.println(); + while ( _msa.getNumberOfSequences() > x ) { + final String id = to_remove_ids.get( i ); + _msa.deleteRow( id ); + if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) { + removeGapColumns(); + realignWithMafft(); + msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + msa_props.add( msa_prop ); + printMsaProperties( id, msa_prop ); + System.out.print( "(realigned)" ); System.out.println(); } + else if ( isPrintMsaStats( i ) ) { + removeGapColumns(); + msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + msa_props.add( msa_prop ); + printMsaProperties( id, msa_prop ); + System.out.println(); + } + ++i; } + return msa_props; + } + + private final boolean isPrintMsaStats( final int i ) { + return ( ( _step_for_diagnostics < 2 ) || ( ( ( i + 1 ) % _step_for_diagnostics ) == 0 ) ); + } + + private final boolean isPrintMsaStatsWriteOutfileAndRealign( final int i ) { + return ( ( _step < 2 ) || ( ( ( i + 1 ) % _step ) == 0 ) ); } public final void setGapRatio( final double gap_ratio ) { @@ -259,6 +258,10 @@ public class MsaCompactor { _step_for_diagnostics = step_for_diagnostics; } + public final void setReportAlnMeanIdentity( final boolean report_aln_mean_identity ) { + _report_aln_mean_identity = report_aln_mean_identity; + } + final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { final Double gr = MsaMethods.calcGapRatio( _msa ); final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" @@ -277,24 +280,6 @@ public class MsaCompactor { return ng; } - Phylogeny pi( final String matrix ) { - final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); - final int seed = 15; - final int n = 100; - final ResampleableMsa resampleable_msa = new ResampleableMsa( _msa ); - final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(), - n, - seed ); - final Phylogeny[] eval_phys = new Phylogeny[ n ]; - for( int i = 0; i < n; ++i ) { - resampleable_msa.resample( resampled_column_positions[ i ] ); - eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null ); - } - ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 ); - PhylogenyMethods.extractFastaInformation( master_phy ); - return master_phy; - } - private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) { final double gappiness[] = calcGappiness(); final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ]; @@ -331,10 +316,10 @@ public class MsaCompactor { return gappiness; } - private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, - final Msa msa, - final boolean write_matrix, - final String matrix_name ) { + private final Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, + final Msa msa, + final boolean write_matrix, + final String matrix_name ) { BasicSymmetricalDistanceMatrix m = null; switch ( pwd_distance_method ) { case KIMURA_DISTANCE: @@ -362,39 +347,55 @@ public class MsaCompactor { return phy; } - private StringBuilder msaStatsAsSB() { - final StringBuilder sb = new StringBuilder(); - sb.append( _msa.getNumberOfSequences() ); - sb.append( "\t" ); - sb.append( _msa.getLength() ); - sb.append( "\t" ); - sb.append( NF_4.format( MsaMethods.calcGapRatio( _msa ) ) ); - sb.append( "\t" ); - sb.append( NF_4.format( MsaMethods.calculateIdentityRatio( 0, _msa.getLength() - 1, _msa ).arithmeticMean() ) ); - return sb; + private final Phylogeny pi( final String matrix ) { + final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); + final int seed = 15; + final int n = 100; + final ResampleableMsa resampleable_msa = new ResampleableMsa( _msa ); + final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(), + n, + seed ); + final Phylogeny[] eval_phys = new Phylogeny[ n ]; + for( int i = 0; i < n; ++i ) { + resampleable_msa.resample( resampled_column_positions[ i ] ); + eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null ); + } + ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 ); + PhylogenyMethods.extractFastaInformation( master_phy ); + return master_phy; } - private final void printMsaStats( final String id ) { + private final static void printMsaProperties( final String id, final MsaProperties msa_properties ) { System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); System.out.print( "\t" ); - final StringBuilder sb = msaStatsAsSB(); + final StringBuilder sb = msaPropertiesAsSB( msa_properties ); System.out.print( sb ); System.out.print( "\t" ); } - final private void printMsaStatsWriteOutfileAndRealign( final boolean realign, - final boolean verbose, - final String id ) throws IOException, InterruptedException { + private final static StringBuilder msaPropertiesAsSB( final MsaProperties msa_properties ) { + final StringBuilder sb = new StringBuilder(); + sb.append( msa_properties.getNumberOfSequences() ); + sb.append( "\t" ); + sb.append( msa_properties.getLength() ); + sb.append( "\t" ); + sb.append( NF_4.format( msa_properties.getGapRatio() ) ); + if ( msa_properties.getAverageIdentityRatio() >= 0 ) { + sb.append( "\t" ); + sb.append( NF_4.format( msa_properties.getAverageIdentityRatio() ) ); + } + return sb; + } + + final private void printMsaStatsWriteOutfileAndRealign( final boolean realign, final String id ) + throws IOException, InterruptedException { if ( realign ) { realignWithMafft(); } - if ( verbose ) { - printMsaStats( id ); - } + final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); + printMsaProperties( id, msa_prop ); final String s = writeOutfile(); - if ( verbose ) { - System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) ); - } + System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) ); } final private void realignWithMafft() throws IOException, InterruptedException { @@ -418,7 +419,7 @@ public class MsaCompactor { w.close(); } - private String writeOutfile() throws IOException { + private final String writeOutfile() throws IOException { final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" ); //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" ); return s; diff --git a/forester/java/src/org/forester/msa_compactor/MsaProperties.java b/forester/java/src/org/forester/msa_compactor/MsaProperties.java index 57cc40c..deccb79 100644 --- a/forester/java/src/org/forester/msa_compactor/MsaProperties.java +++ b/forester/java/src/org/forester/msa_compactor/MsaProperties.java @@ -34,13 +34,6 @@ public final class MsaProperties { final private int _length; final private int _number_of_sequences; - public MsaProperties( final Msa msa ) { - _number_of_sequences = msa.getNumberOfSequences(); - _length = msa.getLength(); - _gap_ratio = MsaMethods.calcGapRatio( msa ); - _average_identity_ratio = MsaMethods.calculateIdentityRatio( 0, msa.getLength() - 1, msa ).arithmeticMean(); - } - public MsaProperties( final int number_of_sequences, final int length, final double gap_ratio, @@ -51,6 +44,18 @@ public final class MsaProperties { _average_identity_ratio = average_identity_ratio; } + public MsaProperties( final Msa msa, final boolean calculate_aln_mean_identity ) { + _number_of_sequences = msa.getNumberOfSequences(); + _length = msa.getLength(); + _gap_ratio = MsaMethods.calcGapRatio( msa ); + if ( calculate_aln_mean_identity ) { + _average_identity_ratio = MsaMethods.calculateIdentityRatio( 0, msa.getLength() - 1, msa ).arithmeticMean(); + } + else { + _average_identity_ratio = -1; + } + } + public final double getAverageIdentityRatio() { return _average_identity_ratio; }