X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=forester%2Fjava%2Fsrc%2Forg%2Fforester%2Fmsa_compactor%2FMsaCompactor.java;h=8a13b5cc6bdb2ff28954e78fdb04751ad8c850d9;hb=cb49ee5684c6907b3161db82ff9aea72961b8548;hp=3ade90faebcb4be5fc2706fd000fef4b1dab882d;hpb=0465b8ebe16d57902022bba6f90f25dd1a18e656;p=jalview.git diff --git a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java index 3ade90f..8a13b5c 100644 --- a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java +++ b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java @@ -24,6 +24,7 @@ package org.forester.msa_compactor; +import java.awt.Color; import java.io.File; import java.io.IOException; import java.io.Writer; @@ -36,11 +37,16 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; +import org.forester.archaeopteryx.Archaeopteryx; +import org.forester.archaeopteryx.Configuration; import org.forester.evoinference.distance.NeighborJoiningF; import org.forester.evoinference.distance.PairwiseDistanceCalculator; import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD; import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix; import org.forester.evoinference.tools.BootstrapResampler; +import org.forester.io.parsers.nhx.NHXParser.TAXONOMY_EXTRACTION; +import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException; +import org.forester.io.parsers.util.ParserUtils; import org.forester.io.writers.SequenceWriter; import org.forester.io.writers.SequenceWriter.SEQ_FORMAT; import org.forester.msa.DeleteableMsa; @@ -52,38 +58,44 @@ import org.forester.msa.MsaMethods; import org.forester.msa.ResampleableMsa; import org.forester.phylogeny.Phylogeny; import org.forester.phylogeny.PhylogenyMethods; -import org.forester.sequence.Sequence; +import org.forester.phylogeny.PhylogenyMethods.DESCENDANT_SORT_PRIORITY; +import org.forester.phylogeny.PhylogenyNode; +import org.forester.phylogeny.data.NodeVisualData; +import org.forester.phylogeny.data.NodeVisualData.NodeFill; +import org.forester.phylogeny.data.NodeVisualData.NodeShape; +import org.forester.phylogeny.iterators.PhylogenyNodeIterator; +import org.forester.sequence.MolecularSequence; import org.forester.tools.ConfidenceAssessor; +import org.forester.util.BasicDescriptiveStatistics; +import org.forester.util.DescriptiveStatistics; import org.forester.util.ForesterUtil; public class MsaCompactor { - final private static NumberFormat NF_3 = new DecimalFormat( "#.###" ); - final private static NumberFormat NF_4 = new DecimalFormat( "#.####" ); - private double _gap_ratio = -1; + final private static NumberFormat NF_1 = new DecimalFormat( "0.#" ); + final private static NumberFormat NF_3 = new DecimalFormat( "0.###" ); + final private static NumberFormat NF_4 = new DecimalFormat( "0.####" ); + private boolean _calculate_shannon_entropy = false; // - private final String _maffts_opts = "--auto"; - private int _min_length = -1; + private String _infile_name = null; + private final short _longest_id_length; // - private DeleteableMsa _msa = null; - private boolean _norm = true; - private File _out_file_base = null; - private MSA_FORMAT _output_format = MSA_FORMAT.FASTA; - private String _path_to_mafft = null; + private String _maffts_opts = "--auto"; + private DeleteableMsa _msa = null; + private boolean _normalize_for_effective_seq_length = true; + private File _out_file_base = null; + private MSA_FORMAT _output_format = MSA_FORMAT.FASTA; + private String _path_to_mafft = null; + private boolean _phylogentic_inference = false; // - private boolean _realign = false; - private final SortedSet _removed_seq_ids; - private File _removed_seqs_out_base = null; - - public final void setRemovedSeqsOutBase( final File removed_seqs_out_base ) { - _removed_seqs_out_base = removed_seqs_out_base; - } - private boolean _report_aln_mean_identity = false; - private int _step = -1; - private int _step_for_diagnostics = -1; - private final short _longest_id_length; - private final ArrayList _removed_seqs; + private boolean _realign = false; + private final SortedSet _removed_seq_ids; + private final ArrayList _removed_seqs; + private File _removed_seqs_out_base = null; + private int _step = -1; + private int _step_for_diagnostics = -1; static { + NF_1.setRoundingMode( RoundingMode.HALF_UP ); NF_4.setRoundingMode( RoundingMode.HALF_UP ); NF_3.setRoundingMode( RoundingMode.HALF_UP ); } @@ -92,163 +104,391 @@ public class MsaCompactor { _msa = msa; _removed_seq_ids = new TreeSet(); _longest_id_length = _msa.determineMaxIdLength(); - _removed_seqs = new ArrayList(); + _removed_seqs = new ArrayList(); + } + + public final Phylogeny calcTree() { + final Phylogeny phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, false, "" ); + PhylogenyMethods.midpointRoot( phy ); + PhylogenyMethods.orderAppearance( phy.getRoot(), true, true, DESCENDANT_SORT_PRIORITY.NODE_NAME ); + final boolean x = PhylogenyMethods.extractFastaInformation( phy ); + if ( !x ) { + final PhylogenyNodeIterator it = phy.iteratorExternalForward(); + while ( it.hasNext() ) { + final PhylogenyNode n = it.next(); + final String name = n.getName().trim(); + if ( !ForesterUtil.isEmpty( name ) ) { + try { + ParserUtils.extractTaxonomyDataFromNodeName( n, TAXONOMY_EXTRACTION.AGGRESSIVE ); + } + catch ( final PhyloXmlDataFormatException e ) { + // Ignore. + } + } + } + } + return phy; + } + + public final List chart( final int step, final boolean realign, final boolean normalize_for_effective_seq_length ) + throws IOException, InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( normalize_for_effective_seq_length ); + final List to_remove_ids = new ArrayList(); + final List msa_props = new ArrayList(); + for( final GapContribution gap_gontribution : stats ) { + to_remove_ids.add( gap_gontribution.getId() ); + } + Phylogeny phy = null; + if ( _phylogentic_inference ) { + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + phy = calcTree(); + addSeqs2Tree( _msa, phy ); + } + if ( !_realign ) { + _step = -1; + } + int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 10.0 ); + if ( x < 2 ) { + x = 2; + } + MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printTableHeader(); + printMsaProperties( msa_prop ); + System.out.println(); + int i = 0; + while ( _msa.getNumberOfSequences() > x ) { + final String id = to_remove_ids.get( i ); + _msa.deleteRow( id, false ); + if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) { + removeGapColumns(); + realignWithMafft(); + msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.print( "(realigned)" ); + System.out.println(); + } + else if ( isPrintMsaStats( i ) ) { + removeGapColumns(); + msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.println(); + } + ++i; + } + + if ( _phylogentic_inference ) { + decorateTree( phy, msa_props, true ); + displayTree( phy ); + } + return msa_props; + } + + private final static void addSeqs2Tree( final Msa msa, final Phylogeny phy ) { + for( int i = 0; i < msa.getNumberOfSequences(); ++i ) { + final MolecularSequence seq = msa.getSequence( i ); + final String seq_name = seq.getIdentifier(); + final PhylogenyNode n = phy.getNode( seq_name ); + if ( !n.getNodeData().isHasSequence() ) { + n.getNodeData().addSequence( new org.forester.phylogeny.data.Sequence() ); + } + else { + throw new IllegalArgumentException( "this should not have happened" ); + } + n.getNodeData().getSequence().setMolecularSequence( seq.getMolecularSequenceAsString() ); + n.getNodeData().getSequence().setMolecularSequenceAligned( true ); + n.getNodeData().getSequence().setName( seq_name ); + } + } + + private final static void decorateTree( final Phylogeny phy, + final List msa_props, + final boolean chart_only ) { + final BasicDescriptiveStatistics length_stats = new BasicDescriptiveStatistics(); + for( int i = 0; i < msa_props.size(); ++i ) { + final MsaProperties msa_prop = msa_props.get( i ); + final String id = msa_prop.getRemovedSeq(); + if ( !ForesterUtil.isEmpty( id ) ) { + length_stats.addValue( msa_prop.getLength() ); + } + } + final double mean = length_stats.arithmeticMean(); + final double min = length_stats.getMin(); + final double max = length_stats.getMax(); + final Color min_color = new Color( 0, 255, 0 ); + final Color max_color = new Color( 255, 0, 0 ); + final Color mean_color = new Color( 255, 255, 0 ); + final PhylogenyNodeIterator it = phy.iteratorExternalForward(); + if ( chart_only ) { + while ( it.hasNext() ) { + final NodeVisualData vis = new NodeVisualData(); + vis.setFillType( NodeFill.SOLID ); + vis.setShape( NodeShape.RECTANGLE ); + vis.setNodeColor( min_color ); + it.next().getNodeData().setNodeVisualData( vis ); + } + } + for( int i = 0; i < msa_props.size(); ++i ) { + final MsaProperties msa_prop = msa_props.get( i ); + final String id = msa_prop.getRemovedSeq(); + if ( !ForesterUtil.isEmpty( id ) ) { + final PhylogenyNode n = phy.getNode( id ); + n.setName( n.getName() + " [" + i + "]" ); + if ( !chart_only ) { + final NodeVisualData vis = new NodeVisualData(); + vis.setFillType( NodeFill.SOLID ); + vis.setShape( NodeShape.RECTANGLE ); + vis.setNodeColor( ForesterUtil.calcColor( msa_prop.getLength(), min, max, mean_color, max_color ) ); + n.getNodeData().setNodeVisualData( vis ); + } + else { + n.getNodeData() + .getNodeVisualData() + .setNodeColor( ForesterUtil.calcColor( msa_prop.getLength(), + min, + max, + mean, + min_color, + max_color, + mean_color ) ); + } + } + } + } + + final public void deleteGapColumns( final double max_allowed_gap_ratio ) { + _msa.deleteGapColumns( max_allowed_gap_ratio ); + } + + public final void displayTree( final Phylogeny phy ) { + final Configuration config = new Configuration(); + config.setDisplayAsPhylogram( true ); + config.setUseStyle( true ); + config.setDisplayTaxonomyCode( false ); + config.setDisplayTaxonomyCommonNames( false ); + config.setDisplayTaxonomyScientificNames( false ); + config.setDisplaySequenceNames( false ); + config.setDisplaySequenceSymbols( false ); + config.setDisplayGeneNames( false ); + config.setDisplayMultipleSequenceAlignment( true ); + config.setShowScale( true ); + config.setAddTaxonomyImagesCB( false ); + config.setBaseFontSize( 9 ); + config.setBaseFontFamilyName( "Arial" ); + Archaeopteryx.createApplication( phy, config, _infile_name ); } final public Msa getMsa() { return _msa; } - final public SortedSet getRemovedSeqIds() { - return _removed_seq_ids; + public final void removeSequencesByMinimalLength( final int min_effective_length ) throws IOException { + _msa = DeleteableMsa.createInstance( MsaMethods.removeSequencesByMinimalLength( _msa, min_effective_length ) ); + removeGapColumns(); + final String s = writeOutfile(); + final DescriptiveStatistics msa_stats = MsaMethods.calculateEffectiveLengthStatistics( _msa ); + System.out.println( "Output MSA : " + s ); + System.out.println( " MSA length : " + _msa.getLength() ); + System.out.println( " Number of sequences : " + _msa.getNumberOfSequences() ); + System.out.println( " Median sequence length : " + NF_1.format( msa_stats.median() ) ); + System.out.println( " Mean sequence length : " + NF_1.format( msa_stats.arithmeticMean() ) ); + System.out.println( " Max sequence length : " + ( ( int ) msa_stats.getMax() ) ); + System.out.println( " Min sequence length : " + ( ( int ) msa_stats.getMin() ) ); + System.out.println( " Gap ratio : " + NF_4.format( MsaMethods.calcGapRatio( _msa ) ) ); + System.out.println( " Normalized Shannon Entropy (entn21): " + + NF_4.format( MsaMethods.calcNormalizedShannonsEntropy( 21, _msa ) ) ); + System.out.println(); } - public final void removeViaGapAverage( final double mean_gapiness ) throws IOException, InterruptedException { - final GapContribution stats[] = calcGapContribtionsStats( _norm ); + public final List removeViaGapAverage( final double mean_gapiness ) throws IOException, + InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( _normalize_for_effective_seq_length ); final List to_remove_ids = new ArrayList(); + final List msa_props = new ArrayList(); for( final GapContribution gap_gontribution : stats ) { to_remove_ids.add( gap_gontribution.getId() ); } + Phylogeny phy = null; + if ( _phylogentic_inference ) { + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + phy = calcTree(); + addSeqs2Tree( _msa, phy ); + } printTableHeader(); + MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.println(); int i = 0; while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) { final String id = to_remove_ids.get( i ); _removed_seq_ids.add( id ); - final Sequence deleted = _msa.deleteRow( id, true ); + final MolecularSequence deleted = _msa.deleteRow( id, true ); _removed_seqs.add( deleted ); removeGapColumns(); - if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) - || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, id ); + if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) { + msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id ); + msa_props.add( msa_prop ); + System.out.println(); } - else { - final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - printMsaProperties( id, msa_prop ); + else if ( isPrintMsaStats( i ) ) { + msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.println(); } - System.out.println(); ++i; } if ( _removed_seqs_out_base != null ) { final String msg = writeAndAlignRemovedSeqs(); + System.out.println(); System.out.println( msg ); } + if ( _phylogentic_inference ) { + decorateTree( phy, msa_props, false ); + displayTree( phy ); + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + final Phylogeny phy2 = calcTree(); + addSeqs2Tree( _msa, phy2 ); + displayTree( phy2 ); + } + + + return msa_props; } - public void removeViaLength( final int length ) throws IOException, InterruptedException { - final GapContribution stats[] = calcGapContribtionsStats( _norm ); + public List removeViaLength( final int length ) throws IOException, InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( _normalize_for_effective_seq_length ); final List to_remove_ids = new ArrayList(); + final List msa_props = new ArrayList(); for( final GapContribution gap_gontribution : stats ) { to_remove_ids.add( gap_gontribution.getId() ); } + Phylogeny phy = null; + if ( _phylogentic_inference ) { + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + phy = calcTree(); + addSeqs2Tree( _msa, phy ); + } printTableHeader(); + MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.println(); int i = 0; while ( _msa.getLength() > length ) { final String id = to_remove_ids.get( i ); _removed_seq_ids.add( id ); - final Sequence deleted = _msa.deleteRow( id, true ); + final MolecularSequence deleted = _msa.deleteRow( id, true ); _removed_seqs.add( deleted ); removeGapColumns(); - if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) || ( _msa.getLength() <= length ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, id ); + if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( _msa.getLength() <= length ) ) { + msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id ); + msa_props.add( msa_prop ); + System.out.println(); + } + else if ( isPrintMsaStats( i ) ) { + msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + printMsaProperties( msa_prop ); + msa_props.add( msa_prop ); + System.out.println(); } - final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - printMsaProperties( id, msa_prop ); - System.out.println(); ++i; } if ( _removed_seqs_out_base != null ) { final String msg = writeAndAlignRemovedSeqs(); + System.out.println(); System.out.println( msg ); } + if ( _phylogentic_inference ) { + decorateTree( phy, msa_props, false ); + displayTree( phy ); + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + final Phylogeny phy2 = calcTree(); + addSeqs2Tree( _msa, phy2 ); + displayTree( phy2 ); + } + + return msa_props; } - public final void removeWorstOffenders( final int to_remove ) throws IOException, InterruptedException { - final GapContribution stats[] = calcGapContribtionsStats( _norm ); + public final List removeWorstOffenders( final int to_remove ) throws IOException, + InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( _normalize_for_effective_seq_length ); final List to_remove_ids = new ArrayList(); + final List msa_props = new ArrayList(); for( int j = 0; j < to_remove; ++j ) { to_remove_ids.add( stats[ j ].getId() ); - _removed_seq_ids.add( stats[ j ].getId() ); + } + Phylogeny phy = null; + if ( _phylogentic_inference ) { + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + phy = calcTree(); + addSeqs2Tree( _msa, phy ); } printTableHeader(); + MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); + System.out.println(); for( int i = 0; i < to_remove_ids.size(); ++i ) { final String id = to_remove_ids.get( i ); _removed_seq_ids.add( id ); - final Sequence deleted = _msa.deleteRow( id, true ); + final MolecularSequence deleted = _msa.deleteRow( id, true ); _removed_seqs.add( deleted ); removeGapColumns(); if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( i == ( to_remove_ids.size() - 1 ) ) ) { - printMsaStatsWriteOutfileAndRealign( _realign, id ); + msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id ); + msa_props.add( msa_prop ); System.out.println(); } else if ( isPrintMsaStats( i ) ) { - final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - printMsaProperties( id, msa_prop ); + msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + msa_props.add( msa_prop ); + printMsaProperties( msa_prop ); System.out.println(); } } if ( _removed_seqs_out_base != null ) { final String msg = writeAndAlignRemovedSeqs(); + System.out.println(); System.out.println( msg ); } - } - - public final List chart( final int step, final boolean realign, final boolean norm ) - throws IOException, InterruptedException { - final GapContribution stats[] = calcGapContribtionsStats( norm ); - final List to_remove_ids = new ArrayList(); - final List msa_props = new ArrayList(); - for( final GapContribution gap_gontribution : stats ) { - to_remove_ids.add( gap_gontribution.getId() ); - } - printTableHeader(); - int i = 0; - final int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 ); - MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - msa_props.add( msa_prop ); - printMsaProperties( "", msa_prop ); - System.out.println(); - while ( _msa.getNumberOfSequences() > x ) { - final String id = to_remove_ids.get( i ); - _msa.deleteRow( id, false ); - if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) { - removeGapColumns(); - realignWithMafft(); - msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - msa_props.add( msa_prop ); - printMsaProperties( id, msa_prop ); - System.out.print( "(realigned)" ); - System.out.println(); - } - else if ( isPrintMsaStats( i ) ) { - removeGapColumns(); - msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - msa_props.add( msa_prop ); - printMsaProperties( id, msa_prop ); - System.out.println(); - } - ++i; - } + if ( _phylogentic_inference ) { + decorateTree( phy, msa_props, false ); + displayTree( phy ); + System.out.println( "calculating phylogentic tree..." ); + System.out.println(); + final Phylogeny phy2 = calcTree(); + addSeqs2Tree( _msa, phy2 ); + displayTree( phy2 ); + } + return msa_props; } - private final boolean isPrintMsaStats( final int i ) { - return ( ( _step_for_diagnostics < 2 ) || ( ( ( i + 1 ) % _step_for_diagnostics ) == 0 ) ); - } - - private final boolean isPrintMsaStatsWriteOutfileAndRealign( final int i ) { - return ( ( _step < 2 ) || ( ( ( i + 1 ) % _step ) == 0 ) ); + public final void setCalculateNormalizedShannonEntropy( final boolean calculate_shannon_entropy ) { + _calculate_shannon_entropy = calculate_shannon_entropy; } - public final void setGapRatio( final double gap_ratio ) { - _gap_ratio = gap_ratio; + public void setInfileName( final String infile_name ) { + _infile_name = infile_name; } - public final void setMinLength( final int min_length ) { - _min_length = min_length; + public final void setMafftOptions( final String maffts_opts ) { + _maffts_opts = maffts_opts; } - public final void setNorm( final boolean norm ) { - _norm = norm; + public final void setNorm( final boolean normalize_for_effective_seq_length ) { + _normalize_for_effective_seq_length = normalize_for_effective_seq_length; } final public void setOutFileBase( final File out_file_base ) { @@ -263,10 +503,18 @@ public class MsaCompactor { _path_to_mafft = path_to_mafft; } + public void setPeformPhylogenticInference( final boolean phylogentic_inference ) { + _phylogentic_inference = phylogentic_inference; + } + public final void setRealign( final boolean realign ) { _realign = realign; } + public final void setRemovedSeqsOutBase( final File removed_seqs_out_base ) { + _removed_seqs_out_base = removed_seqs_out_base; + } + public final void setStep( final int step ) { _step = step; } @@ -275,23 +523,11 @@ public class MsaCompactor { _step_for_diagnostics = step_for_diagnostics; } - public final void setReportAlnMeanIdentity( final boolean report_aln_mean_identity ) { - _report_aln_mean_identity = report_aln_mean_identity; - } - - final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { - final Double gr = MsaMethods.calcGapRatio( _msa ); - final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" - + ForesterUtil.roundToInt( gr * 100 ); - writeMsa( _msa, s + suffix, format ); - return s; - } - final public String writeAndAlignRemovedSeqs() throws IOException, InterruptedException { final StringBuilder msg = new StringBuilder(); final String n = _removed_seqs_out_base + "_" + _removed_seqs.size() + ".fasta"; SequenceWriter.writeSeqs( _removed_seqs, new File( n ), SEQ_FORMAT.FASTA, 100 ); - msg.append( "wrote " + _removed_seqs.size() + " removed sequences to " + " to \"" + n + "\"" ); + msg.append( "wrote " + _removed_seqs.size() + " removed sequences to " + "\"" + n + "\"" ); if ( _realign ) { final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft ); final List opts = new ArrayList(); @@ -302,13 +538,7 @@ public class MsaCompactor { final Double gr = MsaMethods.calcGapRatio( removed_msa ); String s = _removed_seqs_out_base + "_" + removed_msa.getNumberOfSequences() + "_" + removed_msa.getLength() + "_" + ForesterUtil.roundToInt( gr * 100 ); - String suffix = ""; - if ( _output_format == MSA_FORMAT.FASTA ) { - suffix = ".fasta"; - } - else if ( _output_format == MSA_FORMAT.PHYLIP ) { - suffix = ".aln"; - } + final String suffix = obtainSuffix(); s += suffix; writeMsa( removed_msa, s, _output_format ); msg.append( ", and as MSA of length " + removed_msa.getLength() + " to \"" + s + "\"" ); @@ -316,7 +546,15 @@ public class MsaCompactor { return msg.toString(); } - final int calcNonGapResidues( final Sequence seq ) { + final public String writeMsa( final File outfile ) throws IOException { + final Double gr = MsaMethods.calcGapRatio( _msa ); + final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" + + ForesterUtil.roundToInt( gr * 100 ); + writeMsa( _msa, s + obtainSuffix(), _output_format ); + return s; + } + + final int calcNonGapResidues( final MolecularSequence seq ) { int ng = 0; for( int i = 0; i < seq.getLength(); ++i ) { if ( !seq.isGapAt( i ) ) { @@ -346,8 +584,8 @@ public class MsaCompactor { return stats; } - final private GapContribution[] calcGapContribtionsStats( final boolean norm ) { - final GapContribution stats[] = calcGapContribtions( norm ); + final private GapContribution[] calcGapContribtionsStats( final boolean normalize_for_effective_seq_length ) { + final GapContribution stats[] = calcGapContribtions( normalize_for_effective_seq_length ); Arrays.sort( stats ); return stats; } @@ -362,6 +600,12 @@ public class MsaCompactor { return gappiness; } + private final Phylogeny collapse( final Msa msa, final int threshold ) { + final BasicSymmetricalDistanceMatrix m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa ); + //TODO + return null; + } + private final Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, final Msa msa, final boolean write_matrix, @@ -393,7 +637,43 @@ public class MsaCompactor { return phy; } - private final Phylogeny pi( final String matrix ) { + private final boolean isPrintMsaStats( final int i ) { + return ( ( ( _step == 1 ) && ( _step_for_diagnostics == 1 ) ) || ( ( _step_for_diagnostics > 0 ) && ( ( ( i + 1 ) % _step_for_diagnostics ) == 0 ) ) ); + } + + private final boolean isPrintMsaStatsWriteOutfileAndRealign( final int i ) { + return ( ( ( _step == 1 ) && ( _step_for_diagnostics == 1 ) ) || ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) ); + } + + private final StringBuilder msaPropertiesAsSB( final MsaProperties msa_properties ) { + final StringBuilder sb = new StringBuilder(); + sb.append( msa_properties.getNumberOfSequences() ); + sb.append( "\t" ); + sb.append( msa_properties.getLength() ); + sb.append( "\t" ); + sb.append( NF_4.format( msa_properties.getGapRatio() ) ); + sb.append( "\t" ); + sb.append( NF_1.format( msa_properties.getAvgNumberOfGaps() ) ); + if ( _calculate_shannon_entropy ) { + sb.append( "\t" ); + sb.append( NF_4.format( msa_properties.getEntropy7() ) ); + sb.append( "\t" ); + sb.append( NF_4.format( msa_properties.getEntropy21() ) ); + } + return sb; + } + + private String obtainSuffix() { + if ( _output_format == MSA_FORMAT.FASTA ) { + return ".fasta"; + } + else if ( _output_format == MSA_FORMAT.PHYLIP ) { + return ".aln"; + } + return ""; + } + + private final Phylogeny pi( final String matrix, final int boostrap ) { final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); final int seed = 15; final int n = 100; @@ -411,37 +691,47 @@ public class MsaCompactor { return master_phy; } - private final void printMsaProperties( final String id, final MsaProperties msa_properties ) { - System.out.print( ForesterUtil.pad( id, _longest_id_length, ' ', false ) ); - System.out.print( "\t" ); - final StringBuilder sb = msaPropertiesAsSB( msa_properties ); - System.out.print( sb ); - System.out.print( "\t" ); - } - - private final StringBuilder msaPropertiesAsSB( final MsaProperties msa_properties ) { - final StringBuilder sb = new StringBuilder(); - sb.append( msa_properties.getNumberOfSequences() ); - sb.append( "\t" ); - sb.append( msa_properties.getLength() ); - sb.append( "\t" ); - sb.append( NF_4.format( msa_properties.getGapRatio() ) ); - if ( _report_aln_mean_identity /*msa_properties.getAverageIdentityRatio() >= 0*/) { - sb.append( "\t" ); - sb.append( NF_4.format( msa_properties.getAverageIdentityRatio() ) ); + private final void printMsaProperties( final MsaProperties msa_properties ) { + if ( ( _step == 1 ) || ( _step_for_diagnostics == 1 ) ) { + System.out.print( ForesterUtil.pad( msa_properties.getRemovedSeq(), _longest_id_length, ' ', false ) ); + System.out.print( "\t" ); } - return sb; + System.out.print( msaPropertiesAsSB( msa_properties ) ); + System.out.print( "\t" ); } - final private void printMsaStatsWriteOutfileAndRealign( final boolean realign, final String id ) + final private MsaProperties printMsaStatsWriteOutfileAndRealign( final boolean realign, final String id ) throws IOException, InterruptedException { if ( realign ) { realignWithMafft(); } - final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity ); - printMsaProperties( id, msa_prop ); + final MsaProperties msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy ); + printMsaProperties( msa_prop ); final String s = writeOutfile(); System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) ); + return msa_prop; + } + + private final void printTableHeader() { + if ( ( _step == 1 ) || ( _step_for_diagnostics == 1 ) ) { + System.out.print( ForesterUtil.pad( "Id", _longest_id_length, ' ', false ) ); + System.out.print( "\t" ); + } + System.out.print( "Seqs" ); + System.out.print( "\t" ); + System.out.print( "Length" ); + System.out.print( "\t" ); + System.out.print( "Gap R" ); + System.out.print( "\t" ); + System.out.print( "Gaps" ); + System.out.print( "\t" ); + if ( _calculate_shannon_entropy ) { + System.out.print( "entn7" ); + System.out.print( "\t" ); + System.out.print( "entn21" ); + System.out.print( "\t" ); + } + System.out.println(); } final private void realignWithMafft() throws IOException, InterruptedException { @@ -457,16 +747,8 @@ public class MsaCompactor { _msa.deleteGapOnlyColumns(); } - final private static void writeMsa( final Msa msa, final String outfile, final MSA_FORMAT format ) - throws IOException { - final Writer w = ForesterUtil.createBufferedWriter( outfile ); - msa.write( w, format ); - w.close(); - } - private final String writeOutfile() throws IOException { - final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" ); - //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" ); + final String s = writeMsa( _out_file_base ); return s; } @@ -502,19 +784,10 @@ public class MsaCompactor { return null; } - private final void printTableHeader() { - System.out.print( ForesterUtil.pad( "Id", _longest_id_length, ' ', false ) ); - System.out.print( "\t" ); - System.out.print( "Seqs" ); - System.out.print( "\t" ); - System.out.print( "Length" ); - System.out.print( "\t" ); - System.out.print( "Gaps" ); - System.out.print( "\t" ); - if ( _report_aln_mean_identity ) { - System.out.print( "MSA qual" ); - System.out.print( "\t" ); - } - System.out.println(); + final private static void writeMsa( final Msa msa, final String outfile, final MSA_FORMAT format ) + throws IOException { + final Writer w = ForesterUtil.createBufferedWriter( outfile ); + msa.write( w, format ); + w.close(); } }