import org.forester.phylogeny.data.NodeVisualData.NodeFill;
import org.forester.phylogeny.data.NodeVisualData.NodeShape;
import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
-import org.forester.sequence.Sequence;
+import org.forester.sequence.MolecularSequence;
import org.forester.tools.ConfidenceAssessor;
+import org.forester.util.BasicDescriptiveStatistics;
+import org.forester.util.DescriptiveStatistics;
import org.forester.util.ForesterUtil;
public class MsaCompactor {
- final private static NumberFormat NF_3 = new DecimalFormat( "#.###" );
- final private static NumberFormat NF_4 = new DecimalFormat( "#.####" );
- private double _gap_ratio = -1;
- private final short _longest_id_length;
+ final private static NumberFormat NF_1 = new DecimalFormat( "0.#" );
+ final private static NumberFormat NF_3 = new DecimalFormat( "0.###" );
+ final private static NumberFormat NF_4 = new DecimalFormat( "0.####" );
+ private boolean _calculate_shannon_entropy = false;
//
- private String _maffts_opts = "--auto";
- private int _min_length = -1;
+ private String _infile_name = null;
+ private final short _longest_id_length;
//
- private String _infile_name = null;
- private DeleteableMsa _msa = null;
- private boolean _norm = true;
- private File _out_file_base = null;
- private MSA_FORMAT _output_format = MSA_FORMAT.FASTA;
- private String _path_to_mafft = null;
+ private String _maffts_opts = "--auto";
+ private DeleteableMsa _msa = null;
+ private boolean _norm = true;
+ private File _out_file_base = null;
+ private MSA_FORMAT _output_format = MSA_FORMAT.FASTA;
+ private String _path_to_mafft = null;
+ private boolean _phylogentic_inference = false;
//
- private boolean _realign = false;
- private final SortedSet<String> _removed_seq_ids;
- private final ArrayList<Sequence> _removed_seqs;
- private File _removed_seqs_out_base = null;
- private boolean _report_aln_mean_identity = false;
- private int _step = -1;
- private int _step_for_diagnostics = -1;
- private boolean _phylogentic_inference = false;
+ private boolean _realign = false;
+ private final SortedSet<String> _removed_seq_ids;
+ private final ArrayList<MolecularSequence> _removed_seqs;
+ private File _removed_seqs_out_base = null;
+ private int _step = -1;
+ private int _step_for_diagnostics = -1;
static {
+ NF_1.setRoundingMode( RoundingMode.HALF_UP );
NF_4.setRoundingMode( RoundingMode.HALF_UP );
NF_3.setRoundingMode( RoundingMode.HALF_UP );
}
_msa = msa;
_removed_seq_ids = new TreeSet<String>();
_longest_id_length = _msa.determineMaxIdLength();
- _removed_seqs = new ArrayList<Sequence>();
+ _removed_seqs = new ArrayList<MolecularSequence>();
+ }
+
+ public final Phylogeny calcTree() {
+ final Phylogeny phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, false, "" );
+ PhylogenyMethods.midpointRoot( phy );
+ PhylogenyMethods.orderAppearance( phy.getRoot(), true, true, DESCENDANT_SORT_PRIORITY.NODE_NAME );
+ final boolean x = PhylogenyMethods.extractFastaInformation( phy );
+ if ( !x ) {
+ final PhylogenyNodeIterator it = phy.iteratorExternalForward();
+ while ( it.hasNext() ) {
+ final PhylogenyNode n = it.next();
+ final String name = n.getName().trim();
+ if ( !ForesterUtil.isEmpty( name ) ) {
+ try {
+ ParserUtils.extractTaxonomyDataFromNodeName( n, TAXONOMY_EXTRACTION.AGGRESSIVE );
+ }
+ catch ( final PhyloXmlDataFormatException e ) {
+ // Ignore.
+ }
+ }
+ }
+ }
+ return phy;
}
public final List<MsaProperties> chart( final int step, final boolean realign, final boolean norm )
for( final GapContribution gap_gontribution : stats ) {
to_remove_ids.add( gap_gontribution.getId() );
}
+ Phylogeny phy = null;
+ if ( _phylogentic_inference ) {
+ System.out.println( "calculating phylogentic tree..." );
+ System.out.println();
+ phy = calcTree();
+ }
if ( !_realign ) {
_step = -1;
}
- int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 );
- if ( x < 1 ) {
- x = 1;
+ int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 10.0 );
+ if ( x < 2 ) {
+ x = 2;
}
- MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy );
msa_props.add( msa_prop );
- if ( _phylogentic_inference ) {
- System.out.println( "calculating phylogentic tree..." );
- System.out.println();
- pi( to_remove_ids );
- }
printTableHeader();
- printMsaProperties( "", msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
int i = 0;
while ( _msa.getNumberOfSequences() > x ) {
if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) {
removeGapColumns();
realignWithMafft();
- msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( id, msa_prop );
+ printMsaProperties( msa_prop );
System.out.print( "(realigned)" );
System.out.println();
}
else if ( isPrintMsaStats( i ) ) {
removeGapColumns();
- msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( id, msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
}
++i;
}
+ if ( _phylogentic_inference ) {
+ decorateTree( phy, msa_props, true );
+ displayTree( phy );
+ }
return msa_props;
}
+ public final void decorateTree( final Phylogeny phy, final List<MsaProperties> msa_props, final boolean chart_only ) {
+ final BasicDescriptiveStatistics length_stats = new BasicDescriptiveStatistics();
+ for( int i = 0; i < msa_props.size(); ++i ) {
+ final MsaProperties msa_prop = msa_props.get( i );
+ final String id = msa_prop.getRemovedSeq();
+ if ( !ForesterUtil.isEmpty( id ) ) {
+ length_stats.addValue( msa_prop.getLength() );
+ }
+ }
+ final double mean = length_stats.arithmeticMean();
+ final double min = length_stats.getMin();
+ final double max = length_stats.getMax();
+ final Color min_color = new Color( 0, 255, 0 );
+ final Color max_color = new Color( 255, 0, 0 );
+ final Color mean_color = new Color( 255, 255, 0 );
+ final PhylogenyNodeIterator it = phy.iteratorExternalForward();
+ if ( chart_only ) {
+ while ( it.hasNext() ) {
+ final NodeVisualData vis = new NodeVisualData();
+ vis.setFillType( NodeFill.SOLID );
+ vis.setShape( NodeShape.RECTANGLE );
+ vis.setNodeColor( min_color );
+ it.next().getNodeData().setNodeVisualData( vis );
+ }
+ }
+ for( int i = 0; i < msa_props.size(); ++i ) {
+ final MsaProperties msa_prop = msa_props.get( i );
+ final String id = msa_prop.getRemovedSeq();
+ if ( !ForesterUtil.isEmpty( id ) ) {
+ final PhylogenyNode n = phy.getNode( id );
+ n.setName( n.getName() + " [" + i + "]" );
+ if ( !chart_only ) {
+ final NodeVisualData vis = new NodeVisualData();
+ vis.setFillType( NodeFill.SOLID );
+ vis.setShape( NodeShape.RECTANGLE );
+ vis.setNodeColor( ForesterUtil.calcColor( msa_prop.getLength(), min, max, mean_color, max_color ) );
+ n.getNodeData().setNodeVisualData( vis );
+ }
+ else {
+ n.getNodeData()
+ .getNodeVisualData()
+ .setNodeColor( ForesterUtil.calcColor( msa_prop.getLength(),
+ min,
+ max,
+ mean,
+ min_color,
+ max_color,
+ mean_color ) );
+ }
+ }
+ }
+ }
+
final public void deleteGapColumns( final double max_allowed_gap_ratio ) {
_msa.deleteGapColumns( max_allowed_gap_ratio );
}
- final public Msa getMsa() {
- return _msa;
+ public final void displayTree( final Phylogeny phy ) {
+ final Configuration config = new Configuration();
+ config.setDisplayAsPhylogram( true );
+ config.setUseStyle( true );
+ config.setDisplayTaxonomyCode( false );
+ config.setDisplayTaxonomyCommonNames( false );
+ config.setDisplayTaxonomyScientificNames( false );
+ config.setDisplaySequenceNames( false );
+ config.setDisplaySequenceSymbols( false );
+ config.setDisplayGeneNames( false );
+ config.setShowScale( true );
+ config.setAddTaxonomyImagesCB( false );
+ config.setBaseFontSize( 9 );
+ config.setBaseFontFamilyName( "Arial" );
+ Archaeopteryx.createApplication( phy, config, _infile_name );
}
- final public SortedSet<String> getRemovedSeqIds() {
- return _removed_seq_ids;
+ final public Msa getMsa() {
+ return _msa;
}
- public final void removeSequencesByMinimalLength( final int min_effective_length ) {
- printMsaProperties( "", new MsaProperties( _msa, _report_aln_mean_identity ) );
- System.out.println();
+ public final void removeSequencesByMinimalLength( final int min_effective_length ) throws IOException {
_msa = DeleteableMsa.createInstance( MsaMethods.removeSequencesByMinimalLength( _msa, min_effective_length ) );
removeGapColumns();
- printMsaProperties( "", new MsaProperties( _msa, _report_aln_mean_identity ) );
+ final String s = writeOutfile();
+ final DescriptiveStatistics msa_stats = MsaMethods.calculateEffectiveLengthStatistics( _msa );
+ System.out.println( "Output MSA : " + s );
+ System.out.println( " MSA length : " + _msa.getLength() );
+ System.out.println( " Number of sequences : " + _msa.getNumberOfSequences() );
+ System.out.println( " Median sequence length : " + NF_1.format( msa_stats.median() ) );
+ System.out.println( " Mean sequence length : " + NF_1.format( msa_stats.arithmeticMean() ) );
+ System.out.println( " Max sequence length : " + ( ( int ) msa_stats.getMax() ) );
+ System.out.println( " Min sequence length : " + ( ( int ) msa_stats.getMin() ) );
+ System.out.println( " Gap ratio : " + NF_4.format( MsaMethods.calcGapRatio( _msa ) ) );
+ System.out.println( " Normalized Shannon Entropy (entn21): "
+ + NF_4.format( MsaMethods.calcNormalizedShannonsEntropy( 21, _msa ) ) );
System.out.println();
}
for( final GapContribution gap_gontribution : stats ) {
to_remove_ids.add( gap_gontribution.getId() );
}
+ Phylogeny phy = null;
+ if ( _phylogentic_inference ) {
+ System.out.println( "calculating phylogentic tree..." );
+ System.out.println();
+ phy = calcTree();
+ }
printTableHeader();
- MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( "", msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
int i = 0;
while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) {
final String id = to_remove_ids.get( i );
_removed_seq_ids.add( id );
- final Sequence deleted = _msa.deleteRow( id, true );
+ final MolecularSequence deleted = _msa.deleteRow( id, true );
_removed_seqs.add( deleted );
removeGapColumns();
if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) {
System.out.println();
}
else if ( isPrintMsaStats( i ) ) {
- msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( id, msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
}
++i;
System.out.println();
System.out.println( msg );
}
+ if ( _phylogentic_inference ) {
+ decorateTree( phy, msa_props, false );
+ displayTree( phy );
+ }
return msa_props;
}
for( final GapContribution gap_gontribution : stats ) {
to_remove_ids.add( gap_gontribution.getId() );
}
+ Phylogeny phy = null;
+ if ( _phylogentic_inference ) {
+ System.out.println( "calculating phylogentic tree..." );
+ System.out.println();
+ phy = calcTree();
+ }
printTableHeader();
- MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( "", msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
int i = 0;
while ( _msa.getLength() > length ) {
final String id = to_remove_ids.get( i );
_removed_seq_ids.add( id );
- final Sequence deleted = _msa.deleteRow( id, true );
+ final MolecularSequence deleted = _msa.deleteRow( id, true );
_removed_seqs.add( deleted );
removeGapColumns();
if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( _msa.getLength() <= length ) ) {
System.out.println();
}
else if ( isPrintMsaStats( i ) ) {
- msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
- printMsaProperties( id, msa_prop );
+ msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
+ printMsaProperties( msa_prop );
msa_props.add( msa_prop );
System.out.println();
}
System.out.println();
System.out.println( msg );
}
+ if ( _phylogentic_inference ) {
+ decorateTree( phy, msa_props, false );
+ displayTree( phy );
+ }
return msa_props;
}
final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
for( int j = 0; j < to_remove; ++j ) {
to_remove_ids.add( stats[ j ].getId() );
- _removed_seq_ids.add( stats[ j ].getId() );
+ }
+ Phylogeny phy = null;
+ if ( _phylogentic_inference ) {
+ System.out.println( "calculating phylogentic tree..." );
+ System.out.println();
+ phy = calcTree();
}
printTableHeader();
- MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ MsaProperties msa_prop = new MsaProperties( _msa, "", _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( "", msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
for( int i = 0; i < to_remove_ids.size(); ++i ) {
final String id = to_remove_ids.get( i );
_removed_seq_ids.add( id );
- final Sequence deleted = _msa.deleteRow( id, true );
+ final MolecularSequence deleted = _msa.deleteRow( id, true );
_removed_seqs.add( deleted );
removeGapColumns();
if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( i == ( to_remove_ids.size() - 1 ) ) ) {
System.out.println();
}
else if ( isPrintMsaStats( i ) ) {
- msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
+ msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
msa_props.add( msa_prop );
- printMsaProperties( id, msa_prop );
+ printMsaProperties( msa_prop );
System.out.println();
}
}
System.out.println();
System.out.println( msg );
}
+ if ( _phylogentic_inference ) {
+ decorateTree( phy, msa_props, false );
+ displayTree( phy );
+ }
return msa_props;
}
- public final void setGapRatio( final double gap_ratio ) {
- _gap_ratio = gap_ratio;
+ public final void setCalculateNormalizedShannonEntropy( final boolean calculate_shannon_entropy ) {
+ _calculate_shannon_entropy = calculate_shannon_entropy;
}
- public final void setMafftOptions( final String maffts_opts ) {
- _maffts_opts = maffts_opts;
+ public void setInfileName( final String infile_name ) {
+ _infile_name = infile_name;
}
- public final void setMinLength( final int min_length ) {
- _min_length = min_length;
+ public final void setMafftOptions( final String maffts_opts ) {
+ _maffts_opts = maffts_opts;
}
public final void setNorm( final boolean norm ) {
_path_to_mafft = path_to_mafft;
}
+ public void setPeformPhylogenticInference( final boolean phylogentic_inference ) {
+ _phylogentic_inference = phylogentic_inference;
+ }
+
public final void setRealign( final boolean realign ) {
_realign = realign;
}
_removed_seqs_out_base = removed_seqs_out_base;
}
- public final void setReportAlnMeanIdentity( final boolean report_aln_mean_identity ) {
- _report_aln_mean_identity = report_aln_mean_identity;
- }
-
public final void setStep( final int step ) {
_step = step;
}
return s;
}
- final int calcNonGapResidues( final Sequence seq ) {
+ final int calcNonGapResidues( final MolecularSequence seq ) {
int ng = 0;
for( int i = 0; i < seq.getLength(); ++i ) {
if ( !seq.isGapAt( i ) ) {
return gappiness;
}
+ private final Phylogeny collapse( final Msa msa, final int threshold ) {
+ final BasicSymmetricalDistanceMatrix m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa );
+ //TODO
+ return null;
+ }
+
private final Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method,
final Msa msa,
final boolean write_matrix,
sb.append( msa_properties.getLength() );
sb.append( "\t" );
sb.append( NF_4.format( msa_properties.getGapRatio() ) );
- if ( _report_aln_mean_identity ) {
+ if ( _calculate_shannon_entropy ) {
sb.append( "\t" );
- sb.append( NF_4.format( msa_properties.getAverageIdentityRatio() ) );
+ sb.append( NF_4.format( msa_properties.getEntropy7() ) );
+ sb.append( "\t" );
+ sb.append( NF_4.format( msa_properties.getEntropy21() ) );
}
return sb;
}
return master_phy;
}
- private final Phylogeny pi( final List<String> to_remove_ids ) {
- final Phylogeny phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, false, "" );
- for( int i = 0; i < to_remove_ids.size(); ++i ) {
- final String id = to_remove_ids.get( i );
- final PhylogenyNode n = phy.getNode( id );
- n.setName( n.getName() + " [" + ( i + 1 ) + "]" );
- final NodeVisualData vis = new NodeVisualData();
- vis.setFillType( NodeFill.SOLID );
- vis.setShape( NodeShape.RECTANGLE );
- vis.setSize( 6 );
- vis.setNodeColor( new Color( i > 255 ? 0 : 255 - i, 0, 0 ) );
- n.getNodeData().setNodeVisualData( vis );
- }
- PhylogenyMethods.midpointRoot( phy );
- PhylogenyMethods.orderAppearance( phy.getRoot(), true, true, DESCENDANT_SORT_PRIORITY.NODE_NAME );
- final boolean x = PhylogenyMethods.extractFastaInformation( phy );
- if ( !x ) {
- final PhylogenyNodeIterator it = phy.iteratorExternalForward();
- while ( it.hasNext() ) {
- final PhylogenyNode n = it.next();
- final String name = n.getName().trim();
- if ( !ForesterUtil.isEmpty( name ) ) {
- try {
- ParserUtils.extractTaxonomyDataFromNodeName( n, TAXONOMY_EXTRACTION.AGGRESSIVE );
- }
- catch ( final PhyloXmlDataFormatException e ) {
- // Ignore.
- }
- }
- }
- }
- final Configuration config = new Configuration();
- config.setDisplayAsPhylogram( true );
- config.setUseStyle( true );
- config.setDisplayTaxonomyCode( false );
- config.setDisplayTaxonomyCommonNames( false );
- config.setDisplayTaxonomyScientificNames( false );
- config.setDisplaySequenceNames( false );
- config.setDisplaySequenceSymbols( false );
- config.setDisplayGeneNames( false );
- config.setShowScale( true );
- config.setAddTaxonomyImagesCB( false );
- config.setBaseFontSize( 9 );
- config.setBaseFontFamilyName( "Arial" );
- Archaeopteryx.createApplication( phy, config, _infile_name );
- return phy;
- }
-
- private final void printMsaProperties( final String id, final MsaProperties msa_properties ) {
+ private final void printMsaProperties( final MsaProperties msa_properties ) {
if ( ( _step == 1 ) || ( _step_for_diagnostics == 1 ) ) {
- System.out.print( ForesterUtil.pad( id, _longest_id_length, ' ', false ) );
+ System.out.print( ForesterUtil.pad( msa_properties.getRemovedSeq(), _longest_id_length, ' ', false ) );
System.out.print( "\t" );
}
System.out.print( msaPropertiesAsSB( msa_properties ) );
if ( realign ) {
realignWithMafft();
}
- final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
- printMsaProperties( id, msa_prop );
+ final MsaProperties msa_prop = new MsaProperties( _msa, id, _calculate_shannon_entropy );
+ printMsaProperties( msa_prop );
final String s = writeOutfile();
System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) );
return msa_prop;
System.out.print( "\t" );
System.out.print( "Gaps" );
System.out.print( "\t" );
- if ( _report_aln_mean_identity ) {
- System.out.print( "MSA qual" );
+ if ( _calculate_shannon_entropy ) {
+ System.out.print( "entn7" );
+ System.out.print( "\t" );
+ System.out.print( "entn21" );
System.out.print( "\t" );
}
System.out.println();
msa.write( w, format );
w.close();
}
-
- public void setPeformPhylogenticInference( final boolean phylogentic_inference ) {
- _phylogentic_inference = phylogentic_inference;
- }
-
- public void setInfileName( final String infile_name ) {
- _infile_name = infile_name;
- }
}