X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=forester%2Fjava%2Fsrc%2Forg%2Fforester%2Fapplication%2Fmsa_compactor.java;h=19d5caf03ab28b601ff2f5c45f7a24dfb0fcf2ad;hb=327fc7c7a9a46cbe8983b0ca2a54cc6f45cd1695;hp=8e71f5e6d83584e26f17a1cbe3fe057c9587a89e;hpb=0bbf7742023f862d29f1266c5a172b4cb99b7939;p=jalview.git diff --git a/forester/java/src/org/forester/application/msa_compactor.java b/forester/java/src/org/forester/application/msa_compactor.java index 8e71f5e..19d5caf 100644 --- a/forester/java/src/org/forester/application/msa_compactor.java +++ b/forester/java/src/org/forester/application/msa_compactor.java @@ -26,6 +26,10 @@ package org.forester.application; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; +import java.math.RoundingMode; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; @@ -44,30 +48,35 @@ import org.forester.util.ForesterUtil; public class msa_compactor { - final static private String HELP_OPTION_1 = "help"; - final static private String HELP_OPTION_2 = "h"; - final static private String REMOVE_WORST_OFFENDERS_OPTION = "r"; - final static private String AV_GAPINESS_OPTION = "g"; - final static private String STEP_OPTION = "s"; - final static private String LENGTH_OPTION = "l"; - final static private String REALIGN_OPTION = "a"; + final private static NumberFormat NF_1 = new DecimalFormat( "#.0" ); + static { + NF_1.setRoundingMode( RoundingMode.HALF_UP ); + } + final static private String HELP_OPTION_1 = "help"; + final static private String HELP_OPTION_2 = "h"; + final static private String REMOVE_WORST_OFFENDERS_OPTION = "r"; + final static private String AV_GAPINESS_OPTION = "g"; + final static private String STEP_OPTION = "s"; + final static private String LENGTH_OPTION = "l"; + final static private String REALIGN_OPTION = "a"; // - final static private String STEP_FOR_DIAGNOSTICS_OPTION = "sd"; - final static private String MIN_LENGTH_OPTION = "ml"; - final static private String GAP_RATIO_LENGTH_OPTION = "gr"; - final static private String REPORT_ALN_MEAN_IDENTITY = "q"; - final static private String OUTPUT_FORMAT_PHYLIP_OPTION = "f"; - final static private String OUTPUT_REMOVED_SEQS_OPTION = "ro"; - final static private String MAFFT_OPTIONS = "mo"; + final static private String STEP_FOR_DIAGNOSTICS_OPTION = "sd"; + final static private String MIN_LENGTH_OPTION = "ml"; + final static private String GAP_RATIO_LENGTH_OPTION = "gr"; + final static private String REPORT_ENTROPY = "e"; + final static private String OUTPUT_FORMAT_PHYLIP_OPTION = "p"; + final static private String OUTPUT_REMOVED_SEQS_OPTION = "ro"; + final static private String MAFFT_OPTIONS = "mo"; + final static private String PERFORM_PHYLOGENETIC_INFERENCE = "t"; // - final static private String PATH_TO_MAFFT_OPTION = "mafft"; - final static private String DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION = "nn"; - final static private String PRG_NAME = "msa_compactor"; - final static private String PRG_DESC = "multiple sequence aligment compactor"; - final static private String PRG_VERSION = "0.02"; - final static private String PRG_DATE = "140316"; - final static private String E_MAIL = "czmasek@sanfordburham.org"; - final static private String WWW = "https://sites.google.com/site/cmzmasek/home/software/forester"; + final static private String PATH_TO_MAFFT_OPTION = "mafft"; + final static private String DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION = "nn"; + final static private String PRG_NAME = "msa_compactor"; + final static private String PRG_DESC = "multiple sequence aligment compactor"; + final static private String PRG_VERSION = "0.3"; + final static private String PRG_DATE = "140508"; + final static private String E_MAIL = "czmasek@sanfordburham.org"; + final static private String WWW = "https://sites.google.com/site/cmzmasek/home/software/forester"; public static void main( final String args[] ) { try { @@ -85,17 +94,18 @@ public class msa_compactor { int worst_remove = -1; double av_gap = -1; int length = -1; - int step = -1; + int step = 1; boolean realign = false; boolean norm = true; String path_to_mafft = null; - int step_for_diagnostics = -1; + int step_for_diagnostics = 1; int min_length = -1; double gap_ratio = -1; - boolean report_aln_mean_identity = false; + boolean report_entropy = false; MSA_FORMAT output_format = MSA_FORMAT.FASTA; File removed_seqs_out_base = null; String mafft_options = "--auto"; + boolean perform_phylogenetic_inference = false; final List allowed_options = new ArrayList(); allowed_options.add( REMOVE_WORST_OFFENDERS_OPTION ); allowed_options.add( AV_GAPINESS_OPTION ); @@ -107,10 +117,11 @@ public class msa_compactor { allowed_options.add( STEP_FOR_DIAGNOSTICS_OPTION ); allowed_options.add( MIN_LENGTH_OPTION ); allowed_options.add( GAP_RATIO_LENGTH_OPTION ); - allowed_options.add( REPORT_ALN_MEAN_IDENTITY ); + allowed_options.add( REPORT_ENTROPY ); allowed_options.add( OUTPUT_FORMAT_PHYLIP_OPTION ); allowed_options.add( OUTPUT_REMOVED_SEQS_OPTION ); allowed_options.add( MAFFT_OPTIONS ); + allowed_options.add( PERFORM_PHYLOGENETIC_INFERENCE ); final String dissallowed_options = cla.validateAllowedOptionsAsString( allowed_options ); if ( dissallowed_options.length() > 0 ) { ForesterUtil.fatalError( PRG_NAME, "unknown option(s): " + dissallowed_options ); @@ -124,7 +135,12 @@ public class msa_compactor { msa = DeleteableMsa.createInstance( GeneralMsaParser.parse( is ) ); } final DescriptiveStatistics initial_msa_stats = MsaMethods.calculateEffectiveLengthStatistics( msa ); - System.out.println( initial_msa_stats.toString() ); + final boolean chart_only = ( !cla.isOptionSet( LENGTH_OPTION ) ) + && ( !cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) + && ( !cla.isOptionSet( AV_GAPINESS_OPTION ) ); + if ( !chart_only && ( out == null ) ) { + ForesterUtil.fatalError( PRG_NAME, "outfile file missing" ); + } if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) { worst_remove = cla.getOptionValueAsInt( REMOVE_WORST_OFFENDERS_OPTION ); if ( ( worst_remove < 1 ) || ( worst_remove >= msa.getNumberOfSequences() - 1 ) ) { @@ -148,8 +164,15 @@ public class msa_compactor { System.exit( 0 ); } length = cla.getOptionValueAsInt( LENGTH_OPTION ); - if ( ( length < 2 ) || ( length >= msa.getLength() ) ) { - ForesterUtil.fatalError( PRG_NAME, "target length is out of range: " + length ); + if ( length >= msa.getLength() ) { + ForesterUtil.fatalError( PRG_NAME, + "target length is out of range [longer than MSA (" + msa.getLength() + + ")]: " + length ); + } + else if ( length < initial_msa_stats.getMin() ) { + ForesterUtil.fatalError( PRG_NAME, + "target length is out of range [shorter than the shortest sequence (" + + initial_msa_stats.getMin() + ") ]: " + length ); } } if ( cla.isOptionSet( STEP_OPTION ) ) { @@ -171,7 +194,6 @@ public class msa_compactor { if ( cla.isOptionSet( DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION ) ) { norm = false; } - // if ( cla.isOptionSet( STEP_FOR_DIAGNOSTICS_OPTION ) ) { step_for_diagnostics = cla.getOptionValueAsInt( STEP_FOR_DIAGNOSTICS_OPTION ); if ( ( step_for_diagnostics < 1 ) @@ -187,20 +209,20 @@ public class msa_compactor { + min_length ); } } - if ( cla.isOptionSet( MIN_LENGTH_OPTION ) ) { + if ( cla.isOptionSet( GAP_RATIO_LENGTH_OPTION ) ) { gap_ratio = cla.getOptionValueAsDouble( GAP_RATIO_LENGTH_OPTION ); if ( ( gap_ratio < 0 ) || ( gap_ratio > 1 ) ) { ForesterUtil.fatalError( PRG_NAME, "gap ratio is out of range: " + gap_ratio ); } } - if ( cla.isOptionSet( REPORT_ALN_MEAN_IDENTITY ) ) { - report_aln_mean_identity = true; + if ( cla.isOptionSet( REPORT_ENTROPY ) ) { + report_entropy = true; } if ( cla.isOptionSet( OUTPUT_FORMAT_PHYLIP_OPTION ) ) { output_format = MSA_FORMAT.PHYLIP; } if ( cla.isOptionSet( OUTPUT_REMOVED_SEQS_OPTION ) ) { - String s = cla.getOptionValueAsCleanString( OUTPUT_REMOVED_SEQS_OPTION ); + final String s = cla.getOptionValueAsCleanString( OUTPUT_REMOVED_SEQS_OPTION ); removed_seqs_out_base = new File( s ); } if ( realign ) { @@ -210,11 +232,35 @@ public class msa_compactor { checkPathToMafft( path_to_mafft ); if ( cla.isOptionSet( MAFFT_OPTIONS ) ) { mafft_options = cla.getOptionValueAsCleanString( MAFFT_OPTIONS ); - if ( ForesterUtil.isEmpty( mafft_options ) || mafft_options.length() < 3 ) { - ForesterUtil.fatalError( PRG_NAME, "gap ratio is out of range: " + gap_ratio ); + if ( ForesterUtil.isEmpty( mafft_options ) || ( mafft_options.length() < 3 ) ) { + ForesterUtil.fatalError( PRG_NAME, "illegal or empty MAFFT options: " + mafft_options ); } } } + else if ( cla.isOptionSet( MAFFT_OPTIONS ) ) { + ForesterUtil.fatalError( PRG_NAME, "no need to indicate MAFFT options without realigning" ); + } + if ( cla.isOptionSet( PERFORM_PHYLOGENETIC_INFERENCE ) ) { + perform_phylogenetic_inference = true; + } + if ( chart_only ) { + if ( ( out != null ) || ( removed_seqs_out_base != null ) ) { + ForesterUtil + .fatalError( PRG_NAME, + "chart only, no outfile(s) produced, thus no need to indicate output file(s)" ); + } + if ( !realign && cla.isOptionSet( STEP_OPTION ) ) { + ForesterUtil.fatalError( PRG_NAME, + "chart only, no re-aligning, thus no need to use step for output and re-aligning; use -" + + STEP_FOR_DIAGNOSTICS_OPTION + " instead" ); + } + } + if ( perform_phylogenetic_inference ) { + if ( step_for_diagnostics != 1 ) { + ForesterUtil.fatalError( PRG_NAME, + "step for diagnostics reports needs to be set to 1 for tree calculation" ); + } + } ForesterUtil.printProgramInformation( PRG_NAME, PRG_DESC, PRG_VERSION, @@ -222,9 +268,15 @@ public class msa_compactor { E_MAIL, WWW, ForesterUtil.getForesterLibraryInformation() ); - // System.out.println( "Input MSA : " + in ); - if ( out != null ) { + System.out.println( " MSA length : " + msa.getLength() ); + System.out.println( " Number of sequences : " + msa.getNumberOfSequences() ); + System.out.println( " Median sequence length : " + NF_1.format( initial_msa_stats.median() ) ); + System.out.println( " Mean sequence length : " + + NF_1.format( initial_msa_stats.arithmeticMean() ) ); + System.out.println( " Max sequence length : " + ( ( int ) initial_msa_stats.getMax() ) ); + System.out.println( " Min sequence length : " + ( ( int ) initial_msa_stats.getMin() ) ); + if ( !chart_only ) { System.out.println( "Output : " + out ); } else { @@ -236,96 +288,98 @@ public class msa_compactor { if ( worst_remove > 0 ) { System.out.println( "Number of worst offenders to remove : " + worst_remove ); } - else if ( av_gap > 0 ) { + if ( av_gap > 0 ) { System.out.println( "Target gap-ratio : " + av_gap ); } - else if ( length > 0 ) { + if ( length > 0 ) { System.out.println( "Target MSA length : " + length ); } - else { - System.out.println( "Chart and diagnostics only : true" ); + if ( min_length > 1 ) { + System.out.println( "Minimal effective sequence length : " + min_length ); + } + if ( gap_ratio > -1 ) { + System.out.println( "Maximum allowed gap ratio per column : " + gap_ratio ); } - if ( out != null || removed_seqs_out_base != null ) { + if ( ( out != null ) || ( removed_seqs_out_base != null ) ) { System.out.println( "Output format : " + ( output_format == MSA_FORMAT.FASTA ? "fasta" : "phylip" ) ); } - System.out.println( "Step for output and re-aligning) : " + step ); + if ( chart_only && !realign ) { + System.out.println( "Step for output and re-aligning : n/a" ); + } + else { + if ( chart_only ) { + System.out.println( "Step for re-aligning : " + step ); + } + else { + System.out.println( "Step for output and re-aligning : " + step ); + } + } System.out.println( "Step for diagnostics reports : " + step_for_diagnostics ); - System.out.println( "Report mean identity (\"MSA quality\") : " + report_aln_mean_identity ); + System.out.println( "Calculate normalized Shannon Entropy : " + report_entropy ); if ( !norm ) { System.out.println( "Normalize : " + norm ); } - System.out.println( "Realign : " + realign ); + System.out.println( "Realign with MAFFT : " + realign ); if ( realign ) { System.out.println( "MAFFT options : " + mafft_options ); } - if ( min_length > -1 ) { - System.out.println( "Minimal effective sequence length : " + min_length ); + System.out.println( "Simple tree (Kimura distances, NJ) : " + perform_phylogenetic_inference ); + System.out.println(); + final int initial_number_of_seqs = msa.getNumberOfSequences(); + List msa_props = null; + final MsaCompactor mc = new MsaCompactor( msa ); + mc.setInfileName( in.getName() ); + mc.setNorm( norm ); + mc.setRealign( realign ); + if ( realign ) { + mc.setPathToMafft( path_to_mafft ); + mc.setMafftOptions( mafft_options ); + } + mc.setStep( step ); + mc.setStepForDiagnostics( step_for_diagnostics ); + mc.setCalculateNormalizedShannonEntropy( report_entropy ); + mc.setPeformPhylogenticInference( perform_phylogenetic_inference ); + if ( ( worst_remove > 0 ) || ( av_gap > 0 ) || ( length > 0 ) ) { + mc.setOutputFormat( output_format ); + mc.setOutFileBase( out ); + if ( removed_seqs_out_base != null ) { + mc.setRemovedSeqsOutBase( removed_seqs_out_base ); + } } - if ( gap_ratio > -1 ) { - System.out.println( "Maximum allowed gap ratio per column : " + gap_ratio ); + if ( min_length > 1 ) { + mc.removeSequencesByMinimalLength( min_length ); + mc.writeMsa( new File( "removed" ) ); } - System.out.println(); - // - // if ( worst_remove > 0 ) { - final MsaCompactor mc = new MsaCompactor( msa ); - mc.setRealign( realign ); - if ( realign ) { - mc.setPathToMafft( path_to_mafft ); - } - mc.setNorm( norm ); - mc.setOutFileBase( out ); - mc.setStep( step ); - mc.removeWorstOffenders( worst_remove, true ); + msa_props = mc.removeWorstOffenders( worst_remove ); } else if ( av_gap > 0 ) { - final MsaCompactor mc = new MsaCompactor( msa ); - mc.setRealign( realign ); - if ( realign ) { - mc.setPathToMafft( path_to_mafft ); - } - mc.setNorm( norm ); - mc.setOutFileBase( out ); - mc.setStep( step ); - mc.removeViaGapAverage( av_gap, true ); + msa_props = mc.removeViaGapAverage( av_gap ); } else if ( length > 0 ) { - // TODO if < shortest seq -> error - final MsaCompactor mc = new MsaCompactor( msa ); - mc.setRealign( realign ); - if ( realign ) { - mc.setPathToMafft( path_to_mafft ); - } - mc.setNorm( norm ); - mc.setOutFileBase( out ); - mc.setStep( step ); - mc.removeViaLength( length, true ); + msa_props = mc.removeViaLength( length ); } else { - //MsaCompactor.chart( msa, step, realign, norm, path_to_mafft ); - final int initial_number_of_seqs = msa.getNumberOfSequences(); - final MsaCompactor mc = new MsaCompactor( msa ); - mc.setRealign( realign ); - if ( realign ) { - mc.setPathToMafft( path_to_mafft ); - } - mc.setNorm( norm ); - mc.setOutFileBase( out ); - mc.setStep( step ); - final List msa_props = mc.chart( step, realign, norm, true ); - Chart.display( msa_props, initial_number_of_seqs ); + msa_props = mc.chart( step, realign, norm ); } + Chart.display( msa_props, initial_number_of_seqs, report_entropy, in.getName() ); + } + catch ( final IllegalArgumentException iae ) { + // iae.printStackTrace(); //TODO remove me + ForesterUtil.fatalError( PRG_NAME, iae.getMessage() ); + } + catch ( final IOException ioe ) { + // ioe.printStackTrace(); //TODO remove me + ForesterUtil.fatalError( PRG_NAME, ioe.getMessage() ); } catch ( final Exception e ) { - e.printStackTrace(); - ForesterUtil.fatalError( PRG_NAME, e.getMessage() ); + ForesterUtil.unexpectedFatalError( PRG_NAME, e ); } } private static void checkPathToMafft( final String path_to_mafft ) { if ( !ForesterUtil.isEmpty( path_to_mafft ) && MsaInferrer.isInstalled( path_to_mafft ) ) { - ForesterUtil.programMessage( PRG_NAME, "using MAFFT at \"" + path_to_mafft + "\"" ); } else { if ( ForesterUtil.isEmpty( path_to_mafft ) ) { @@ -356,7 +410,7 @@ public class msa_compactor { } System.out.println( "Usage:" ); System.out.println(); - System.out.println( PRG_NAME + " " ); + System.out.println( PRG_NAME + " " ); System.out.println(); System.out.println( " options: " ); System.out.println(); @@ -364,20 +418,22 @@ public class msa_compactor { + "= number of worst offender sequences to remove" ); System.out.println( " -" + LENGTH_OPTION + "= target MSA length" ); System.out.println( " -" + AV_GAPINESS_OPTION + "= target gap-ratio (0.0-1.0)" ); - System.out.println( " -" + STEP_OPTION + "= step for output and re-aligning (default: 1)" ); System.out.println( " -" + REALIGN_OPTION + " to realign using MAFFT" + mafft_comment ); System.out.println( " -" + MAFFT_OPTIONS + "= options for MAFFT (default: --auto)" ); + System.out.println( " -" + STEP_OPTION + "= step for output and re-aligning (default: 1)" ); System.out.println( " -" + STEP_FOR_DIAGNOSTICS_OPTION + "= step for diagnostics reports (default: 1)" ); + System.out.println( " -" + REPORT_ENTROPY + + " to calculate normalized Shannon Entropy (not recommended for very large alignments)" ); + System.out.println( " -" + OUTPUT_FORMAT_PHYLIP_OPTION + + " to write output alignments in phylip format instead of fasta" ); + System.out.println( " -" + OUTPUT_REMOVED_SEQS_OPTION + "= to output the removed sequences" ); System.out.println( " -" + MIN_LENGTH_OPTION + "= minimal effecive sequence length (for deleting of shorter sequences)" ); System.out.println( " -" + GAP_RATIO_LENGTH_OPTION + "= maximal allowed gap ratio per column (for deleting of columms) (0.0-1.0)" ); - System.out.println( " -" + REPORT_ALN_MEAN_IDENTITY - + " to report mean identity diagnostic (not recommended for very large alignments)" ); - System.out.println( " -" + OUTPUT_FORMAT_PHYLIP_OPTION - + " to write output alignments in phylip format instead of fasta" ); - System.out.println( " -" + OUTPUT_REMOVED_SEQS_OPTION + "= to output the removed sequences" ); + System.out.println( " -" + PERFORM_PHYLOGENETIC_INFERENCE + + " to calculate a simple phylogenetic tree (Kimura distances, NJ)" ); System.out.println(); System.out.println(); System.out.println();