From db5333c93e7d29d2a6eb301a017300583bc9f44c Mon Sep 17 00:00:00 2001 From: pvtroshin Date: Wed, 22 Jun 2011 17:11:48 +0000 Subject: [PATCH] Fix for testcases git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@4307 e3abac25-378b-4346-85de-24260fe3988d --- IDEAS.txt | 30 ++++ TODO.txt | 155 +++++++------------- conf/Executable.properties | 1 + .../engine/client/EnvVariableProcessor.java | 13 +- engine/compbio/engine/client/PathValidator.java | 2 +- temp-testng-customsuite.xml | 20 +-- testsrc/compbio/engine/LoadBalancerTester.java | 14 +- .../engine/conf/PropertyHelperManagerTester.java | 2 +- .../collector/ExecutionStatCollectorTester.java | 13 +- testsrc/compbio/stat/collector/StatDBTester.java | 46 +++--- .../stat/collector/ExecutionStatCollector.java | 8 +- webservices/compbio/stat/collector/StatDB.java | 1 + 12 files changed, 150 insertions(+), 155 deletions(-) diff --git a/IDEAS.txt b/IDEAS.txt index bab0292..3be0c19 100644 --- a/IDEAS.txt +++ b/IDEAS.txt @@ -1,5 +1,35 @@ FUTURE TODO +Add interface for Jalview annotation + +Develop generic Interface to return Jalview annotation for easy to add new +services (?) + +Philogeny Mrbayes + Philip + +USE CASE - TURN ALIGNMENT INTO PROFILE AND SEARCH SEQUENCE DATABASE USECASE +- Receive user alignment +- use hmmerbuild to turn it to profile +- use hmmersearch to search the database + +#END OF - TURN ALIGNMENT INTO PROFILE AND SEARCH SEQUENCE DATABASE USECASE + +New data model for representing psiblast,blast,phmmer,jackhmmer results + +new parsers for the above programmes output (Stockholm MSA format?) + +Think hard on what to do with large output files? +e.g. serve the hits table in full, but retrieve alignments on demand. +What actually needs to be sent? + + +Add facility to distribute other results of the calculations like the trees and +annotation file for probcons. + +(later) WRAP Amps + +(later) Implement utility to rerun died tasks from command line + Registry for jalview WS - gives list of web services providers - checks the status of the web services - web service providers gives list of available WS diff --git a/TODO.txt b/TODO.txt index 60dbccd..45a50d4 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,38 +1,13 @@ -TODO ideally should introduce native Mac support (and maybe abandon native windows support in -favior of VM) +? ideally should introduce native Mac support (and maybe abandon native windows support in +favour of VM) TODO write some help on the executable.properties within this file! check that after binaries relocation build tasks point to correct locations! -+ GET rid of binaries/help directory! -+ update binaries - + muscle binary version 3.8.31 for win/lin32/lin64 and sources, docs in website/prog_docs - - test Muscle on the cluster - - test Muscle compilation - + probcons up-to-date - + clustal up-to-date - + update tcoffee (8.99 latest) source code updated docs updated - + tcoffee compilation for IA32 - + update Mafft (6.857 latest) for win/lin32/lin64 and sources, no docs updates -+ make sure conf files are optional! - -TODO: -Registry 1 week -webservices - 1 week -client - 1 week -docs - 1 week Cluster stats: - +cancelled should not be reported as incomplete! -Remove hyperlinks from tasks which workdirs were removed -graph generation -user documentation (do not forget how to enable follow symlinks!) - -use the same name for output for different executables - +ant task to clean up clean stat db task before packing web app - +pack web apps with stat db (NOT TESTED!) - +change input size bites - +add help info for each column in the detailed job list table - +replace default -1 with ? - +delete and re-import the database to fix clustal input staff add to help text: To disable a web service remove it from WEB-INF/sun-jaxws.xml descriptor @@ -40,80 +15,21 @@ rename jaba.war to jabaws.war and make sure jabaws context path is used througho Current stat collector has to rely on file dates! -Test all WS as names from Executables were removed - -Add default names for input and output in every executable and use them consistently -throughts (e.g. in all WS). Best of all use the same name from SceletalExectuable -For statistics. - -FIXME: -Conecting to JABAWS version 2 service -09-Feb-2011 15:27:53 compbio.ws.client.Jws2Client connect -INFO: Connected successfully! -Exception in thread "main" java.lang.NullPointerException - at compbio.ws.client.MetadataHelper.getParametersList(MetadataHelper.java:30) - at compbio.ws.client.Jws2Client.(Jws2Client.java:179) - at compbio.ws.client.Jws2Client.main(Jws2Client.java:483) - -ScoreManager should output scores properly - -Allow empty Parameters and Preset files! - -Check the WS input and reject it on submission rather then of access with error message - Globprot need a proper reference to bio python and sav_gol binaries -> they should be somehow taken from disembl. -+Add registry service to query services status -+Refactor web services checker to enable a programmatic access to its methods. -+Finish the client - -Add interface for Jalview annotation -Add the method to return Jalview Annotation to SequenceAnnotation IF - -Develop generic Interface to return Jalview annotation for easy to add new -services (?) - -Replace conservation.Method with server.ws.Method -and try building WS. If this does not work - get rid of Method +? Replace conservation.Method with server.ws.Method and try building WS. If this does not work - get rid of Method Output file parsing for stat reporting -cluster engine stat of www-jws2 user +install cluster engine stat of www-jws2 user integrate the above to tweak the size of the local job -Add AACon ws -Add iupred ws http://iupred.enzim.hu/ -Add globprot ws - does not report raw scores, just regions -Add ronn ws - -Philogeny Mrbayes + Philip - -USE CASE - TURN ALIGNMENT INTO PROFILE AND SEARCH SEQUENCE DATABASE USECASE -- Receive user alignment -- use hmmerbuild to turn it to profile -- use hmmersearch to search the database - -#END OF - TURN ALIGNMENT INTO PROFILE AND SEARCH SEQUENCE DATABASE USECASE - -New data model for representing psiblast,blast,phmmer,jackhmmer results - -new parsers for the above programmes output (Stockholm MSA format?) - -Think hard on what to do with large output files? -e.g. serve the hits table in full, but retrieve alignments on demand. -What actually needs to be sent? - -Add facility to distribute other results of the calculations like the trees and -annotation file for probcons. - -# END OF SEARCHING SEQUENCE DATABASE USECASE - Good toString method for Limits (test with command line client -limits) - +Document the Limits for presets JABA DOCS -(later) Define limits for presets - if required + (later) Add documentation on Presets, Parameters and Limits (later) Explain how to define a limit @@ -121,20 +37,63 @@ Add test for repeated result collection from cluster engine (low) Pack the test cases and build file to run them in one of the distributives -(low) Add logging facility to WSTester so more details of the problem can be reported -(low) Statistics collector for engines (number of operations/timing) +DONE -(later) WRAP Amps +version 2 -(later) Implement utility to rerun died tasks +DONE: GET rid of binaries/help directory! +DONE: update binaries + DONE: muscle binary version 3.8.31 for win/lin32/lin64 and sources, docs in website/prog_docs + DONE: test Muscle on the cluster + DONE: test Muscle compilation + DONE: probcons up-to-date + DONE: clustal up-to-date + DONE: update tcoffee (8.99 latest) source code updated docs updated + DONE: tcoffee compilation for IA32 + DONE: update Mafft (6.857 latest) for win/lin32/lin64 and sources, no docs updates +DONE: make sure conf files are optional! -(later) Add recognition for executables for the different architectures +Cluster stats: + DONE: cancelled should not be reported as incomplete! + DONE: ant task to clean up clean stat db task before packing web app + DONE: pack web apps with stat db (NOT TESTED!) + DONE: change input size bites + DONE: add help info for each column in the detailed job list table + DONE: replace default -1 with ? + DONE: delete and re-import the database to fix clustal input staff + +DONE: Test all WS as names from Executables were removed +DONE: Add default names for input and output in every executable and use them consistently + throughts (e.g. in all WS). Best of all use the same name from SceletalExectuable + For statistics. + +DONE: FIXED Conecting to JABAWS version 2 service + 09-Feb-2011 15:27:53 compbio.ws.client.Jws2Client connect + INFO: Connected successfully! + Exception in thread "main" java.lang.NullPointerException + at compbio.ws.client.MetadataHelper.getParametersList(MetadataHelper.java:30) + at compbio.ws.client.Jws2Client.(Jws2Client.java:179) + at compbio.ws.client.Jws2Client.main(Jws2Client.java:483) -(later) use Latch to manage engine reservations. Reservations for parallel jobs? +DONE: ScoreManager should output scores properly +DONE: Allow empty Parameters and Preset files! +DONE: Check the WS input and reject it on submission rather then of access with error message + +DONE: Add registry service to query services status +DONE: Refactor web services checker to enable a programmatic access to its methods. +DONE: Finish the client +DONE: Add AACon ws +DONE: Add iupred ws http://iupred.enzim.hu/ +DONE: Add globprot ws - does not report raw scores, just regions +DONE: Add ronn ws +DONE: Define limits for presets - if required +DONE: Add logging facility to WSTester so more details of the problem can be reported +DONE: Statistics collector for engines (number of operations/timing) + + +version 1 -DONE -WILL NOT DO: Progress bars - needs assessing how long does it take to run a task DONE: LocalExecutor service must be shutdown on web application undeploy event not on JVM shutdown. This is because JVM is still running even after web application is shut down! DONE: Use absolute path for web site download links as archives are not included into distro! diff --git a/conf/Executable.properties b/conf/Executable.properties index 7ee672f..b08efaa 100644 --- a/conf/Executable.properties +++ b/conf/Executable.properties @@ -86,6 +86,7 @@ disembl.cluster.settings=-l h_cpu=24:00:00 -l h_vmem=6000M -l ram=6000M #local.disembl.bin.windows= local.globplot.bin=binaries/src/globplot/GlobPlot.py cluster.globplot.bin=/homes/pvtroshin/workspace/jaba2/binaries/src/globplot/GlobPlot.py +#globplot.bin.env=PYTHONHOME#/usr/local/lib/python2.6; # globplot.parameters.file=conf/settings/GlobPlotParameters.xml globplot.limits.file=conf/settings/GlobPlotLimits.xml globplot.cluster.settings=-l h_cpu=24:00:00 -l h_vmem=6000M -l ram=6000M diff --git a/engine/compbio/engine/client/EnvVariableProcessor.java b/engine/compbio/engine/client/EnvVariableProcessor.java index 3423c8f..5d161df 100644 --- a/engine/compbio/engine/client/EnvVariableProcessor.java +++ b/engine/compbio/engine/client/EnvVariableProcessor.java @@ -18,6 +18,7 @@ package compbio.engine.client; +import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -46,6 +47,7 @@ public class EnvVariableProcessor { } static String[] getEnvVariableList(String property) { + property = property.trim(); if (containsMultipleVariables(property)) { return property.split(NEXT_ENV_PROPERTY_DELIMITER); } @@ -63,7 +65,8 @@ public class EnvVariableProcessor { "Must be only one value per property! Make sure the property does not contain '" + PROP_NAME_VALUE_SEPARATOR + "' symbol!"); } - if (vars.length == 0) { + if (vars.length == 0 || vars.length==1) { + log.warn("Environmental variable '"+property+"' does not have a value!"); return ""; } return vars[1]; @@ -76,14 +79,7 @@ public class EnvVariableProcessor { return property.contains(PROP_NAME_VALUE_SEPARATOR); } - /** - * This method change relative paths to absolute for two environmental - * variables MAFFT-BINARIES and FASTA_4_MAFFT to avoid requesting every user - * to specify these path for every installation - */ - void absolutiseMafftpaths() { - } /* * This is a horrible hack, but it only requires to simplify configuration @@ -108,6 +104,7 @@ public class EnvVariableProcessor { + PROP_NAME_VALUE_SEPARATOR + "propertyValue. " + "Given values is: " + evar); } + evar=evar.trim(); String varName = getEnvVariableName(evar); String varValue = getEnvVariableValue(evar); // absolutise local paths to mafft env variables diff --git a/engine/compbio/engine/client/PathValidator.java b/engine/compbio/engine/client/PathValidator.java index 97b5315..92c5999 100644 --- a/engine/compbio/engine/client/PathValidator.java +++ b/engine/compbio/engine/client/PathValidator.java @@ -54,7 +54,7 @@ public final class PathValidator { return false; } File exec = new File(directory); - return exec.exists() && exec.isDirectory(); + return exec.exists() && exec.isDirectory() && exec.canRead(); } /** diff --git a/temp-testng-customsuite.xml b/temp-testng-customsuite.xml index b683c13..f01791c 100644 --- a/temp-testng-customsuite.xml +++ b/temp-testng-customsuite.xml @@ -1,22 +1,24 @@ - - - - + + + - +--> diff --git a/testsrc/compbio/engine/LoadBalancerTester.java b/testsrc/compbio/engine/LoadBalancerTester.java index 7d0661a..f345720 100644 --- a/testsrc/compbio/engine/LoadBalancerTester.java +++ b/testsrc/compbio/engine/LoadBalancerTester.java @@ -39,18 +39,22 @@ public class LoadBalancerTester { ClustalW clustal = new ClustalW(); /** - * ClustalW local limit is 40 sequences per 500 letters + * ClustalW local limit is 2 sequences per 500 letters */ ConfiguredExecutable confClust = Configurator .configureExecutable(clustal); ExecProvider aEngine = LoadBalancer.getEngine(confClust, data); - Limit locExec = confClust - .getLimit(PresetManager.LOCAL_ENGINE_LIMIT_PRESET); + Limit locExec = new Limit(2, 500, PresetManager.LOCAL_ENGINE_LIMIT_PRESET); + // For testing of production configuration uncomment + //Limit locExec = confClust + // .getLimit(PresetManager.LOCAL_ENGINE_LIMIT_PRESET); assertTrue(locExec.getSeqNumber() <= data.size() || locExec.getAvgSeqLength() * locExec.getSeqNumber() <= Limit .getAvgSequenceLength(data)); - - assertEquals(aEngine, ExecProvider.Cluster); + // Engine will be local because LoadBalancer accessed the presets & limits + // directly + // For testing of production configuration uncomment + // assertEquals(aEngine, ExecProvider.Cluster); } catch (JobSubmissionException e) { e.printStackTrace(); diff --git a/testsrc/compbio/engine/conf/PropertyHelperManagerTester.java b/testsrc/compbio/engine/conf/PropertyHelperManagerTester.java index 4e24134..ea4c887 100644 --- a/testsrc/compbio/engine/conf/PropertyHelperManagerTester.java +++ b/testsrc/compbio/engine/conf/PropertyHelperManagerTester.java @@ -49,7 +49,7 @@ public class PropertyHelperManagerTester { void validateDirProp(String propName) { assertNotNull(ph); assertNotNull(ph.getProperty(propName)); - File tmp = new File(ph.getProperty(propName)); + File tmp = new File(ph.getProperty(propName).trim()); assertTrue(tmp.exists()); assertTrue(tmp.isDirectory()); assertTrue(tmp.canRead()); diff --git a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java index 19f1ab9..1489dcd 100644 --- a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java +++ b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java @@ -29,18 +29,19 @@ public class ExecutionStatCollectorTester { LOCAL_JOBS, 1); ExecutionStatCollector cl_jobs = new ExecutionStatCollector( CLUSTER_JOBS, 24); - + + // Collect statistics prior to call getStats()! + local_jobs.collectStatistics(); + cl_jobs.collectStatistics(); + StatProcessor local_stats = local_jobs.getStats(); StatProcessor cl_stats = cl_jobs.getStats(); - // System.out.println("L: " + local_stats.stats); - System.out.println("\n\n"); - // System.out.println("C: " + cl_stats.stats); - + assertEquals(local_stats.getJobNumber(), 12); // ClustalW#1015343425414965 - empty assertEquals(local_stats.getIncompleteJobs().size(), 1); assertEquals(local_stats.getFailedJobs().size(), 0); - assertEquals(local_stats.getAbandonedJobs().size(), 2); + assertEquals(local_stats.getAbandonedJobs().size(), 1); } diff --git a/testsrc/compbio/stat/collector/StatDBTester.java b/testsrc/compbio/stat/collector/StatDBTester.java index 010b6ea..4b135dd 100644 --- a/testsrc/compbio/stat/collector/StatDBTester.java +++ b/testsrc/compbio/stat/collector/StatDBTester.java @@ -19,11 +19,21 @@ import compbio.ws.client.Services; public class StatDBTester { StatDB statdb; - - @BeforeClass + Timestamp from; + Timestamp to; + + @BeforeClass(alwaysRun=true) public void init() { try { statdb = new StatDB(true); + Calendar fromCal = Calendar.getInstance(); + fromCal.set(2011, 4, 1); + + Calendar toCal = Calendar.getInstance(); + toCal.set(2011, 5, 1); + from = new Timestamp(fromCal.getTimeInMillis()); + to = new Timestamp(toCal.getTimeInMillis()); + } catch (SQLException e) { e.printStackTrace(); Assert.fail(e.getLocalizedMessage()); @@ -33,11 +43,10 @@ public class StatDBTester { @Test public void testReadYearDataAndShutdown() { Calendar cal = Calendar.getInstance(); - cal.roll(Calendar.YEAR, false); + cal.set(2010, 4, 1); try { List jobs = statdb.readData( - new Timestamp(cal.getTimeInMillis()), new Timestamp( - new Date().getTime()), Services.MuscleWS, false); + new Timestamp(cal.getTimeInMillis()), to, Services.MuscleWS, false); assertNotNull(jobs); assertEquals(jobs.size(), 1294); // statdb.shutdownDBServer(); @@ -47,43 +56,28 @@ public class StatDBTester { } } + @Test public void testReadOneMonthData() { - Calendar fromCal = Calendar.getInstance(); - fromCal.set(2011, 4, 1); - - Calendar toCal = Calendar.getInstance(); - toCal.set(2011, 5, 1); - try { - List jobs = statdb.readData( - new Timestamp(fromCal.getTimeInMillis()), - new Timestamp(toCal.getTimeInMillis()), Services.TcoffeeWS, false); + List jobs = statdb.readData(from, to, Services.TcoffeeWS, false); assertNotNull(jobs); assertEquals(jobs.size(), 36); - jobs = statdb.readData(new Timestamp(fromCal.getTimeInMillis()), - new Timestamp(toCal.getTimeInMillis()), Services.ClustalWS, - false); + jobs = statdb.readData(from,to, Services.ClustalWS,false); assertNotNull(jobs); assertEquals(jobs.size(), 137); - jobs = statdb.readData(new Timestamp(fromCal.getTimeInMillis()), - new Timestamp(toCal.getTimeInMillis()), Services.MafftWS, - false); + jobs = statdb.readData(from, to, Services.MafftWS,false); assertNotNull(jobs); assertEquals(jobs.size(), 126); - jobs = statdb.readData(new Timestamp(fromCal.getTimeInMillis()), - new Timestamp(toCal.getTimeInMillis()), Services.ProbconsWS, - false); + jobs = statdb.readData(from,to, Services.ProbconsWS,false); assertNotNull(jobs); assertEquals(jobs.size(), 9); - jobs = statdb.readData(new Timestamp(fromCal.getTimeInMillis()), - new Timestamp(toCal.getTimeInMillis()), Services.MuscleWS, - false); + jobs = statdb.readData(from,to, Services.MuscleWS,false); assertNotNull(jobs); assertEquals(jobs.size(), 63); diff --git a/webservices/compbio/stat/collector/ExecutionStatCollector.java b/webservices/compbio/stat/collector/ExecutionStatCollector.java index 785f659..ec35a15 100644 --- a/webservices/compbio/stat/collector/ExecutionStatCollector.java +++ b/webservices/compbio/stat/collector/ExecutionStatCollector.java @@ -102,7 +102,13 @@ public class ExecutionStatCollector implements Runnable { return ((System.currentTimeMillis() - jd.jobdir.lastModified()) / (1000 * 60 * 60)) > timeOutInHours; } + /* Make sure that collectStatistics methods was called prior to calling this! + * TODO consider running collectStatistics from here on the first call + */ StatProcessor getStats() { + if(stats.isEmpty()) { + log.info("Please make sure collectStatistics method was called prior to calling getStats()!"); + } return new StatProcessor(stats); } @@ -314,7 +320,7 @@ public class ExecutionStatCollector implements Runnable { } } - private void collectStatistics() { + void collectStatistics() { File[] files = workDirectory.listFiles(directories); for (File file : files) { JobDirectory jd = new JobDirectory(file); diff --git a/webservices/compbio/stat/collector/StatDB.java b/webservices/compbio/stat/collector/StatDB.java index 98b6fda..08b99f9 100644 --- a/webservices/compbio/stat/collector/StatDB.java +++ b/webservices/compbio/stat/collector/StatDB.java @@ -76,6 +76,7 @@ public class StatDB { Connection conn = DriverManager.getConnection(protocol + statDBName + ";create=false"); conn.setAutoCommit(true); + log.debug("Connecting to the TEST database!"); return conn; } -- 1.7.10.2