From: pvtroshin Date: Tue, 14 Jun 2011 17:12:12 +0000 (+0000) Subject: IUPred wrapper, tester and new binary for X64 linux systems X-Git-Url: http://source.jalview.org/gitweb/?p=jabaws.git;a=commitdiff_plain;h=66c8ec5cbad54362baa57c9d9db6a69834d2bb0a IUPred wrapper, tester and new binary for X64 linux systems git-svn-id: link to svn.lifesci.dundee.ac.uk/svn/barton/ptroshin/JABA2@4265 e3abac25-378b-4346-85de-24260fe3988d --- diff --git a/binaries/src/iupred/iupred b/binaries/src/iupred/iupred index ce0fa41..42a76b0 100644 Binary files a/binaries/src/iupred/iupred and b/binaries/src/iupred/iupred differ diff --git a/datamodel/compbio/data/sequence/SequenceUtil.java b/datamodel/compbio/data/sequence/SequenceUtil.java index f65ec9e..1c6d2e8 100644 --- a/datamodel/compbio/data/sequence/SequenceUtil.java +++ b/datamodel/compbio/data/sequence/SequenceUtil.java @@ -319,10 +319,11 @@ public final class SequenceUtil { * @throws IOException * @throws UnknownFileFormatException */ - public static Map readIUPred(final File result, - IUPredResult type) throws IOException, UnknownFileFormatException { + public static Map readIUPred(final File result) + throws IOException, UnknownFileFormatException { InputStream input = new FileInputStream(result); - Map sequences = readIUPred(input, type); + Map sequences = readIUPred(input, + IUPredResult.getType(result)); input.close(); return sequences; } @@ -357,7 +358,7 @@ public final class SequenceUtil { while (scan.hasNext()) { String nextEntry = scan.next(); Scanner entry = new Scanner(nextEntry); - String name = entry.nextLine(); + String name = entry.nextLine().trim(); // inside entry: if (IUPredResult.Glob == type) { // parse domains @@ -845,5 +846,23 @@ enum IUPredResult { /** * Globular domains */ - Glob + Glob; + + static IUPredResult getType(File file) { + assert file != null; + String name = file.getName(); + if (name.endsWith(Long.toString().toLowerCase())) { + return Long; + } + if (name.endsWith(Short.toString().toLowerCase())) { + return Short; + } + if (name.endsWith(Glob.toString().toLowerCase())) { + return Glob; + } + throw new AssertionError( + "IUPred result file type cannot be recognised! " + + "\nFile must ends with one of [glob, long or short]" + + "\n but given file name was: " + file.getName()); + } } \ No newline at end of file diff --git a/runner/compbio/runner/disorder/IUPred.java b/runner/compbio/runner/disorder/IUPred.java new file mode 100644 index 0000000..30e8776 --- /dev/null +++ b/runner/compbio/runner/disorder/IUPred.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2011 Peter Troshin JAva Bioinformatics Analysis Web Services + * (JABAWS) @version: 2.0 This library is free software; you can redistribute it + * and/or modify it under the terms of the Apache License version 2 as published + * by the Apache Software Foundation This library is distributed in the hope + * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. A copy of the license is in + * apache_license.txt. It is also available here: + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt Any republication or + * derived work distributed in source code form must include this copyright and + * license notice. + */ + +package compbio.runner.disorder; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.log4j.Logger; + +import compbio.data.sequence.Score; +import compbio.data.sequence.ScoreManager; +import compbio.data.sequence.SequenceUtil; +import compbio.data.sequence.UnknownFileFormatException; +import compbio.engine.client.SkeletalExecutable; +import compbio.metadata.ResultNotAvailableException; + +/** + * iupred sequenceFile + * + * Maximum sequence length is 40000 chars. Single string length max is a 1000 + * chars! + * + */ +public class IUPred extends SkeletalExecutable { + + private static Logger log = Logger.getLogger(IUPred.class); + private static final String GLOB_SUFFIX = ".glob"; + private static final String SHORT_SUFFIX = ".short"; + private static final String LONG_SUFFIX = ".long"; + + @Override + @SuppressWarnings("unchecked") + public ScoreManager getResults(String workDirectory) + throws ResultNotAvailableException { + + ScoreManager results = null; + try { + + Map globScores = SequenceUtil.readIUPred(new File( + workDirectory, getOutput() + GLOB_SUFFIX)); + Map shortScores = SequenceUtil.readIUPred(new File( + workDirectory, getOutput() + SHORT_SUFFIX)); + Map longScores = SequenceUtil.readIUPred(new File( + workDirectory, getOutput() + LONG_SUFFIX)); + Map> combined = new TreeMap>(); + for (String key : globScores.keySet()) { + Set all = new TreeSet(); + Score globsc = globScores.get(key); + assert globsc != null; + Score shortsc = shortScores.get(key); + assert shortsc != null; + all.add(shortsc); + Score longsc = longScores.get(key); + assert longsc != null; + all.add(longsc); + combined.put(key, all); + } + results = ScoreManager.newInstance(combined); + + } catch (FileNotFoundException e) { + log.error(e.getMessage(), e.getCause()); + throw new ResultNotAvailableException(e); + } catch (IOException e) { + log.error(e.getMessage(), e.getCause()); + throw new ResultNotAvailableException(e); + } catch (UnknownFileFormatException e) { + log.error(e.getMessage(), e.getCause()); + throw new ResultNotAvailableException(e); + } catch (NullPointerException e) { + log.error(e.getMessage(), e.getCause()); + throw new ResultNotAvailableException(e); + } + return results; + } + + @Override + public IUPred setInput(String inFile) { + super.setInput(inFile); + cbuilder.setFirst(inFile); + return this; + } + + @SuppressWarnings("unchecked") + @Override + public Class getType() { + return (Class) this.getClass(); + } + +} diff --git a/testsrc/compbio/data/sequence/SequenceUtilTester.java b/testsrc/compbio/data/sequence/SequenceUtilTester.java index d992216..7621dcc 100644 --- a/testsrc/compbio/data/sequence/SequenceUtilTester.java +++ b/testsrc/compbio/data/sequence/SequenceUtilTester.java @@ -291,10 +291,17 @@ public class SequenceUtilTester { public void testReadIUPredForShortAndLongDisorder() { try { Map scores = SequenceUtil.readIUPred(new File( - AllTestSuit.TEST_DATA_PATH, "output.long"), - IUPredResult.Long); + AllTestSuit.TEST_DATA_PATH, "output.long")); ScoreManager man = ScoreManager.newInstanceSingleScore(scores); - man.writeOut(new PrintWriter(System.out, true)); + // man.writeOut(new PrintWriter(System.out, true)); + assertNotNull(scores); + assertEquals(2, scores.size()); + + Score score = scores.get("P50_HUMAN"); + assertNotNull(score); + assertEquals(0, score.getRanges().size()); + assertEquals(393, score.getScores().size()); + assertEquals("Long", score.getMethod()); } catch (IOException e) { e.printStackTrace(); fail(e.getLocalizedMessage()); @@ -308,10 +315,20 @@ public class SequenceUtilTester { public void testReadIUPredForGlobDomain() { try { Map scores = SequenceUtil.readIUPred(new File( - AllTestSuit.TEST_DATA_PATH, "output.glob"), - IUPredResult.Glob); + AllTestSuit.TEST_DATA_PATH, "output.glob")); + assertNotNull(scores); + assertEquals(2, scores.size()); ScoreManager man = ScoreManager.newInstanceSingleScore(scores); - man.writeOut(new PrintWriter(System.out, true)); + // man.writeOut(new PrintWriter(System.out, true)); + assertEquals(2, man.getNumberOfSeq()); + Score score = scores.get("P53_HUMA"); + assertNotNull(score); + assertEquals(2, score.getRanges().size()); + assertEquals(0, score.getScores().size()); + assertEquals("Glob", score.getMethod()); + + score = scores.get("Foobar_dundeefriends"); + assertEquals(0, score.getRanges().size()); } catch (IOException e) { e.printStackTrace(); fail(e.getLocalizedMessage()); @@ -320,7 +337,6 @@ public class SequenceUtilTester { fail(e.getLocalizedMessage()); } } - @Test public void testReadAAConResults() { try { diff --git a/testsrc/compbio/runner/disorder/IUPredTester.java b/testsrc/compbio/runner/disorder/IUPredTester.java new file mode 100644 index 0000000..1ef709b --- /dev/null +++ b/testsrc/compbio/runner/disorder/IUPredTester.java @@ -0,0 +1,310 @@ +/* Copyright (c) 2011 Peter Troshin + * + * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 2.0 + * + * This library is free software; you can redistribute it and/or modify it under the terms of the + * Apache License version 2 as published by the Apache Software Foundation + * + * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache + * License for more details. + * + * A copy of the license is in apache_license.txt. It is also available here: + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt + * + * Any republication or derived work distributed in source code form + * must include this copyright and license notice. + */ + +package compbio.runner.disorder; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.ParseException; + +import javax.xml.bind.ValidationException; + +import org.ggf.drmaa.DrmaaException; +import org.ggf.drmaa.JobInfo; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import compbio.data.sequence.ScoreManager; +import compbio.engine.AsyncExecutor; +import compbio.engine.Configurator; +import compbio.engine.SyncExecutor; +import compbio.engine.client.ConfExecutable; +import compbio.engine.client.ConfiguredExecutable; +import compbio.engine.client.Executable; +import compbio.engine.client.RunConfiguration; +import compbio.engine.cluster.drmaa.ClusterUtil; +import compbio.engine.cluster.drmaa.JobRunner; +import compbio.engine.cluster.drmaa.StatisticManager; +import compbio.engine.local.LocalRunner; +import compbio.metadata.AllTestSuit; +import compbio.metadata.JobExecutionException; +import compbio.metadata.JobStatus; +import compbio.metadata.JobSubmissionException; +import compbio.metadata.LimitsManager; +import compbio.metadata.PresetManager; +import compbio.metadata.ResultNotAvailableException; +import compbio.metadata.RunnerConfig; +import compbio.util.SysPrefs; + +public class IUPredTester { + + public static String test_outfile = "output"; + + private IUPred iupred; + + @BeforeMethod(alwaysRun = true) + void init() { + iupred = new IUPred(); + iupred.setInput(AllTestSuit.test_input).setOutput(test_outfile); + } + + @Test(groups = {AllTestSuit.test_group_cluster, + AllTestSuit.test_group_runner}) + public void testRunOnCluster() { + assertFalse(SysPrefs.isWindows, + "Cluster execution can only be in unix environment"); + try { + ConfiguredExecutable confIUPred = Configurator + .configureExecutable(iupred, + Executable.ExecProvider.Cluster); + JobRunner runner = JobRunner.getInstance(confIUPred); + + assertNotNull(runner, "Runner is NULL"); + runner.executeJob(); + // assertNotNull("JobId is null", jobId1); + JobStatus status = runner.getJobStatus(); + assertTrue(status == JobStatus.PENDING + || status == JobStatus.RUNNING, + "Status of the process is wrong!"); + JobInfo info = runner.getJobInfo(); + assertNotNull(info, "JobInfo is null"); + StatisticManager sm = new StatisticManager(info); + assertNotNull(sm, "Statictic manager is null"); + try { + + String exits = sm.getExitStatus(); + assertNotNull("Exit status is null", exits); + // cut 4 trailing zeros from the number + int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits) + .intValue(); + assertEquals(0, exitsInt); + System.out.println(sm.getAllStats()); + + } catch (ParseException e) { + e.printStackTrace(); + fail("Parse Exception: " + e.getMessage()); + } + // assertFalse(runner.cleanup()); + assertTrue(sm.hasExited()); + assertFalse(sm.wasAborted()); + assertFalse(sm.hasDump()); + assertFalse(sm.hasSignaled()); + + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (DrmaaException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } + } + + /** + * This tests fails from time to time depending on the cluster load or some + * other factors. Any client code has to adjust for this issue + */ + @Test(groups = {AllTestSuit.test_group_cluster, + AllTestSuit.test_group_runner}) + public void testRunOnClusterAsync() { + assertFalse(SysPrefs.isWindows, + "Cluster execution can only be in unix environment"); + try { + ConfiguredExecutable confIUPred = Configurator + .configureExecutable(iupred, + Executable.ExecProvider.Cluster); + AsyncExecutor aengine = Configurator.getAsyncEngine(confIUPred); + String jobId = aengine.submitJob(confIUPred); + assertNotNull(jobId, "Runner is NULL"); + // let drmaa to start + Thread.sleep(500); + JobStatus status = aengine.getJobStatus(jobId); + while (status != JobStatus.FINISHED) { + System.out.println("Job Status: " + status); + Thread.sleep(1000); + status = aengine.getJobStatus(jobId); + ConfiguredExecutable result = (ConfiguredExecutable) aengine + .getResults(jobId); + assertNotNull(result); + System.out.println("RES:" + result); + // Some times the job could be removed from the cluster + // accounting + // before it has been reported to finish. Make sure + // to stop waiting in such case + if (status == JobStatus.UNDEFINED) { + break; + } + } + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testRunLocally() { + try { + ConfiguredExecutable confIUPred = Configurator + .configureExecutable(iupred, Executable.ExecProvider.Local); + + // For local execution use relative + LocalRunner lr = new LocalRunner(confIUPred); + lr.executeJob(); + ConfiguredExecutable al1 = lr.waitForResult(); + assertNotNull(al1.getResults()); + ScoreManager al2 = confIUPred.getResults(); + assertNotNull(al2); + assertEquals(al2.asMap().size(), 3); + assertEquals(al1.getResults(), al2); + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void readStatistics() { + // No execution statistics is available! + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testPersistance() { + try { + IUPred disembl = new IUPred(); + disembl.setError("errrr.txt").setInput(AllTestSuit.test_input) + .setOutput("outtt.txt"); + assertEquals(disembl.getInput(), AllTestSuit.test_input); + assertEquals(disembl.getError(), "errrr.txt"); + assertEquals(disembl.getOutput(), "outtt.txt"); + ConfiguredExecutable cIUPred = Configurator + .configureExecutable(disembl, Executable.ExecProvider.Local); + + SyncExecutor sexec = Configurator.getSyncEngine(cIUPred); + sexec.executeJob(); + ConfiguredExecutable al = sexec.waitForResult(); + assertNotNull(al.getResults()); + // Save run configuration + assertTrue(cIUPred.saveRunConfiguration()); + + // See if loaded configuration is the same as saved + RunConfiguration loadedRun = RunConfiguration + .load(new FileInputStream(new File(cIUPred + .getWorkDirectory(), RunConfiguration.rconfigFile))); + assertEquals( + ((ConfExecutable) cIUPred).getRunConfiguration(), + loadedRun); + // Load run configuration as ConfExecutable + ConfiguredExecutable resurrectedCIUPred = (ConfiguredExecutable) cIUPred + .loadRunConfiguration(new FileInputStream(new File(cIUPred + .getWorkDirectory(), RunConfiguration.rconfigFile))); + assertNotNull(resurrectedCIUPred); + assertEquals(resurrectedCIUPred.getExecutable().getInput(), + AllTestSuit.test_input); + assertEquals(resurrectedCIUPred.getExecutable().getError(), + "errrr.txt"); + assertEquals(resurrectedCIUPred.getExecutable().getOutput(), + "outtt.txt"); + // See in details whether executables are the same + assertEquals(resurrectedCIUPred.getExecutable(), disembl); + + ConfiguredExecutable resIUPred = Configurator + .configureExecutable(resurrectedCIUPred.getExecutable(), + Executable.ExecProvider.Local); + + sexec = Configurator.getSyncEngine(resIUPred, + Executable.ExecProvider.Local); + sexec.executeJob(); + al = sexec.waitForResult(); + assertNotNull(al); + + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testConfigurationLoading() { + try { + RunnerConfig disemblConfig = ConfExecutable + .getRunnerOptions(IUPred.class); + // There is no disembl parameters + assertNull(disemblConfig); + + // If there were a IUPredParameter.xml file and reference to it, + // it would be like that + // assertTrue(disemblConfig.getArguments().size() == 0); + + PresetManager disemblPresets = ConfExecutable + .getRunnerPresets(IUPred.class); + assertNull(disemblPresets); // there is no presets + + LimitsManager disemblLimits = ConfExecutable + .getRunnerLimits(IUPred.class); + assertNotNull(disemblLimits); + assertTrue(disemblLimits.getLimits().size() > 0); + disemblLimits.validate(disemblPresets); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (ValidationException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } + } + +}