From e958f5b5d56d9bd9d865f04ed3a173e3175fbb22 Mon Sep 17 00:00:00 2001 From: Sasha Sherstnev Date: Wed, 7 Aug 2013 17:28:09 +0100 Subject: [PATCH] Necessary modifications in JABAWS for Jpred --- datamodel/compbio/data/sequence/Program.java | 6 +- datamodel/compbio/data/sequence/SequenceUtil.java | 10 +- runner/compbio/runner/predictors/Jpred.java | 18 +- testsrc/compbio/runner/predictors/JpredTester.java | 363 ++++++++++++++++++++ 4 files changed, 388 insertions(+), 9 deletions(-) create mode 100644 testsrc/compbio/runner/predictors/JpredTester.java diff --git a/datamodel/compbio/data/sequence/Program.java b/datamodel/compbio/data/sequence/Program.java index 2ae3c3a..e5f4e76 100644 --- a/datamodel/compbio/data/sequence/Program.java +++ b/datamodel/compbio/data/sequence/Program.java @@ -49,5 +49,9 @@ public enum Program { /** * Probcons */ - Probcons + Probcons, + /** + * Jpred + */ + Jpred } diff --git a/datamodel/compbio/data/sequence/SequenceUtil.java b/datamodel/compbio/data/sequence/SequenceUtil.java index 14dedf4..3a19e44 100644 --- a/datamodel/compbio/data/sequence/SequenceUtil.java +++ b/datamodel/compbio/data/sequence/SequenceUtil.java @@ -23,6 +23,7 @@ import java.io.BufferedWriter; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -30,6 +31,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.util.ArrayList; import java.util.HashMap; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -286,6 +288,12 @@ public final class SequenceUtil { outWriter.close(); } + + public static final List readJpredFile(InputStream result) + throws IOException, FileNotFoundException,NullPointerException { + return readFasta (result); + } + /** * Read IUPred output * @@ -299,7 +307,7 @@ public final class SequenceUtil { InputStream input = new FileInputStream(result); Map sequences = readIUPred(input, IUPredResult.getType(result)); - input.close(); + input.close(); return sequences; } diff --git a/runner/compbio/runner/predictors/Jpred.java b/runner/compbio/runner/predictors/Jpred.java index 017dcfc..7c27a37 100644 --- a/runner/compbio/runner/predictors/Jpred.java +++ b/runner/compbio/runner/predictors/Jpred.java @@ -28,17 +28,21 @@ import java.util.List; import org.apache.log4j.Logger; +import compbio.data.sequence.Alignment; +import compbio.data.sequence.AlignmentMetadata; +import compbio.data.sequence.Program; import compbio.data.sequence.ScoreManager; import compbio.data.sequence.SequenceUtil; import compbio.engine.client.CommandBuilder; import compbio.engine.client.Executable; import compbio.engine.client.SkeletalExecutable; import compbio.metadata.ResultNotAvailableException; +import compbio.runner.Util; /** * Command line * - * jpred.pl -in d16vpa_.fas -out res_d16vpa_ -dbname ported_db -dbpath /data/UNIREFdb -ncpu 4 + * jpred.pl -in d16vpa_.fas -outfile res_d16vpa_ -dbname ported_db -dbpath /data/UNIREFdb -ncpu 4 * * @author asherstnev * @@ -57,17 +61,17 @@ public class Jpred extends SkeletalExecutable { public static final String STAT_FILE = "stat.txt"; public Jpred() { -// addParameters(Arrays.asList()); + addParameters(Arrays.asList("-logfile " + STAT_FILE)); } + // HashMap @Override - public ScoreManager getResults(String workDirectory) + public Alignment getResults(String workDirectory) throws ResultNotAvailableException { - ScoreManager annotations = null; + Alignment annotations = null; try { InputStream inStream = new FileInputStream(new File(workDirectory, getOutput())); - annotations = ScoreManager.newInstanceSingleSequence(SequenceUtil.readAAConResults(inStream)); - inStream.close(); + annotations = new Alignment(SequenceUtil.readJpredFile(inStream), new AlignmentMetadata(Program.Jpred, '-')); } catch (FileNotFoundException e) { log.error(e.getMessage(), e.getCause()); throw new ResultNotAvailableException(e); @@ -96,7 +100,7 @@ public class Jpred extends SkeletalExecutable { @Override public Jpred setOutput(String outFile) { super.setOutput(outFile); - cbuilder.setParam("-out " + outFile); + cbuilder.setParam("-outfile " + outFile); return this; } diff --git a/testsrc/compbio/runner/predictors/JpredTester.java b/testsrc/compbio/runner/predictors/JpredTester.java new file mode 100644 index 0000000..7d6c61d --- /dev/null +++ b/testsrc/compbio/runner/predictors/JpredTester.java @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2010 Alexander Sherstnev, Java Bioinformatics Analysis Web Services + * (JABAWS) @version: 2.5 + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the Apache License version 2 as published + * by the Apache Software Foundation This library is distributed in the hope + * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. A copy of the license is in + * apache_license.txt. It is also available here: + * + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt + * + * Any republication or derived work distributed in source code form must include + * this copyright and license notice. + */ +package compbio.runner.predictors; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; + +import javax.xml.bind.ValidationException; + +import org.ggf.drmaa.DrmaaException; +import org.ggf.drmaa.JobInfo; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import compbio.data.sequence.Alignment; +import compbio.engine.AsyncExecutor; +import compbio.engine.Configurator; +import compbio.engine.FilePuller; +import compbio.engine.SyncExecutor; +import compbio.engine.client.ConfExecutable; +import compbio.engine.client.ConfiguredExecutable; +import compbio.engine.client.Executable; +import compbio.engine.client.RunConfiguration; +import compbio.engine.cluster.drmaa.ClusterUtil; +import compbio.engine.cluster.drmaa.JobRunner; +import compbio.engine.cluster.drmaa.StatisticManager; +import compbio.engine.local.LocalRunner; +import compbio.metadata.AllTestSuit; +import compbio.metadata.ChunkHolder; +import compbio.metadata.JobExecutionException; +import compbio.metadata.JobStatus; +import compbio.metadata.JobSubmissionException; +import compbio.metadata.LimitsManager; +import compbio.metadata.Preset; +import compbio.metadata.PresetManager; +import compbio.metadata.ResultNotAvailableException; +import compbio.metadata.RunnerConfig; +import compbio.runner.Util; +import compbio.util.FileWatcher; +import compbio.util.SysPrefs; + +public class JpredTester { + + public static final String CURRENT_DIRECTORY = SysPrefs.getCurrentDirectory() + File.separator; + + public static String test_output = "Jpred.test1.out"; + public static String test_input = CURRENT_DIRECTORY + "testsrc" + File.separator + "testdata" + File.separator + "Jpred.test1.fasta"; + private Jpred pred; + + @BeforeMethod(alwaysRun = true) + void init() { + pred = new Jpred(); + pred.setInput(test_input); + pred.setOutput(test_output); + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testRunOnCluster() { + assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment"); + try { + PresetManager jpredPreset = Util.getPresets(Jpred.class); + assert jpredPreset != null; + ConfiguredExecutable confpred = Configurator.configureExecutable(pred, Executable.ExecProvider.Cluster); + Preset conf = jpredPreset.getPresetByName("cluster configuration"); + confpred.addParameters(conf.getOptions()); + JobRunner runner = JobRunner.getInstance(confpred); + assertNotNull(runner, "Runner is NULL"); + + runner.executeJob(); + JobStatus status = runner.getJobStatus(); + assertTrue(status == JobStatus.PENDING || status == JobStatus.RUNNING, "Status of the process is wrong!"); + JobInfo info = runner.getJobInfo(); + assertNotNull(info, "JobInfo is null"); + StatisticManager sm = new StatisticManager(info); + assertNotNull(sm, "Statistic manager is null"); + try { + String exits = sm.getExitStatus(); + assertNotNull("Exit status is null", exits); + // cut 4 trailing zeros from the number + int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits).intValue(); + assertEquals(0, exitsInt); + System.out.println(sm.getAllStats()); + } catch (ParseException e) { + e.printStackTrace(); + fail("Parse Exception: " + e.getMessage()); + } + assertTrue(sm.hasExited()); + assertFalse(sm.wasAborted()); + assertFalse(sm.hasDump()); + assertFalse(sm.hasSignaled()); + + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (DrmaaException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } + } + + /** + * This tests fails from time to time depending on the cluster load or some + * other factors. Any client code has to adjust for this issue + */ + @Test(groups = {AllTestSuit.test_group_runner, AllTestSuit.test_group_cluster}) + public void testRunOnClusterAsync() { + assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment"); + try { + pred.addParameters(Arrays.asList("-dbname uniref90", "-dbpath /homes/www-jpred/databases/")); + ConfiguredExecutable confpred = Configurator.configureExecutable(pred, Executable.ExecProvider.Cluster); + AsyncExecutor aengine = Configurator.getAsyncEngine(confpred); + String jobId = aengine.submitJob(confpred); + assertNotNull(jobId, "Runner is NULL"); + // let drmaa to start + Thread.sleep(500); + JobStatus status = aengine.getJobStatus(jobId); + while (status != JobStatus.FINISHED) { + Thread.sleep(1000); + status = aengine.getJobStatus(jobId); + ConfiguredExecutable result = (ConfiguredExecutable) aengine.getResults(jobId); + assertNotNull(result); + if (status == JobStatus.UNDEFINED || status == JobStatus.FAILED) { + fail("job " + jobId +" failed!"); + break; + } + } + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail("DrmaaException caught:" + e.getMessage()); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testRunLocally() { + try { + ConfiguredExecutable confpred = Configurator.configureExecutable(pred, Executable.ExecProvider.Local); + confpred.addParameters(Arrays.asList("-dbname ported_db", "-dbpath /data/UNIREFdb/")); + + // For local execution use relative + LocalRunner lr = new LocalRunner(confpred); + lr.executeJob(); + ConfiguredExecutable al1 = lr.waitForResult(); + assertNotNull(al1.getResults()); + Alignment annotations = confpred.getResults(); + assertNotNull(annotations); + assertEquals(annotations.getSize(), 19); + assertEquals(al1.getResults(), annotations); + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testRunLocallyWithPreset() { + try { + ConfiguredExecutable confpred = Configurator.configureExecutable(pred, Executable.ExecProvider.Local); + PresetManager preset = Util.getPresets(Jpred.class); + assert preset != null; + Preset conf = preset.getPresetByName("laptop configuration"); + confpred.addParameters(conf.getOptions()); + + // For local execution use relative + LocalRunner lr = new LocalRunner(confpred); + lr.executeJob(); + ConfiguredExecutable al1 = lr.waitForResult(); + assertNotNull(al1.getResults()); + Alignment annotations = confpred.getResults(); + assertNotNull(annotations); + assertEquals(annotations.getSize(), 19); + assertEquals(al1.getResults(), annotations); + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void readStatistics() { + try { + Jpred jpred = new Jpred(); + jpred.setInput(test_input); + jpred.setOutput(test_output); + ConfiguredExecutable confpred = Configurator.configureExecutable(jpred, Executable.ExecProvider.Local); + PresetManager preset = Util.getPresets(Jpred.class); + assert preset != null; + Preset conf = preset.getPresetByName("laptop configuration"); + confpred.addParameters(conf.getOptions()); + + // For local execution use relative + AsyncExecutor sexec = Configurator.getAsyncEngine(confpred); + String jobId = sexec.submitJob(confpred); + String file = confpred.getWorkDirectory() + File.separator + Jpred.getStatFile(); + FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES); + int count = 0; + long position = 0; + fw.waitForFile(2); + JobStatus status = sexec.getJobStatus(jobId); + do { + if (fw.hasMoreData()) { + ChunkHolder ch = fw.pull(position); + String chunk = ch.getChunk(); + position = ch.getNextPosition(); + } + count++; + // Make sure the loop is terminated if the job fails + if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) { + fail("job " + jobId +" failed!"); + break; + } + Thread.sleep(200); + status = sexec.getJobStatus(jobId); + } while (status != JobStatus.FINISHED || fw.hasMoreData()); + + assertTrue(count >= 1); + ConfiguredExecutable al = sexec.getResults(jobId); + assertNotNull(al.getResults()); + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testPersistance() { + try { + Jpred jpred = new Jpred(); + jpred.setError("errrr.txt"); + jpred.setInput(test_input); + jpred.setOutput("outtt.txt"); + assertEquals(jpred.getInput(), test_input); + assertEquals(jpred.getError(), "errrr.txt"); + assertEquals(jpred.getOutput(), "outtt.txt"); + ConfiguredExecutable confpred = Configurator.configureExecutable(jpred, Executable.ExecProvider.Local); + confpred.addParameters(Arrays.asList("-dbname ported_db", "-dbpath /data/UNIREFdb/")); + + SyncExecutor sexec = Configurator.getSyncEngine(confpred); + sexec.executeJob(); + ConfiguredExecutable al = sexec.waitForResult(); + assertNotNull(al.getResults()); + + // Save run configuration + assertTrue(confpred.saveRunConfiguration()); + + // See if loaded configuration is the same as saved + RunConfiguration loadedRun = RunConfiguration.load(new FileInputStream(new File(confpred.getWorkDirectory(), RunConfiguration.rconfigFile))); + assertEquals(((ConfExecutable) confpred).getRunConfiguration(),loadedRun); + + // Load run configuration as ConfExecutable + ConfiguredExecutable jpred2 = (ConfiguredExecutable) confpred.loadRunConfiguration(new FileInputStream(new File(confpred.getWorkDirectory(), RunConfiguration.rconfigFile))); + assertNotNull(jpred2); + assertEquals(jpred2.getExecutable().getInput(), test_input); + assertEquals(jpred2.getExecutable().getError(), "errrr.txt"); + assertEquals(jpred2.getExecutable().getOutput(), "outtt.txt"); + + // See in details whether executables are the same + assertEquals(jpred2.getExecutable(), jpred); + ConfiguredExecutable resjpred2 = Configurator.configureExecutable(jpred2.getExecutable(), Executable.ExecProvider.Local); + + sexec = Configurator.getSyncEngine(resjpred2, Executable.ExecProvider.Local); + sexec.executeJob(); + al = sexec.waitForResult(); + assertNotNull(al); + } catch (JobSubmissionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (JobExecutionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } catch (ResultNotAvailableException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test(groups = {AllTestSuit.test_group_runner}) + public void testConfigurationLoading() { + try { + RunnerConfig jpredConfig = ConfExecutable.getRunnerOptions(Jpred.class); + assertNotNull(jpredConfig); + assertTrue(jpredConfig.getArguments().size() > 0); + + PresetManager jpredPreset = ConfExecutable.getRunnerPresets(Jpred.class); + assertNotNull(jpredPreset); + + LimitsManager jpredLimits = ConfExecutable.getRunnerLimits(Jpred.class); + assertNotNull(jpredLimits); + assertTrue(jpredLimits.getLimits().size() > 0); + jpredLimits.validate(jpredPreset); + } catch (FileNotFoundException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } catch (ValidationException e) { + e.printStackTrace(); + fail(e.getLocalizedMessage()); + } + } + +} -- 1.7.10.2