testsrc/compbio/runner/msa/ClustalOTester.java

   1 /* Copyright (c) 2009 Peter Troshin\r
   2  *  \r
   3  *  JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0     \r
   4  * \r
   5  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
   6  *  Apache License version 2 as published by the Apache Software Foundation\r
   7  * \r
   8  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
   9  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
  10  *  License for more details.\r
  11  * \r
  12  *  A copy of the license is in apache_license.txt. It is also available here:\r
  13  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
  14  * \r
  15  * Any republication or derived work distributed in source code form\r
  16  * must include this copyright and license notice.\r
  17  */\r
  18 \r
  19 package compbio.runner.msa;\r
  20 \r
  21 import static org.testng.Assert.assertEquals;\r
  22 import static org.testng.Assert.assertFalse;\r
  23 import static org.testng.Assert.assertNotNull;\r
  24 import static org.testng.Assert.assertTrue;\r
  25 import static org.testng.Assert.fail;\r
  26 \r
  27 import java.io.File;\r
  28 import java.io.FileInputStream;\r
  29 import java.io.FileNotFoundException;\r
  30 import java.io.IOException;\r
  31 import java.text.ParseException;\r
  32 import java.util.ArrayList;\r
  33 import java.util.Collections;\r
  34 import java.util.List;\r
  35 \r
  36 import javax.xml.bind.JAXBException;\r
  37 import javax.xml.bind.ValidationException;\r
  38 \r
  39 import org.ggf.drmaa.DrmaaException;\r
  40 import org.ggf.drmaa.JobInfo;\r
  41 import org.testng.annotations.Test;\r
  42 \r
  43 import compbio.data.sequence.FastaSequence;\r
  44 import compbio.engine.AsyncExecutor;\r
  45 import compbio.engine.Configurator;\r
  46 import compbio.engine.FilePuller;\r
  47 import compbio.engine.SyncExecutor;\r
  48 import compbio.engine.client.ConfExecutable;\r
  49 import compbio.engine.client.ConfiguredExecutable;\r
  50 import compbio.engine.client.Executable;\r
  51 import compbio.engine.client.Executable.ExecProvider;\r
  52 import compbio.engine.client.RunConfiguration;\r
  53 import compbio.engine.cluster.drmaa.ClusterUtil;\r
  54 import compbio.engine.cluster.drmaa.JobRunner;\r
  55 import compbio.engine.cluster.drmaa.StatisticManager;\r
  56 import compbio.engine.conf.RunnerConfigMarshaller;\r
  57 import compbio.engine.local.AsyncLocalRunner;\r
  58 import compbio.engine.local.LocalExecutorService;\r
  59 import compbio.engine.local.LocalRunner;\r
  60 import compbio.metadata.AllTestSuit;\r
  61 import compbio.metadata.ChunkHolder;\r
  62 import compbio.metadata.JobExecutionException;\r
  63 import compbio.metadata.JobStatus;\r
  64 import compbio.metadata.JobSubmissionException;\r
  65 import compbio.metadata.LimitsManager;\r
  66 import compbio.metadata.PresetManager;\r
  67 import compbio.metadata.ResultNotAvailableException;\r
  68 import compbio.metadata.RunnerConfig;\r
  69 import compbio.runner.OptionCombinator;\r
  70 import compbio.runner.Util;\r
  71 import compbio.util.FileWatcher;\r
  72 import compbio.util.SysPrefs;\r
  73 \r
  74 public class ClustalOTester {\r
  75 \r
  76         static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH\r
  77                         + "ClustalParameters.xml";\r
  78         public static String test_outfile = "TO1381.clustal.out";\r
  79         public static String cluster_test_outfile = "TO1381.clustal.cluster.out";\r
  80 \r
  81         @Test(groups = {AllTestSuit.test_group_cluster,\r
  82                         AllTestSuit.test_group_runner})\r
  83         public void testRunOnCluster() {\r
  84                 ClustalW clustal = new ClustalW();\r
  85                 assertFalse(SysPrefs.isWindows,\r
  86                                 "Cluster execution can only be in unix environment");\r
  87                 clustal.setInput(AllTestSuit.test_input)\r
  88                                 .setOutput(cluster_test_outfile);\r
  89 \r
  90                 try {\r
  91 \r
  92                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
  93                                         .configureExecutable(clustal);\r
  94                         JobRunner runner = JobRunner.getInstance(confClustal);\r
  95                         // ClusterSession csession = JobRunner.getSession();\r
  96                         assertNotNull(runner);\r
  97                         runner.executeJob();\r
  98                         // assertNotNull("JobId is null", jobId1);\r
  99                         JobStatus status = runner.getJobStatus();\r
 100                         assertTrue(status == JobStatus.PENDING\r
 101                                         || status == JobStatus.RUNNING);\r
 102                         JobInfo info = runner.getJobInfo();\r
 103                         assertNotNull(info);\r
 104                         StatisticManager sm = new StatisticManager(info);\r
 105                         assertNotNull(sm);\r
 106                         try {\r
 107                                 String exits = sm.getExitStatus();\r
 108                                 assertNotNull("Exit status is null", exits);\r
 109                                 // cut 4 trailing zeros from the number\r
 110                                 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)\r
 111                                                 .intValue();\r
 112                                 assertEquals(0, exitsInt);\r
 113                                 System.out.println(sm.getAllStats());\r
 114 \r
 115                         } catch (ParseException e) {\r
 116                                 e.printStackTrace();\r
 117                                 fail("Parse Exception: " + e.getMessage());\r
 118                         }\r
 119                         // At present the task directory could not be completely removed\r
 120                         // @see JobRunner.cleanup()\r
 121                         assertFalse(runner.cleanup(),\r
 122                                         "Could not remove some files whilst cleaning up ");\r
 123                         assertTrue(sm.hasExited());\r
 124                         assertFalse(sm.wasAborted());\r
 125                         assertFalse(sm.hasDump());\r
 126                         assertFalse(sm.hasSignaled());\r
 127 \r
 128                 } catch (JobSubmissionException e) {\r
 129                         e.printStackTrace();\r
 130                         fail("DrmaaException caught:" + e.getMessage());\r
 131                 } catch (JobExecutionException e) {\r
 132                         e.printStackTrace();\r
 133                         fail("DrmaaException caught:" + e.getMessage());\r
 134                 } catch (DrmaaException e) {\r
 135                         e.printStackTrace();\r
 136                         fail("DrmaaException caught:" + e.getMessage());\r
 137                 }\r
 138         }\r
 139 \r
 140         @Test(groups = {AllTestSuit.test_group_runner})\r
 141         public void testRunLocally() {\r
 142                 ClustalO clustal = new ClustalO();\r
 143                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
 144                 try {\r
 145 \r
 146                         // For local execution use relavive\r
 147                         ConfiguredExecutable<ClustalO> confClustal = Configurator\r
 148                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
 149                         LocalRunner lr = new LocalRunner(confClustal);\r
 150                         lr.executeJob();\r
 151                         confClustal = (ConfiguredExecutable<ClustalO>) lr.waitForResult();\r
 152                         assertNotNull(confClustal.getResults());\r
 153                 } catch (JobSubmissionException e) {\r
 154                         e.printStackTrace();\r
 155                         fail(e.getLocalizedMessage());\r
 156                 } catch (JobExecutionException e) {\r
 157                         e.printStackTrace();\r
 158                         fail(e.getLocalizedMessage());\r
 159                 } catch (ResultNotAvailableException e) {\r
 160                         e.printStackTrace();\r
 161                         fail(e.getLocalizedMessage());\r
 162                 }\r
 163         }\r
 164 \r
 165         @Test(groups = {AllTestSuit.test_group_runner})\r
 166         public void testConfigurationLoading() {\r
 167                 try {\r
 168                         RunnerConfig<ClustalW> clustalConfig = ConfExecutable\r
 169                                         .getRunnerOptions(ClustalW.class);\r
 170                         assertNotNull(clustalConfig);\r
 171                         assertTrue(clustalConfig.getArguments().size() > 0);\r
 172 \r
 173                         PresetManager<ClustalW> clustalPresets = ConfExecutable\r
 174                                         .getRunnerPresets(ClustalW.class);\r
 175                         assertNotNull(clustalPresets);\r
 176                         assertTrue(clustalPresets.getPresets().size() > 0);\r
 177                         clustalPresets.validate(clustalConfig);\r
 178 \r
 179                         LimitsManager<ClustalW> clustalLimits = ConfExecutable\r
 180                                         .getRunnerLimits(ClustalW.class);\r
 181                         assertNotNull(clustalLimits);\r
 182                         assertTrue(clustalLimits.getLimits().size() > 0);\r
 183                         clustalLimits.validate(clustalPresets);\r
 184 \r
 185                 } catch (FileNotFoundException e) {\r
 186                         e.printStackTrace();\r
 187                         fail(e.getLocalizedMessage());\r
 188                 } catch (IOException e) {\r
 189                         e.printStackTrace();\r
 190                         fail(e.getLocalizedMessage());\r
 191                 } catch (ValidationException e) {\r
 192                         e.printStackTrace();\r
 193                         fail(e.getLocalizedMessage());\r
 194                 }\r
 195         }\r
 196 \r
 197         @Test(groups = {AllTestSuit.test_group_runner})\r
 198         public void testOptionsLocally() {\r
 199                 try {\r
 200 \r
 201                         RunnerConfigMarshaller<ClustalW> clustalmarsh = new RunnerConfigMarshaller<ClustalW>(\r
 202                                         RunnerConfig.class);\r
 203 \r
 204                         RunnerConfig<ClustalW> clustalConfig = clustalmarsh.read(\r
 205                                         new FileInputStream(new File(clustalConfigFile)),\r
 206                                         RunnerConfig.class);\r
 207 \r
 208                         OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);\r
 209                         List<String> options = clustalOpc.getOptionsAtRandom();\r
 210                         for (int i = 0; i < options.size(); i++) {\r
 211                                 System.out.println("Using options: " + options);\r
 212                                 ClustalW clustal = new ClustalW();\r
 213                                 clustal.setInput(AllTestSuit.test_input)\r
 214                                                 .setOutput(test_outfile);\r
 215 \r
 216                                 // For local execution use relavive\r
 217                                 ConfiguredExecutable<ClustalW> confClustal = Configurator\r
 218                                                 .configureExecutable(clustal, ExecProvider.Local);\r
 219 \r
 220                                 // Add options to the executable\r
 221                                 confClustal.addParameters(options);\r
 222 \r
 223                                 LocalRunner lr = new LocalRunner(confClustal);\r
 224                                 lr.executeJob();\r
 225                                 confClustal = (ConfiguredExecutable<ClustalW>) lr\r
 226                                                 .waitForResult();\r
 227                                 assertNotNull(confClustal.getResults());\r
 228                                 Collections.shuffle(options);\r
 229                         }\r
 230 \r
 231                 } catch (JobSubmissionException e) {\r
 232                         e.printStackTrace();\r
 233                         fail(e.getLocalizedMessage());\r
 234                 } catch (JobExecutionException e) {\r
 235                         e.printStackTrace();\r
 236                         fail(e.getLocalizedMessage());\r
 237                 } catch (JAXBException e) {\r
 238                         e.printStackTrace();\r
 239                         fail(e.getLocalizedMessage());\r
 240                 } catch (ResultNotAvailableException e) {\r
 241                         e.printStackTrace();\r
 242                         fail(e.getLocalizedMessage());\r
 243                 } catch (FileNotFoundException e) {\r
 244                         e.printStackTrace();\r
 245                         fail(e.getLocalizedMessage());\r
 246                 }\r
 247         }\r
 248 \r
 249         public static final void main(String[] args) throws JobSubmissionException,\r
 250                         JobExecutionException, InterruptedException {\r
 251                 ClustalW clustal = new ClustalW();\r
 252                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
 253                 // For local execution use relavive\r
 254                 ConfiguredExecutable<ClustalW> confClustal = Configurator\r
 255                                 .configureExecutable(clustal);\r
 256                 AsyncExecutor lr = new AsyncLocalRunner();\r
 257                 lr.submitJob(confClustal);\r
 258                 Thread.sleep(3000);\r
 259                 LocalExecutorService.shutDown();\r
 260         }\r
 261 \r
 262         @Test(enabled = false)\r
 263         public void testAddParameters() {\r
 264                 ArrayList<FastaSequence> seqs = new ArrayList<FastaSequence>();\r
 265                 FastaSequence fs = new FastaSequence("tests1",\r
 266                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
 267                 FastaSequence fs2 = new FastaSequence("tests2",\r
 268                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
 269                 FastaSequence fs3 = new FastaSequence("tests3",\r
 270                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
 271                 seqs.add(fs);\r
 272                 seqs.add(fs2);\r
 273                 seqs.add(fs3);\r
 274                 ClustalW cl = new ClustalW();\r
 275                 cl.setInput("input.txt").setOutput("output.txt");\r
 276                 ConfiguredExecutable<ClustalW> confClustal;\r
 277                 try {\r
 278                         confClustal = Configurator.configureExecutable(cl);\r
 279                         Util.writeInput(seqs, confClustal);\r
 280 \r
 281                         LocalRunner lr = new LocalRunner(confClustal);\r
 282                         lr.executeJob();\r
 283                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
 284                         assertNotNull(confClustal.getResults());\r
 285 \r
 286                         assertTrue(confClustal.saveRunConfiguration());\r
 287                         ConfiguredExecutable<ClustalW> cexec = (ConfiguredExecutable<ClustalW>) confClustal\r
 288                                         .loadRunConfiguration(new FileInputStream(new File(\r
 289                                                         confClustal.getWorkDirectory(),\r
 290                                                         RunConfiguration.rconfigFile)));\r
 291                         assertNotNull(cexec);\r
 292 \r
 293                         lr = new LocalRunner(cexec);\r
 294                         lr.executeJob();\r
 295                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
 296                         assertNotNull(confClustal.getResults());\r
 297 \r
 298                         System.out.println("CE:" + cexec);\r
 299                 } catch (JobSubmissionException e) {\r
 300                         e.printStackTrace();\r
 301                         fail(e.getMessage());\r
 302                 } catch (FileNotFoundException e) {\r
 303                         e.printStackTrace();\r
 304                         fail(e.getMessage());\r
 305                 } catch (JobExecutionException e) {\r
 306                         e.printStackTrace();\r
 307                         fail(e.getMessage());\r
 308                 } catch (ResultNotAvailableException e) {\r
 309                         e.printStackTrace();\r
 310                         fail(e.getMessage());\r
 311                 } catch (IOException e) {\r
 312                         e.printStackTrace();\r
 313                         fail(e.getMessage());\r
 314                 }\r
 315         }\r
 316 \r
 317         @Test(groups = {AllTestSuit.test_group_runner})\r
 318         public void testPersistance() {\r
 319                 try {\r
 320                         ClustalW clustal = new ClustalW();\r
 321                         clustal.setError("errrr.txt").setInput(AllTestSuit.test_input)\r
 322                                         .setOutput("outtt.txt");\r
 323                         assertEquals(clustal.getInput(), AllTestSuit.test_input);\r
 324                         assertEquals(clustal.getError(), "errrr.txt");\r
 325                         assertEquals(clustal.getOutput(), "outtt.txt");\r
 326                         ConfiguredExecutable<ClustalW> cClustal = Configurator\r
 327                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
 328 \r
 329                         SyncExecutor sexec = Configurator.getSyncEngine(cClustal);\r
 330                         sexec.executeJob();\r
 331                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
 332                         assertNotNull(cClustal.getResults());\r
 333                         // Save run configuration\r
 334                         assertTrue(cClustal.saveRunConfiguration());\r
 335 \r
 336                         // See if loaded configuration is the same as saved\r
 337                         RunConfiguration loadedRun = RunConfiguration\r
 338                                         .load(new FileInputStream(new File(cClustal\r
 339                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 340                         assertTrue(((ConfExecutable<ClustalW>) cClustal)\r
 341                                         .getRunConfiguration().equals(loadedRun));\r
 342                         // Load run configuration as ConfExecutable\r
 343                         ConfiguredExecutable<ClustalW> resurrectedCclustal = (ConfiguredExecutable<ClustalW>) cClustal\r
 344                                         .loadRunConfiguration(new FileInputStream(new File(cClustal\r
 345                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 346                         assertNotNull(resurrectedCclustal);\r
 347                         // See in details whether executables are the same\r
 348                         assertEquals(resurrectedCclustal.getExecutable(), clustal);\r
 349 \r
 350                         // Finally rerun the job in the new task directory\r
 351                         ConfiguredExecutable<ClustalW> resclustal = Configurator\r
 352                                         .configureExecutable(resurrectedCclustal.getExecutable(),\r
 353                                                         Executable.ExecProvider.Local);\r
 354 \r
 355                         sexec = Configurator.getSyncEngine(resclustal,\r
 356                                         Executable.ExecProvider.Local);\r
 357                         sexec.executeJob();\r
 358                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
 359                         assertNotNull(cClustal.getResults());\r
 360 \r
 361                 } catch (JobSubmissionException e) {\r
 362                         e.printStackTrace();\r
 363                         fail(e.getMessage());\r
 364                 } catch (JobExecutionException e) {\r
 365                         e.printStackTrace();\r
 366                         fail(e.getMessage());\r
 367                 } catch (FileNotFoundException e) {\r
 368                         e.printStackTrace();\r
 369                         fail(e.getMessage());\r
 370                 } catch (IOException e) {\r
 371                         e.printStackTrace();\r
 372                         fail(e.getMessage());\r
 373                 } catch (ResultNotAvailableException e) {\r
 374                         e.printStackTrace();\r
 375                         fail(e.getMessage());\r
 376                 }\r
 377         }\r
 378 \r
 379         @Test(groups = {AllTestSuit.test_group_runner})\r
 380         public void readStatistics() {\r
 381                 try {\r
 382                         ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)\r
 383                                         .setOutput(test_outfile);\r
 384                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
 385                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
 386 \r
 387                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
 388                         String jobId = sexec.submitJob(confClustal);\r
 389                         FilePuller fw = FilePuller.newFilePuller(\r
 390                                         confClustal.getWorkDirectory() + File.separator\r
 391                                                         + ClustalW.getStatFile(),\r
 392                                         FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 393                         int count = 0;\r
 394                         long position = 0;\r
 395                         fw.waitForFile(4);\r
 396                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED\r
 397                                         || sexec.getJobStatus(jobId) == JobStatus.FAILED || sexec\r
 398                                         .getJobStatus(jobId) == JobStatus.UNDEFINED)\r
 399                                         || fw.hasMoreData()) {\r
 400                                 ChunkHolder ch = fw.pull(position);\r
 401                                 String chunk = ch.getChunk();\r
 402                                 position = ch.getNextPosition();\r
 403                                 System.out.print(chunk);\r
 404                                 count++;\r
 405                         }\r
 406                         assertTrue(count > 1);\r
 407                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
 408                         assertNotNull(al.getResults());\r
 409                 } catch (JobSubmissionException e) {\r
 410                         e.printStackTrace();\r
 411                         fail(e.getMessage());\r
 412                 } catch (ResultNotAvailableException e) {\r
 413                         e.printStackTrace();\r
 414                         fail(e.getMessage());\r
 415                 } catch (IOException e) {\r
 416                         e.printStackTrace();\r
 417                         fail(e.getMessage());\r
 418                 }\r
 419         }\r
 420 \r
 421         @Test(groups = {AllTestSuit.test_group_cluster,\r
 422                         AllTestSuit.test_group_runner})\r
 423         public void readStatisticsClusterExecution() {\r
 424                 try {\r
 425                         ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)\r
 426                                         .setOutput(test_outfile);\r
 427                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
 428                                         .configureExecutable(clustal,\r
 429                                                         Executable.ExecProvider.Cluster);\r
 430 \r
 431                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
 432                         String jobId = sexec.submitJob(confClustal);\r
 433                         FilePuller fw = FilePuller.newFilePuller(\r
 434                                         confClustal.getWorkDirectory() + File.separator\r
 435                                                         + ClustalW.getStatFile(),\r
 436                                         FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 437                         int count = 0;\r
 438                         long position = 0;\r
 439                         fw.waitForFile(200);\r
 440                         /*\r
 441                          * Under certain circumstances DRMAA could report the status wrongly\r
 442                          * thus this loop never ends TODO deal with this!\r
 443                          */\r
 444                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec\r
 445                                         .getJobStatus(jobId) == JobStatus.FAILED)\r
 446                                         || fw.hasMoreData()) {\r
 447                                 ChunkHolder ch = fw.pull(position);\r
 448                                 String chunk = ch.getChunk();\r
 449                                 position = ch.getNextPosition();\r
 450                                 System.out.print(chunk);\r
 451                                 count++;\r
 452                                 if (sexec.getJobStatus(jobId) == JobStatus.UNDEFINED) {\r
 453                                         System.out.println("DRMAA reported wrong status for job + "\r
 454                                                         + jobId + " continue anyway!");\r
 455                                         break;\r
 456                                 }\r
 457                         }\r
 458                         assertTrue(count > 1);\r
 459                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
 460                         assertNotNull(al.getResults());\r
 461                 } catch (JobSubmissionException e) {\r
 462                         e.printStackTrace();\r
 463                         fail(e.getMessage());\r
 464                 } catch (ResultNotAvailableException e) {\r
 465                         e.printStackTrace();\r
 466                         fail(e.getMessage());\r
 467                 } catch (IOException e) {\r
 468                         e.printStackTrace();\r
 469                         fail(e.getMessage());\r
 470                 }\r
 471         }\r
 472 }\r