testsrc/compbio/casscode/msa/ClustalOTester.java

   1 /* Copyright (c) 2009 Peter Troshin\r
   2  * Copyright (c) 2013 Alexander Sherstnev\r
   3  *  \r
   4  *  JAva Bioinformatics Analysis Web Services (JABAWS)\r
   5  *  @version: 2.5     \r
   6  * \r
   7  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
   8  *  Apache License version 2 as published by the Apache Software Foundation\r
   9  * \r
  10  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
  11  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
  12  *  License for more details.\r
  13  * \r
  14  *  A copy of the license is in apache_license.txt. It is also available here:\r
  15  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
  16  * \r
  17  * Any republication or derived work distributed in source code form\r
  18  * must include this copyright and license notice.\r
  19  */\r
  20 \r
  21 package compbio.runner.msa;\r
  22 \r
  23 import static org.testng.Assert.assertEquals;\r
  24 import static org.testng.Assert.assertFalse;\r
  25 import static org.testng.Assert.assertNotNull;\r
  26 import static org.testng.Assert.assertTrue;\r
  27 import static org.testng.Assert.fail;\r
  28 \r
  29 import java.io.File;\r
  30 import java.io.FileInputStream;\r
  31 import java.io.FileNotFoundException;\r
  32 import java.io.IOException;\r
  33 import java.text.ParseException;\r
  34 import java.util.Collections;\r
  35 import java.util.List;\r
  36 \r
  37 import javax.xml.bind.JAXBException;\r
  38 \r
  39 import org.ggf.drmaa.DrmaaException;\r
  40 import org.ggf.drmaa.JobInfo;\r
  41 import org.testng.annotations.Test;\r
  42 \r
  43 import compbio.engine.AsyncExecutor;\r
  44 import compbio.engine.Configurator;\r
  45 import compbio.engine.FilePuller;\r
  46 import compbio.engine.SyncExecutor;\r
  47 import compbio.engine.client.EngineUtil;\r
  48 import compbio.engine.client.ConfExecutable;\r
  49 import compbio.engine.client.ConfiguredExecutable;\r
  50 import compbio.engine.client.Executable;\r
  51 import compbio.engine.client.Executable.ExecProvider;\r
  52 import compbio.engine.client.RunConfiguration;\r
  53 import compbio.engine.cluster.drmaa.ClusterEngineUtil;\r
  54 import compbio.engine.cluster.drmaa.ClusterRunner;\r
  55 import compbio.engine.cluster.drmaa.StatisticManager;\r
  56 import compbio.engine.conf.RunnerConfigMarshaller;\r
  57 import compbio.engine.local.AsyncLocalRunner;\r
  58 import compbio.engine.local.LocalExecutorService;\r
  59 import compbio.engine.local.LocalRunner;\r
  60 import compbio.metadata.AllTestSuit;\r
  61 import compbio.metadata.ChunkHolder;\r
  62 import compbio.metadata.JobExecutionException;\r
  63 import compbio.metadata.JobStatus;\r
  64 import compbio.metadata.JobSubmissionException;\r
  65 import compbio.metadata.LimitsManager;\r
  66 import compbio.metadata.ResultNotAvailableException;\r
  67 import compbio.metadata.RunnerConfig;\r
  68 import compbio.runner.OptionCombinator;\r
  69 import compbio.util.FileWatcher;\r
  70 import compbio.util.SysPrefs;\r
  71 \r
  72 public class ClustalOTester {\r
  73 \r
  74         static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH + "ClustaloParameters.xml";\r
  75         public static String test_outfile = "TO1381.clustalo.out";\r
  76         public static String cluster_test_outfile = "TO1381.clustalo.cluster.out";\r
  77 \r
  78         @Test(groups = {AllTestSuit.test_group_runner})\r
  79         public void RunLocally() {\r
  80                 ClustalO clustal = new ClustalO();\r
  81                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
  82                 try {\r
  83                         // For local execution use relavive\r
  84                         ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);\r
  85                         LocalRunner lr = new LocalRunner(confClustal);\r
  86                         lr.executeJob();\r
  87                         confClustal = (ConfiguredExecutable<ClustalO>) lr.waitForResult();\r
  88                         assertNotNull(confClustal.getResults());\r
  89                 } catch (JobSubmissionException e) {\r
  90                         e.printStackTrace();\r
  91                         fail(e.getLocalizedMessage());\r
  92                 } catch (JobExecutionException e) {\r
  93                         e.printStackTrace();\r
  94                         fail(e.getLocalizedMessage());\r
  95                 } catch (ResultNotAvailableException e) {\r
  96                         e.printStackTrace();\r
  97                         fail(e.getLocalizedMessage());\r
  98                 }\r
  99         }\r
 100 \r
 101         @Test(groups = {AllTestSuit.test_group_runner})\r
 102         public void ConfigurationLoading() {\r
 103                 try {\r
 104                         RunnerConfig<ClustalO> clustalConfig = ConfExecutable.getRunnerOptions(ClustalO.class);\r
 105                         assertNotNull(clustalConfig);\r
 106                         assertTrue(clustalConfig.getArguments().size() > 0);\r
 107 \r
 108                         LimitsManager<ClustalO> clustalLimits = ConfExecutable.getRunnerLimits(ClustalO.class);\r
 109                         assertNotNull(clustalLimits);\r
 110                         assertTrue(clustalLimits.getLimits().size() > 0);\r
 111                 } catch (FileNotFoundException e) {\r
 112                         e.printStackTrace();\r
 113                         fail(e.getLocalizedMessage());\r
 114                 } catch (IOException e) {\r
 115                         e.printStackTrace();\r
 116                         fail(e.getLocalizedMessage());\r
 117                 }\r
 118         }\r
 119 \r
 120         @Test(groups = {AllTestSuit.test_group_runner})\r
 121         public void OptionsLocally() {\r
 122                 try {\r
 123                         RunnerConfigMarshaller<ClustalO> clustalmarsh = new RunnerConfigMarshaller<ClustalO>(RunnerConfig.class);\r
 124                         RunnerConfig<ClustalO> clustalConfig = clustalmarsh.read(new FileInputStream(new File(clustalConfigFile)), RunnerConfig.class);\r
 125 \r
 126                         OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);\r
 127                         List<String> options = clustalOpc.getOptionsAtRandom();\r
 128                         for (int i = 0; i < options.size(); i++) {\r
 129                                 System.out.println("Using options: " + options);\r
 130                                 ClustalO clustal = new ClustalO();\r
 131                                 clustal.setInput(AllTestSuit.test_input);\r
 132                                 clustal.setOutput(test_outfile);\r
 133 \r
 134                                 // For local execution use relavive\r
 135                                 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, ExecProvider.Local);\r
 136 \r
 137                                 // Add options to the executable\r
 138                                 confClustal.addParameters(options);\r
 139 \r
 140                                 LocalRunner lr = new LocalRunner(confClustal);\r
 141                                 lr.executeJob();\r
 142                                 confClustal = (ConfiguredExecutable<ClustalO>) lr.waitForResult();\r
 143                                 assertNotNull(confClustal.getResults());\r
 144                                 Collections.shuffle(options);\r
 145                         }\r
 146 \r
 147                 } catch (JobSubmissionException e) {\r
 148                         e.printStackTrace();\r
 149                         fail(e.getLocalizedMessage());\r
 150                 } catch (JobExecutionException e) {\r
 151                         e.printStackTrace();\r
 152                         fail(e.getLocalizedMessage());\r
 153                 } catch (JAXBException e) {\r
 154                         e.printStackTrace();\r
 155                         fail(e.getLocalizedMessage());\r
 156                 } catch (ResultNotAvailableException e) {\r
 157                         e.printStackTrace();\r
 158                         fail(e.getLocalizedMessage());\r
 159                 } catch (FileNotFoundException e) {\r
 160                         e.printStackTrace();\r
 161                         fail(e.getLocalizedMessage());\r
 162                 }\r
 163         }\r
 164 \r
 165         public static final void main(String[] args)\r
 166                         throws JobSubmissionException, JobExecutionException, InterruptedException {\r
 167                 ClustalO clustal = new ClustalO();\r
 168                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
 169                 // For local execution use relavive\r
 170                 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal);\r
 171                 AsyncExecutor lr = new AsyncLocalRunner();\r
 172                 lr.submitJob(confClustal);\r
 173                 Thread.sleep(3000);\r
 174                 LocalExecutorService.shutDown();\r
 175         }\r
 176 \r
 177         @Test(groups = {AllTestSuit.test_group_runner})\r
 178         public void Persistance() {\r
 179                 try {\r
 180                         ClustalO clustal = new ClustalO();\r
 181                         clustal.setError("errrr.txt");\r
 182                         clustal.setInput(AllTestSuit.test_input);\r
 183                         clustal.setOutput("outtt.txt");\r
 184                         assertEquals(clustal.getInput(), AllTestSuit.test_input);\r
 185                         assertEquals(clustal.getError(), "errrr.txt");\r
 186                         assertEquals(clustal.getOutput(), "outtt.txt");\r
 187                         ConfiguredExecutable<ClustalO> cClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);\r
 188 \r
 189                         SyncExecutor sexec = Configurator.getSyncEngine(cClustal);\r
 190                         sexec.executeJob();\r
 191                         cClustal = (ConfiguredExecutable<ClustalO>) sexec.waitForResult();\r
 192                         assertNotNull(cClustal.getResults());\r
 193                         // Save run configuration\r
 194                         assertTrue(cClustal.saveRunConfiguration());\r
 195 \r
 196                         // See if loaded configuration is the same as saved\r
 197                         RunConfiguration loadedRun = RunConfiguration.load(new FileInputStream(new File(cClustal.getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 198                         assertTrue(((ConfExecutable<ClustalO>) cClustal)\r
 199                                         .getRunConfiguration().equals(loadedRun));\r
 200                         // Load run configuration as ConfExecutable\r
 201                         ConfiguredExecutable<ClustalO> resurrectedCclustal = (ConfiguredExecutable<ClustalO>) cClustal\r
 202                                         .loadRunConfiguration(new FileInputStream(new File(cClustal\r
 203                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 204                         assertNotNull(resurrectedCclustal);\r
 205                         // See in details whether executables are the same\r
 206                         assertEquals(resurrectedCclustal.getExecutable(), clustal);\r
 207 \r
 208                         // Finally rerun the job in the new task directory\r
 209                         ConfiguredExecutable<ClustalO> resclustal = Configurator\r
 210                                         .configureExecutable(resurrectedCclustal.getExecutable(),\r
 211                                                         Executable.ExecProvider.Local);\r
 212 \r
 213                         sexec = Configurator.getSyncEngine(resclustal, Executable.ExecProvider.Local);\r
 214                         sexec.executeJob();\r
 215                         cClustal = (ConfiguredExecutable<ClustalO>) sexec.waitForResult();\r
 216                         assertNotNull(cClustal.getResults());\r
 217 \r
 218                 } catch (JobSubmissionException e) {\r
 219                         e.printStackTrace();\r
 220                         fail(e.getMessage());\r
 221                 } catch (JobExecutionException e) {\r
 222                         e.printStackTrace();\r
 223                         fail(e.getMessage());\r
 224                 } catch (FileNotFoundException e) {\r
 225                         e.printStackTrace();\r
 226                         fail(e.getMessage());\r
 227                 } catch (IOException e) {\r
 228                         e.printStackTrace();\r
 229                         fail(e.getMessage());\r
 230                 } catch (ResultNotAvailableException e) {\r
 231                         e.printStackTrace();\r
 232                         fail(e.getMessage());\r
 233                 }\r
 234         }\r
 235 \r
 236         @Test(groups = {AllTestSuit.test_group_runner})\r
 237         public void readStatistics()\r
 238                         throws InterruptedException {\r
 239                 try {\r
 240                         ClustalO al = new ClustalO();\r
 241                         al.setInput(AllTestSuit.test_input);\r
 242                         al.setOutput(test_outfile);\r
 243                         ConfiguredExecutable<ClustalO> confal = Configurator.configureExecutable(al, Executable.ExecProvider.Local);\r
 244 \r
 245                         AsyncExecutor sexec = Configurator.getAsyncEngine(confal);\r
 246                         String jobId = sexec.submitJob(confal);\r
 247                         String file = EngineUtil.getFullPath(confal.getWorkDirectory(), ClustalW.getStatFile());\r
 248                         FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 249                         int count = 0;\r
 250                         long position = 0;\r
 251                         fw.waitForFile(4);\r
 252                         JobStatus status = sexec.getJobStatus(jobId);\r
 253                         while (status != JobStatus.FINISHED || fw.hasMoreData()) {\r
 254                                 if (fw.hasMoreData()) {\r
 255                                         ChunkHolder ch = fw.pull(position);\r
 256                                         String chunk = ch.getChunk();\r
 257                                         position = ch.getNextPosition();\r
 258                                 }\r
 259                                 count++;\r
 260                                 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {\r
 261                                         fail("job " + jobId +" failed!");\r
 262                                         break;\r
 263                                 }\r
 264                                 Thread.sleep(200);\r
 265                                 status = sexec.getJobStatus(jobId);\r
 266                                 System.out.println("CLustalO: Job status = " + status + ", file status = " + fw.hasMoreData());\r
 267                         }\r
 268                         assertTrue(count > 1);\r
 269                         ConfiguredExecutable<?> al2 = sexec.getResults(jobId);\r
 270                         assertNotNull(al2.getResults());\r
 271                 } catch (JobSubmissionException e) {\r
 272                         e.printStackTrace();\r
 273                         fail(e.getMessage());\r
 274                 } catch (ResultNotAvailableException e) {\r
 275                         e.printStackTrace();\r
 276                         fail(e.getMessage());\r
 277                 } catch (IOException e) {\r
 278                         e.printStackTrace();\r
 279                         fail(e.getMessage());\r
 280                 }\r
 281         }\r
 282 \r
 283         @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})\r
 284         public void RunOnCluster() {\r
 285                 ClustalO clustal = new ClustalO();\r
 286                 assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment");\r
 287                 clustal.setInput(AllTestSuit.test_input).setOutput(cluster_test_outfile);\r
 288                 try {\r
 289                         ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal);\r
 290                         ClusterRunner runner = ClusterRunner.getInstance(confClustal);\r
 291                         // ClusterSession csession = JobRunner.getSession();\r
 292                         assertNotNull(runner);\r
 293                         runner.executeJob();\r
 294                         // assertNotNull("JobId is null", jobId1);\r
 295                         JobStatus status = runner.getJobStatus();\r
 296                         assertTrue(status == JobStatus.PENDING || status == JobStatus.RUNNING);\r
 297                         JobInfo info = runner.getJobInfo();\r
 298                         assertNotNull(info);\r
 299                         StatisticManager sm = new StatisticManager(info);\r
 300                         assertNotNull(sm);\r
 301                         try {\r
 302                                 String exits = sm.getExitStatus();\r
 303                                 assertNotNull("Exit status is null", exits);\r
 304                                 // cut 4 trailing zeros from the number\r
 305                                 int exitsInt = ClusterEngineUtil.CLUSTER_STAT_IN_SEC.parse(exits).intValue();\r
 306                                 assertEquals(0, exitsInt);\r
 307                                 System.out.println(sm.getAllStats());\r
 308                         } catch (ParseException e) {\r
 309                                 e.printStackTrace();\r
 310                                 fail("Parse Exception: " + e.getMessage());\r
 311                         }\r
 312                         // At present the task directory could not be completely removed\r
 313                         // @see JobRunner.cleanup()\r
 314                         assertFalse(runner.cleanup(), "Could not remove some files whilst cleaning up ");\r
 315                         assertTrue(sm.hasExited());\r
 316                         assertFalse(sm.wasAborted());\r
 317                         assertFalse(sm.hasDump());\r
 318                         assertFalse(sm.hasSignaled());\r
 319                 } catch (JobSubmissionException e) {\r
 320                         e.printStackTrace();\r
 321                         fail("DrmaaException caught:" + e.getMessage());\r
 322                 } catch (JobExecutionException e) {\r
 323                         e.printStackTrace();\r
 324                         fail("DrmaaException caught:" + e.getMessage());\r
 325                 } catch (DrmaaException e) {\r
 326                         e.printStackTrace();\r
 327                         fail("DrmaaException caught:" + e.getMessage());\r
 328                 }\r
 329         }\r
 330 \r
 331         @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})\r
 332         public void readStatisticsClusterExecution() {\r
 333                 try {\r
 334                         ClustalO clustal = new ClustalO().setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
 335                         ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Cluster);\r
 336 \r
 337                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
 338                         String jobId = sexec.submitJob(confClustal);\r
 339                         String file = EngineUtil.getFullPath(confClustal.getWorkDirectory(), ClustalW.getStatFile());\r
 340                         FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 341                         int count = 0;\r
 342                         long position = 0;\r
 343                         int maxloopcount = 108000; // max waiting time = 6h*60m*60s/0.2(one loop sleep)\r
 344                         fw.waitForFile(200);\r
 345                         /*\r
 346                          * Under certain circumstances DRMAA could report the status wrongly\r
 347                          * thus this loop never ends. maxloopcount ensures hard stop of the loop...\r
 348                          */\r
 349                         while (!( sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec.getJobStatus(jobId) == JobStatus.FAILED) \r
 350                                         || count < maxloopcount || fw.hasMoreData()) {\r
 351                                 ChunkHolder ch = fw.pull(position);\r
 352                                 String chunk = ch.getChunk();\r
 353                                 position = ch.getNextPosition();\r
 354                                 System.out.print(chunk);\r
 355                                 count++;\r
 356                                 Thread.sleep(200);\r
 357                                 if (sexec.getJobStatus(jobId) == JobStatus.UNDEFINED) {\r
 358                                         System.out.println("DRMAA reported wrong status for job + " + jobId + " continue anyway!");\r
 359                                         break;\r
 360                                 }\r
 361                         }\r
 362                         assertTrue(count > 1);\r
 363                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
 364                         assertNotNull(al.getResults());\r
 365                 } catch (JobSubmissionException e) {\r
 366                         e.printStackTrace();\r
 367                         fail(e.getMessage());\r
 368                 } catch (ResultNotAvailableException e) {\r
 369                         e.printStackTrace();\r
 370                         fail(e.getMessage());\r
 371                 } catch (IOException e) {\r
 372                         e.printStackTrace();\r
 373                         fail(e.getMessage());\r
 374                 } catch (InterruptedException e) {\r
 375                         e.printStackTrace();\r
 376                         fail(e.getMessage());\r
 377                 }\r
 378         }\r
 379 }\r