testsrc/compbio/runner/disorder/JronnTester.java

   1 /* Copyright (c) 2009 Peter Troshin\r
   2  * Copyright (c) 2013 Alexander Sherstnev\r
   3  *  \r
   4  *  Java Bioinformatics Analysis Web Services (JABAWS) \r
   5  *  @version: 2.5     \r
   6  * \r
   7  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
   8  *  Apache License version 2 as published by the Apache Software Foundation\r
   9  * \r
  10  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
  11  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
  12  *  License for more details.\r
  13  * \r
  14  *  A copy of the license is in apache_license.txt. It is also available here:\r
  15  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
  16  * \r
  17  * Any republication or derived work distributed in source code form\r
  18  * must include this copyright and license notice.\r
  19  */\r
  20 \r
  21 package compbio.runner.disorder;\r
  22 \r
  23 import static org.testng.Assert.assertEquals;\r
  24 import static org.testng.Assert.assertFalse;\r
  25 import static org.testng.Assert.assertNotNull;\r
  26 import static org.testng.Assert.assertNull;\r
  27 import static org.testng.Assert.assertTrue;\r
  28 import static org.testng.Assert.fail;\r
  29 \r
  30 import java.io.File;\r
  31 import java.io.FileInputStream;\r
  32 import java.io.FileNotFoundException;\r
  33 import java.io.IOException;\r
  34 import java.text.ParseException;\r
  35 \r
  36 import javax.xml.bind.ValidationException;\r
  37 \r
  38 import org.ggf.drmaa.DrmaaException;\r
  39 import org.ggf.drmaa.JobInfo;\r
  40 import org.testng.annotations.BeforeMethod;\r
  41 import org.testng.annotations.Test;\r
  42 \r
  43 import compbio.data.sequence.ScoreManager;\r
  44 import compbio.engine.AsyncExecutor;\r
  45 import compbio.engine.Configurator;\r
  46 import compbio.engine.FilePuller;\r
  47 import compbio.engine.SyncExecutor;\r
  48 import compbio.engine.client.ConfExecutable;\r
  49 import compbio.engine.client.ConfiguredExecutable;\r
  50 import compbio.engine.client.Executable;\r
  51 import compbio.engine.client.RunConfiguration;\r
  52 import compbio.engine.cluster.drmaa.ClusterEngineUtil;\r
  53 import compbio.engine.cluster.drmaa.ClusterRunner;\r
  54 import compbio.engine.cluster.drmaa.StatisticManager;\r
  55 import compbio.engine.local.LocalRunner;\r
  56 import compbio.metadata.AllTestSuit;\r
  57 import compbio.metadata.ChunkHolder;\r
  58 import compbio.metadata.JobExecutionException;\r
  59 import compbio.metadata.JobStatus;\r
  60 import compbio.metadata.JobSubmissionException;\r
  61 import compbio.metadata.LimitsManager;\r
  62 import compbio.metadata.PresetManager;\r
  63 import compbio.metadata.ResultNotAvailableException;\r
  64 import compbio.metadata.RunnerConfig;\r
  65 import compbio.util.FileWatcher;\r
  66 import compbio.util.SysPrefs;\r
  67 \r
  68 public class JronnTester {\r
  69 \r
  70         public static String test_outfile = "TO1381.jronn.out";\r
  71         private Jronn jronn;\r
  72 \r
  73         @BeforeMethod(alwaysRun = true)\r
  74         void init() {\r
  75                 jronn = new Jronn();\r
  76                 jronn.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
  77         }\r
  78 \r
  79         @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})\r
  80         public void RunOnCluster() {\r
  81                 assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment");\r
  82                 try {\r
  83                         ConfiguredExecutable<Jronn> confJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Cluster);\r
  84                         ClusterRunner runner = ClusterRunner.getInstance(confJronn);\r
  85 \r
  86                         assertNotNull(runner, "Runner is NULL");\r
  87                         runner.executeJob();\r
  88                         JobStatus status = runner.getJobStatus();\r
  89                         assertTrue(status == JobStatus.PENDING || status == JobStatus.RUNNING, "Status of the process is wrong!");\r
  90                         JobInfo info = runner.getJobInfo();\r
  91                         assertNotNull(info, "JobInfo is null");\r
  92                         StatisticManager sm = new StatisticManager(info);\r
  93                         assertNotNull(sm, "Statictic manager is null");\r
  94                         try {\r
  95                                 String exits = sm.getExitStatus();\r
  96                                 assertNotNull("Exit status is null", exits);\r
  97                                 // cut 4 trailing zeros from the number\r
  98                                 int exitsInt = ClusterEngineUtil.CLUSTER_STAT_IN_SEC.parse(exits).intValue();\r
  99                                 assertEquals(0, exitsInt);\r
 100                                 //System.out.println(sm.getAllStats());\r
 101                         } catch (ParseException e) {\r
 102                                 e.printStackTrace();\r
 103                                 fail("Parse Exception: " + e.getMessage());\r
 104                         }\r
 105                         assertTrue(sm.hasExited());\r
 106                         assertFalse(sm.wasAborted());\r
 107                         assertFalse(sm.hasDump());\r
 108                         assertFalse(sm.hasSignaled());\r
 109                 } catch (JobSubmissionException e) {\r
 110                         e.printStackTrace();\r
 111                         fail("DrmaaException caught:" + e.getMessage());\r
 112                 } catch (JobExecutionException e) {\r
 113                         e.printStackTrace();\r
 114                         fail("DrmaaException caught:" + e.getMessage());\r
 115                 } catch (DrmaaException e) {\r
 116                         e.printStackTrace();\r
 117                         fail("DrmaaException caught:" + e.getMessage());\r
 118                 }\r
 119         }\r
 120 \r
 121         /**\r
 122          * This tests fails from time to time depending on the cluster load or some\r
 123          * other factors. Any client code has to adjust for this issue\r
 124          */\r
 125         @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})\r
 126         public void RunOnClusterAsync() {\r
 127                 assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment");\r
 128                 try {\r
 129                         ConfiguredExecutable<Jronn> confJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Cluster);\r
 130                         AsyncExecutor aengine = Configurator.getAsyncEngine(confJronn);\r
 131                         String jobId = aengine.submitJob(confJronn);\r
 132                         assertNotNull(jobId, "Runner is NULL");\r
 133                         // let drmaa to start\r
 134                         Thread.sleep(500);\r
 135                         JobStatus status = aengine.getJobStatus(jobId);\r
 136                         while (status != JobStatus.FINISHED) {\r
 137                                 System.out.println("Job Status: " + status);\r
 138                                 Thread.sleep(1000);\r
 139                                 status = aengine.getJobStatus(jobId);\r
 140                                 ConfiguredExecutable<Jronn> result = (ConfiguredExecutable<Jronn>) aengine.getResults(jobId);\r
 141                                 assertNotNull(result);\r
 142                                 System.out.println("RES:" + result);\r
 143                                 // Some times the job could be removed from the cluster accounting before \r
 144                                 // it has been reported to finish. Make sure to stop waiting in such case\r
 145                                 if (status == JobStatus.UNDEFINED) {\r
 146                                         break;\r
 147                                 }\r
 148                         }\r
 149                 } catch (JobSubmissionException e) {\r
 150                         e.printStackTrace();\r
 151                         fail("DrmaaException caught:" + e.getMessage());\r
 152                 } catch (InterruptedException e) {\r
 153                         e.printStackTrace();\r
 154                         fail(e.getMessage());\r
 155                 } catch (ResultNotAvailableException e) {\r
 156                         e.printStackTrace();\r
 157                         fail(e.getMessage());\r
 158                 }\r
 159         }\r
 160 \r
 161         @Test(groups = {AllTestSuit.test_group_runner})\r
 162         public void RunLocally() {\r
 163                 try {\r
 164                         ConfiguredExecutable<Jronn> confJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Local);\r
 165 \r
 166                         // For local execution use relative\r
 167                         LocalRunner lr = new LocalRunner(confJronn);\r
 168                         lr.executeJob();\r
 169                         ConfiguredExecutable<?> al1 = lr.waitForResult();\r
 170                         assertNotNull(al1.getResults());\r
 171                         ScoreManager al2 = confJronn.getResults();\r
 172                         assertNotNull(al2);\r
 173                         assertEquals(al2.asMap().size(), 3);\r
 174                         assertEquals(al1.getResults(), al2);\r
 175                 } catch (JobSubmissionException e) {\r
 176                         e.printStackTrace();\r
 177                         fail(e.getLocalizedMessage());\r
 178                 } catch (ResultNotAvailableException e) {\r
 179                         e.printStackTrace();\r
 180                         fail(e.getLocalizedMessage());\r
 181                 } catch (JobExecutionException e) {\r
 182                         e.printStackTrace();\r
 183                         fail(e.getLocalizedMessage());\r
 184                 }\r
 185         }\r
 186 \r
 187         @Test(groups = {AllTestSuit.test_group_runner})\r
 188         public void RunLocallyOnTwoCpu() {\r
 189                 try {\r
 190                         jronn.setNCore(2);\r
 191                         ConfiguredExecutable<Jronn> confJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Local);\r
 192 \r
 193                         // For local execution use relative\r
 194                         LocalRunner lr = new LocalRunner(confJronn);\r
 195                         lr.executeJob();\r
 196                         ConfiguredExecutable<?> al1 = lr.waitForResult();\r
 197                         assertNotNull(al1.getResults());\r
 198                         ScoreManager al2 = confJronn.getResults();\r
 199                         assertNotNull(al2);\r
 200                         assertEquals(al2.asMap().size(), 3);\r
 201                         assertEquals(al1.getResults(), al2);\r
 202                 } catch (JobSubmissionException e) {\r
 203                         e.printStackTrace();\r
 204                         fail(e.getLocalizedMessage());\r
 205                 } catch (ResultNotAvailableException e) {\r
 206                         e.printStackTrace();\r
 207                         fail(e.getLocalizedMessage());\r
 208                 } catch (JobExecutionException e) {\r
 209                         e.printStackTrace();\r
 210                         fail(e.getLocalizedMessage());\r
 211                 }\r
 212         }\r
 213 \r
 214         @Test(groups = {AllTestSuit.test_group_runner})\r
 215         public void readStatistics() {\r
 216                 try {\r
 217                         ConfiguredExecutable<Jronn> confJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Local);\r
 218                         // For local execution use relavive\r
 219 \r
 220                         AsyncExecutor sexec = Configurator.getAsyncEngine(confJronn);\r
 221                         String jobId = sexec.submitJob(confJronn);\r
 222                         String file = confJronn.getWorkDirectory() + File.separator + Jronn.getStatFile();\r
 223                         FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 224                         int count = 0;\r
 225                         long position = 0;\r
 226                         fw.waitForFile(4);\r
 227                         JobStatus status = sexec.getJobStatus(jobId);\r
 228                         while (status != JobStatus.FINISHED) {\r
 229                                 if (fw.hasMoreData()) {\r
 230                                         ChunkHolder ch = fw.pull(position);\r
 231                                         String chunk = ch.getChunk();\r
 232                                         position = ch.getNextPosition();\r
 233                                 }\r
 234                                 count++;\r
 235                                 // Make sure the loop is terminated if the job fails\r
 236                                 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {\r
 237                                         break;\r
 238                                 }\r
 239                                 Thread.sleep(300);\r
 240                                 status = sexec.getJobStatus(jobId);\r
 241                         }\r
 242                         assertTrue(count > 1);\r
 243                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
 244                         assertNotNull(al.getResults());\r
 245                 } catch (JobSubmissionException e) {\r
 246                         e.printStackTrace();\r
 247                         fail(e.getMessage());\r
 248                 } catch (ResultNotAvailableException e) {\r
 249                         e.printStackTrace();\r
 250                         fail(e.getMessage());\r
 251                 } catch (IOException e) {\r
 252                         e.printStackTrace();\r
 253                         fail(e.getMessage());\r
 254                 } catch (InterruptedException e) {\r
 255                         e.printStackTrace();\r
 256                         fail(e.getMessage());\r
 257                 }\r
 258         }\r
 259 \r
 260         @Test(groups = {AllTestSuit.test_group_runner})\r
 261         public void Persistance() {\r
 262                 try {\r
 263                         Jronn jronn = new Jronn();\r
 264                         jronn.setError("errrr.txt");\r
 265                         jronn.setInput(AllTestSuit.test_input);\r
 266                         jronn.setOutput("outtt.txt");\r
 267                         assertEquals(jronn.getInput(), AllTestSuit.test_input);\r
 268                         assertEquals(jronn.getError(), "errrr.txt");\r
 269                         assertEquals(jronn.getOutput(), "outtt.txt");\r
 270                         ConfiguredExecutable<Jronn> cJronn = Configurator.configureExecutable(jronn, Executable.ExecProvider.Local);\r
 271                         SyncExecutor sexec = Configurator.getSyncEngine(cJronn);\r
 272                         sexec.executeJob();\r
 273                         ConfiguredExecutable<?> al = sexec.waitForResult();\r
 274                         assertNotNull(al.getResults());\r
 275                         // Save run configuration\r
 276                         assertTrue(cJronn.saveRunConfiguration());\r
 277 \r
 278                         // See if loaded configuration is the same as saved\r
 279                         RunConfiguration loadedRun = RunConfiguration\r
 280                                         .load(new FileInputStream(new File(cJronn\r
 281                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 282                         assertEquals(\r
 283                                         ((ConfExecutable<Jronn>) cJronn).getRunConfiguration(),\r
 284                                         loadedRun);\r
 285                         // Load run configuration as ConfExecutable\r
 286                         ConfiguredExecutable<Jronn> resurrectedCMuscle = (ConfiguredExecutable<Jronn>) cJronn\r
 287                                         .loadRunConfiguration(new FileInputStream(new File(cJronn.getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 288                         assertNotNull(resurrectedCMuscle);\r
 289                         assertEquals(resurrectedCMuscle.getExecutable().getInput(),\r
 290                                         AllTestSuit.test_input);\r
 291                         assertEquals(resurrectedCMuscle.getExecutable().getError(),\r
 292                                         "errrr.txt");\r
 293                         assertEquals(resurrectedCMuscle.getExecutable().getOutput(),\r
 294                                         "outtt.txt");\r
 295                         // See in details whether executables are the same\r
 296                         assertEquals(resurrectedCMuscle.getExecutable(), jronn);\r
 297 \r
 298                         ConfiguredExecutable<Jronn> resJronn = Configurator\r
 299                                         .configureExecutable(resurrectedCMuscle.getExecutable(),\r
 300                                                         Executable.ExecProvider.Local);\r
 301 \r
 302                         sexec = Configurator.getSyncEngine(resJronn, Executable.ExecProvider.Local);\r
 303                         sexec.executeJob();\r
 304                         al = sexec.waitForResult();\r
 305                         assertNotNull(al);\r
 306                 } catch (JobSubmissionException e) {\r
 307                         e.printStackTrace();\r
 308                         fail(e.getMessage());\r
 309                 } catch (JobExecutionException e) {\r
 310                         e.printStackTrace();\r
 311                         fail(e.getMessage());\r
 312                 } catch (FileNotFoundException e) {\r
 313                         e.printStackTrace();\r
 314                         fail(e.getMessage());\r
 315                 } catch (IOException e) {\r
 316                         e.printStackTrace();\r
 317                         fail(e.getMessage());\r
 318                 } catch (ResultNotAvailableException e) {\r
 319                         e.printStackTrace();\r
 320                         fail(e.getMessage());\r
 321                 }\r
 322         }\r
 323 \r
 324         @Test(groups = {AllTestSuit.test_group_runner})\r
 325         public void ConfigurationLoading() {\r
 326                 try {\r
 327                         RunnerConfig<Jronn> jronnConfig = ConfExecutable.getRunnerOptions(Jronn.class);\r
 328                         assertNull(jronnConfig);  // There is no supported parameters for Jronn\r
 329 \r
 330                         PresetManager<Jronn> jronnPresets = ConfExecutable.getRunnerPresets(Jronn.class);\r
 331                         assertNull(jronnPresets); // there is no presets\r
 332 \r
 333                         LimitsManager<Jronn> jronnLimits = ConfExecutable.getRunnerLimits(Jronn.class);\r
 334                         assertNotNull(jronnLimits);\r
 335                         assertTrue(jronnLimits.getLimits().size() > 0);\r
 336                         jronnLimits.validate(jronnPresets);\r
 337 \r
 338                 } catch (FileNotFoundException e) {\r
 339                         e.printStackTrace();\r
 340                         fail(e.getLocalizedMessage());\r
 341                 } catch (IOException e) {\r
 342                         e.printStackTrace();\r
 343                         fail(e.getLocalizedMessage());\r
 344                 } catch (ValidationException e) {\r
 345                         e.printStackTrace();\r
 346                         fail(e.getLocalizedMessage());\r
 347                 }\r
 348         }\r
 349 }\r