webservices/compbio/ws/execstat/ExecutionStatCollector.java

   1 package compbio.ws.execstat;\r
   2 \r
   3 import java.io.File;\r
   4 import java.io.FileFilter;\r
   5 import java.io.FileWriter;\r
   6 import java.io.IOException;\r
   7 import java.text.SimpleDateFormat;\r
   8 import java.util.ArrayList;\r
   9 import java.util.HashMap;\r
  10 import java.util.List;\r
  11 import java.util.Map;\r
  12 \r
  13 import org.apache.log4j.Logger;\r
  14 \r
  15 import compbio.engine.client.ConfExecutable;\r
  16 import compbio.engine.conf.PropertyHelperManager;\r
  17 import compbio.metadata.JobStatus;\r
  18 import compbio.util.FileUtil;\r
  19 import compbio.util.PropertyHelper;\r
  20 import compbio.ws.client.Services;\r
  21 \r
  22 /**\r
  23  * Number of runs of each WS = number of folders with name\r
  24  * \r
  25  * Number of successful runs = all runs with no result file\r
  26  * \r
  27  * Per period of time = limit per file creating time Runtime (avg/max) =\r
  28  * \r
  29  * started time - finished time\r
  30  * \r
  31  * Task & result size = result.size\r
  32  * \r
  33  * Abandoned runs - not collected runs\r
  34  * \r
  35  * Cancelled runs - cancelled\r
  36  * \r
  37  * Cluster vs local runs\r
  38  * \r
  39  * Reasons for failure = look in the err out?\r
  40  * \r
  41  * \r
  42  * Metadata required:\r
  43  * \r
  44  * work directory for local and cluster tasks = from Helper or cmd parameter. WS\r
  45  * names - enumeration. Status file names and content.\r
  46  * \r
  47  * @author pvtroshin\r
  48  * \r
  49  */\r
  50 public class ExecutionStatCollector {\r
  51 \r
  52         static final int UNDEFINED = -1;\r
  53 \r
  54         private static final Logger log = Logger\r
  55                         .getLogger(ExecutionStatCollector.class);\r
  56 \r
  57         static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss");\r
  58 \r
  59         static PropertyHelper ph = PropertyHelperManager.getPropertyHelper();\r
  60 \r
  61         static String getClusterJobDir() {\r
  62                 String clusterdir = ph.getProperty("cluster.tmp.directory");\r
  63                 if (clusterdir != null) {\r
  64                         clusterdir.trim();\r
  65                 }\r
  66                 return clusterdir;\r
  67         }\r
  68 \r
  69         static void updateTime(File statFile) throws IOException {\r
  70                 long lastMod = statFile.lastModified();\r
  71                 FileWriter fw = new FileWriter(statFile);\r
  72                 fw.write(new Long(lastMod).toString());\r
  73                 fw.close();\r
  74         }\r
  75 \r
  76         static String getLocalJobDir() {\r
  77                 String locdir = ph.getProperty("local.tmp.directory");\r
  78                 if (locdir != null) {\r
  79                         locdir.trim();\r
  80                 }\r
  81                 return locdir;\r
  82         }\r
  83 \r
  84         /**\r
  85          * \r
  86          * @param args\r
  87          * @throws IOException\r
  88          */\r
  89         public static void main(String[] args) throws IOException {\r
  90 \r
  91                 // updateTime(new File(\r
  92                 // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED"));\r
  93 \r
  94                 String workDir = PropertyHelperManager.getLocalPath()\r
  95                                 + getLocalJobDir().trim();\r
  96                 System.out.println(workDir);\r
  97                 File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/jobsout",\r
  98                                 directories);\r
  99                 List<StatProcessor.JobStat> stats = new ArrayList<StatProcessor.JobStat>();\r
 100                 for (File file : files) {\r
 101                         JobDirectory jd = new JobDirectory(file);\r
 102                         stats.add(jd.getJobStat());\r
 103                         // System.out.println(jd.getJobStat().getJobReportTabulated());\r
 104                 }\r
 105                 StatProcessor sp = new StatProcessor(stats);\r
 106                 System.out.println(sp.reportStat());\r
 107                 System.out.println();\r
 108                 System.out.println("!!!!!!!!!!!!!!!!!!");\r
 109                 System.out.println();\r
 110                 System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat());\r
 111         }\r
 112 \r
 113         static FileFilter directories = new FileFilter() {\r
 114                 @Override\r
 115                 public boolean accept(File pathname) {\r
 116                         return pathname.isDirectory();\r
 117                 }\r
 118         };\r
 119 \r
 120         static class JobDirectory {\r
 121 \r
 122                 File jobdir;\r
 123                 Map<String, File> files = new HashMap<String, File>();\r
 124 \r
 125                 public JobDirectory(File directory) {\r
 126                         this.jobdir = directory;\r
 127                         for (File f : jobdir.listFiles()) {\r
 128                                 files.put(f.getName(), f);\r
 129                         }\r
 130                 }\r
 131 \r
 132                 public boolean hasStatus(JobStatus status) {\r
 133                         return files.containsKey(status.toString());\r
 134                 }\r
 135 \r
 136                 boolean isCollected() {\r
 137                         return hasStatus(JobStatus.COLLECTED);\r
 138                 }\r
 139 \r
 140                 boolean isCancelled() {\r
 141                         return hasStatus(JobStatus.CANCELLED);\r
 142                 }\r
 143 \r
 144                 long getStartTime() {\r
 145                         long starttime = UNDEFINED;\r
 146                         File startfile = files.get(JobStatus.STARTED.toString());\r
 147                         if (startfile == null) {\r
 148                                 startfile = files.get(JobStatus.SUBMITTED.toString());\r
 149                         }\r
 150                         if (startfile != null) {\r
 151                                 starttime = startfile.lastModified();\r
 152                                 /*\r
 153                                  * String start = FileUtil.readFileToString(startfile);\r
 154                                  * starttime = Long.parseLong(start.trim());\r
 155                                  */\r
 156                         }\r
 157                         return starttime;\r
 158                 }\r
 159 \r
 160                 long getFinishedTime() {\r
 161                         long ftime = UNDEFINED;\r
 162                         File finished = files.get(JobStatus.FINISHED.toString());\r
 163                         if (finished != null) {\r
 164                                 ftime = finished.lastModified();\r
 165                                 /*\r
 166                                  * String start = FileUtil.readFileToString(finished); ftime =\r
 167                                  * Long.parseLong(start.trim());\r
 168                                  */\r
 169                                 // System.out.println("f " + ftime);\r
 170                         }\r
 171                         /*\r
 172                          * } catch (IOException e) { log.log(Level.WARN,\r
 173                          * "Cannot parse finished time: " + e.getMessage(), e); } catch\r
 174                          * (NumberFormatException e) { log.log(Level.WARN,\r
 175                          * "Cannot parse finished time: " + e.getMessage(), e); }\r
 176                          */\r
 177                         return ftime;\r
 178                 }\r
 179 \r
 180                 String getWSName() {\r
 181                         String name = jobdir.getName().split("#")[0];\r
 182                         if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) {\r
 183                                 assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1;\r
 184                                 name = name.substring(1);\r
 185                         }\r
 186                         if (name.startsWith("ClustalW")) {\r
 187                                 name = name.trim().substring(name.length() - 1);\r
 188                         }\r
 189                         return name;\r
 190                 }\r
 191 \r
 192                 Services getService() {\r
 193                         return Services.getService(getWSName() + "WS");\r
 194                 }\r
 195                 // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out\r
 196                 // Probcons task:fasta.in result:alignment.out\r
 197                 /*\r
 198                  * TODO replace with Universal names for WS!\r
 199                  */\r
 200                 long getResultSize() {\r
 201                         String name = getWSName();\r
 202                         File f = null;\r
 203                         if (name.equalsIgnoreCase("Probcons")) {\r
 204                                 f = files.get("alignment.out");\r
 205                         }\r
 206                         f = files.get("fasta.out");\r
 207                         if (f != null) {\r
 208                                 return f.length();\r
 209                         }\r
 210                         return UNDEFINED;\r
 211                 }\r
 212 \r
 213                 long getInputSize() {\r
 214                         File input = files.get("fasta.in");\r
 215                         if (input != null) {\r
 216                                 return input.length();\r
 217                         }\r
 218                         return UNDEFINED;\r
 219                 }\r
 220 \r
 221                 StatProcessor.JobStat getJobStat() {\r
 222                         return new StatProcessor.JobStat(getService(), jobdir.getName(),\r
 223                                         getStartTime(), getFinishedTime(), getInputSize(),\r
 224                                         getResultSize(), isCollected(), isCancelled());\r
 225                 }\r
 226 \r
 227                 @Override\r
 228                 public int hashCode() {\r
 229                         final int prime = 31;\r
 230                         int result = 1;\r
 231                         result = prime * result\r
 232                                         + ((jobdir == null) ? 0 : jobdir.hashCode());\r
 233                         return result;\r
 234                 }\r
 235 \r
 236                 @Override\r
 237                 public boolean equals(Object obj) {\r
 238                         if (this == obj)\r
 239                                 return true;\r
 240                         if (obj == null)\r
 241                                 return false;\r
 242                         if (getClass() != obj.getClass())\r
 243                                 return false;\r
 244                         JobDirectory other = (JobDirectory) obj;\r
 245                         if (jobdir == null) {\r
 246                                 if (other.jobdir != null)\r
 247                                         return false;\r
 248                         } else if (!jobdir.equals(other.jobdir))\r
 249                                 return false;\r
 250                         return true;\r
 251                 }\r
 252 \r
 253         }\r
 254 }\r