webservices/compbio/ws/execstat/ExecutionStatCollector.java

   1 package compbio.ws.execstat;\r
   2 \r
   3 import java.io.File;\r
   4 import java.io.FileFilter;\r
   5 import java.io.FileWriter;\r
   6 import java.io.IOException;\r
   7 import java.sql.SQLException;\r
   8 import java.text.SimpleDateFormat;\r
   9 import java.util.ArrayList;\r
  10 import java.util.HashMap;\r
  11 import java.util.HashSet;\r
  12 import java.util.List;\r
  13 import java.util.Map;\r
  14 \r
  15 import org.apache.log4j.Logger;\r
  16 \r
  17 import compbio.engine.client.ConfExecutable;\r
  18 import compbio.engine.conf.PropertyHelperManager;\r
  19 import compbio.metadata.JobStatus;\r
  20 import compbio.util.FileUtil;\r
  21 import compbio.util.PropertyHelper;\r
  22 import compbio.ws.client.Services;\r
  23 \r
  24 /**\r
  25  * Number of runs of each WS = number of folders with name\r
  26  * \r
  27  * Number of successful runs = all runs with no result file\r
  28  * \r
  29  * Per period of time = limit per file creating time Runtime (avg/max) =\r
  30  * \r
  31  * started time - finished time\r
  32  * \r
  33  * Task & result size = result.size\r
  34  * \r
  35  * Abandoned runs - not collected runs\r
  36  * \r
  37  * Cancelled runs - cancelled\r
  38  * \r
  39  * Cluster vs local runs\r
  40  * \r
  41  * Reasons for failure = look in the err out?\r
  42  * \r
  43  * \r
  44  * Metadata required:\r
  45  * \r
  46  * work directory for local and cluster tasks = from Helper or cmd parameter. WS\r
  47  * names - enumeration. Status file names and content.\r
  48  * \r
  49  * @author pvtroshin\r
  50  * \r
  51  */\r
  52 public class ExecutionStatCollector {\r
  53 \r
  54         static final int UNDEFINED = -1;\r
  55 \r
  56         private static final Logger log = Logger\r
  57                         .getLogger(ExecutionStatCollector.class);\r
  58 \r
  59         static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss");\r
  60 \r
  61         static PropertyHelper ph = PropertyHelperManager.getPropertyHelper();\r
  62 \r
  63         static String getClusterJobDir() {\r
  64                 String clusterdir = ph.getProperty("cluster.tmp.directory");\r
  65                 if (clusterdir != null) {\r
  66                         clusterdir.trim();\r
  67                 }\r
  68                 return clusterdir;\r
  69         }\r
  70 \r
  71         static void updateTime(File statFile) throws IOException {\r
  72                 long lastMod = statFile.lastModified();\r
  73                 FileWriter fw = new FileWriter(statFile);\r
  74                 fw.write(new Long(lastMod).toString());\r
  75                 fw.close();\r
  76         }\r
  77 \r
  78         static String getLocalJobDir() {\r
  79                 String locdir = ph.getProperty("local.tmp.directory");\r
  80                 if (locdir != null) {\r
  81                         locdir.trim();\r
  82                 }\r
  83                 return locdir;\r
  84         }\r
  85 \r
  86         /**\r
  87          * \r
  88          * @param args\r
  89          * @throws IOException\r
  90          * @throws SQLException\r
  91          */\r
  92         public static void main(String[] args) throws IOException, SQLException {\r
  93 \r
  94                 // updateTime(new File(\r
  95                 // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED"));\r
  96 \r
  97                 String workDir = PropertyHelperManager.getLocalPath()\r
  98                                 + getLocalJobDir().trim();\r
  99                 System.out.println(workDir);\r
 100                 File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/local_jobsout",\r
 101                                 directories);\r
 102                 List<StatProcessor.JobStat> stats = new ArrayList<StatProcessor.JobStat>();\r
 103                 for (File file : files) {\r
 104                         JobDirectory jd = new JobDirectory(file);\r
 105                         stats.add(jd.getJobStat());\r
 106                         // System.out.println(jd.getJobStat().getJobReportTabulated());\r
 107                 }\r
 108                 StatProcessor sp = new StatProcessor(stats);\r
 109                 System.out.println(sp.reportStat());\r
 110                 System.out.println();\r
 111                 System.out.println("!!!!!!!!!!!!!!!!!!");\r
 112                 System.out.println();\r
 113                 System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat());\r
 114 \r
 115                 StatDB.insertData(new HashSet<StatProcessor.JobStat>(sp\r
 116                                 .getSingleWSStat(Services.TcoffeeWS).stats));\r
 117         }\r
 118 \r
 119         static FileFilter directories = new FileFilter() {\r
 120                 @Override\r
 121                 public boolean accept(File pathname) {\r
 122                         return pathname.isDirectory();\r
 123                 }\r
 124         };\r
 125 \r
 126         static class JobDirectory {\r
 127 \r
 128                 File jobdir;\r
 129                 Map<String, File> files = new HashMap<String, File>();\r
 130 \r
 131                 public JobDirectory(File directory) {\r
 132                         this.jobdir = directory;\r
 133                         for (File f : jobdir.listFiles()) {\r
 134                                 files.put(f.getName(), f);\r
 135                         }\r
 136                 }\r
 137 \r
 138                 public boolean hasStatus(JobStatus status) {\r
 139                         return files.containsKey(status.toString());\r
 140                 }\r
 141 \r
 142                 boolean isCollected() {\r
 143                         return hasStatus(JobStatus.COLLECTED);\r
 144                 }\r
 145 \r
 146                 boolean isCancelled() {\r
 147                         return hasStatus(JobStatus.CANCELLED);\r
 148                 }\r
 149 \r
 150                 long getStartTime() {\r
 151                         long starttime = UNDEFINED;\r
 152                         File startfile = files.get(JobStatus.STARTED.toString());\r
 153                         if (startfile == null) {\r
 154                                 startfile = files.get(JobStatus.SUBMITTED.toString());\r
 155                         }\r
 156                         if (startfile != null) {\r
 157                                 starttime = startfile.lastModified();\r
 158                                 /*\r
 159                                  * String start = FileUtil.readFileToString(startfile);\r
 160                                  * starttime = Long.parseLong(start.trim());\r
 161                                  */\r
 162                         }\r
 163                         return starttime;\r
 164                 }\r
 165 \r
 166                 String getClusterJobID() {\r
 167                         String clustjobId = "";\r
 168                         File jobid = files.get("JOBID");\r
 169                         try {\r
 170                                 if (jobid != null) {\r
 171                                         clustjobId = FileUtil.readFileToString(jobid);\r
 172                                 }\r
 173                         } catch (IOException ioe) {\r
 174                                 ioe.printStackTrace();\r
 175                                 // TODO LOG\r
 176                         }\r
 177                         return clustjobId.trim();\r
 178                 }\r
 179 \r
 180                 long getFinishedTime() {\r
 181                         long ftime = UNDEFINED;\r
 182                         File finished = files.get(JobStatus.FINISHED.toString());\r
 183                         if (finished != null) {\r
 184                                 ftime = finished.lastModified();\r
 185                                 /*\r
 186                                  * String start = FileUtil.readFileToString(finished); ftime =\r
 187                                  * Long.parseLong(start.trim());\r
 188                                  */\r
 189                                 // System.out.println("f " + ftime);\r
 190                         }\r
 191                         /*\r
 192                          * } catch (IOException e) { log.log(Level.WARN,\r
 193                          * "Cannot parse finished time: " + e.getMessage(), e); } catch\r
 194                          * (NumberFormatException e) { log.log(Level.WARN,\r
 195                          * "Cannot parse finished time: " + e.getMessage(), e); }\r
 196                          */\r
 197                         return ftime;\r
 198                 }\r
 199 \r
 200                 String getWSName() {\r
 201                         String name = jobdir.getName().split("#")[0];\r
 202                         if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) {\r
 203                                 assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1;\r
 204                                 name = name.substring(1);\r
 205                         }\r
 206                         if (name.startsWith("ClustalW")) {\r
 207                                 name = name.trim().substring(name.length() - 1);\r
 208                         }\r
 209                         return name;\r
 210                 }\r
 211 \r
 212                 Services getService() {\r
 213                         return Services.getService(getWSName() + "WS");\r
 214                 }\r
 215                 // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out\r
 216                 // Probcons task:fasta.in result:alignment.out\r
 217                 /*\r
 218                  * TODO replace with Universal names for WS!\r
 219                  */\r
 220                 long getResultSize() {\r
 221                         String name = getWSName();\r
 222                         File f = null;\r
 223                         if (name.equalsIgnoreCase("Probcons")) {\r
 224                                 f = files.get("alignment.out");\r
 225                         }\r
 226                         f = files.get("fasta.out");\r
 227                         if (f != null) {\r
 228                                 return f.length();\r
 229                         }\r
 230                         return UNDEFINED;\r
 231                 }\r
 232 \r
 233                 long getInputSize() {\r
 234                         File input = files.get("fasta.in");\r
 235                         if (input != null) {\r
 236                                 return input.length();\r
 237                         }\r
 238                         return UNDEFINED;\r
 239                 }\r
 240 \r
 241                 StatProcessor.JobStat getJobStat() {\r
 242                         return new StatProcessor.JobStat(getService(), getClusterJobID(),\r
 243                                         jobdir.getName(), getStartTime(), getFinishedTime(),\r
 244                                         getInputSize(), getResultSize(), isCollected(),\r
 245                                         isCancelled());\r
 246                 }\r
 247 \r
 248                 @Override\r
 249                 public int hashCode() {\r
 250                         final int prime = 31;\r
 251                         int result = 1;\r
 252                         result = prime * result\r
 253                                         + ((jobdir == null) ? 0 : jobdir.hashCode());\r
 254                         return result;\r
 255                 }\r
 256 \r
 257                 @Override\r
 258                 public boolean equals(Object obj) {\r
 259                         if (this == obj)\r
 260                                 return true;\r
 261                         if (obj == null)\r
 262                                 return false;\r
 263                         if (getClass() != obj.getClass())\r
 264                                 return false;\r
 265                         JobDirectory other = (JobDirectory) obj;\r
 266                         if (jobdir == null) {\r
 267                                 if (other.jobdir != null)\r
 268                                         return false;\r
 269                         } else if (!jobdir.equals(other.jobdir))\r
 270                                 return false;\r
 271                         return true;\r
 272                 }\r
 273 \r
 274         }\r
 275 }\r