package compbio.ws.execstat; import java.io.File; import java.io.FileFilter; import java.io.FileWriter; import java.io.IOException; import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import compbio.engine.client.ConfExecutable; import compbio.engine.conf.PropertyHelperManager; import compbio.metadata.JobStatus; import compbio.util.FileUtil; import compbio.util.PropertyHelper; import compbio.ws.client.Services; /** * Number of runs of each WS = number of folders with name * * Number of successful runs = all runs with no result file * * Per period of time = limit per file creating time Runtime (avg/max) = * * started time - finished time * * Task & result size = result.size * * Abandoned runs - not collected runs * * Cancelled runs - cancelled * * Cluster vs local runs * * Reasons for failure = look in the err out? * * * Metadata required: * * work directory for local and cluster tasks = from Helper or cmd parameter. WS * names - enumeration. Status file names and content. * * @author pvtroshin * */ public class ExecutionStatCollector { static final int UNDEFINED = -1; private static final Logger log = Logger .getLogger(ExecutionStatCollector.class); static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss"); static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); static String getClusterJobDir() { String clusterdir = ph.getProperty("cluster.tmp.directory"); if (clusterdir != null) { clusterdir.trim(); } return clusterdir; } static void updateTime(File statFile) throws IOException { long lastMod = statFile.lastModified(); FileWriter fw = new FileWriter(statFile); fw.write(new Long(lastMod).toString()); fw.close(); } static String getLocalJobDir() { String locdir = ph.getProperty("local.tmp.directory"); if (locdir != null) { locdir.trim(); } return locdir; } /** * * @param args * @throws IOException * @throws SQLException */ public static void main(String[] args) throws IOException, SQLException { // updateTime(new File( // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED")); String workDir = PropertyHelperManager.getLocalPath() + getLocalJobDir().trim(); System.out.println(workDir); File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/local_jobsout", directories); List stats = new ArrayList(); for (File file : files) { JobDirectory jd = new JobDirectory(file); stats.add(jd.getJobStat()); // System.out.println(jd.getJobStat().getJobReportTabulated()); } StatProcessor sp = new StatProcessor(stats); System.out.println(sp.reportStat()); System.out.println(); System.out.println("!!!!!!!!!!!!!!!!!!"); System.out.println(); System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat()); StatDB.insertData(new HashSet(sp .getSingleWSStat(Services.TcoffeeWS).stats)); } static FileFilter directories = new FileFilter() { @Override public boolean accept(File pathname) { return pathname.isDirectory(); } }; static class JobDirectory { File jobdir; Map files = new HashMap(); public JobDirectory(File directory) { this.jobdir = directory; for (File f : jobdir.listFiles()) { files.put(f.getName(), f); } } public boolean hasStatus(JobStatus status) { return files.containsKey(status.toString()); } boolean isCollected() { return hasStatus(JobStatus.COLLECTED); } boolean isCancelled() { return hasStatus(JobStatus.CANCELLED); } long getStartTime() { long starttime = UNDEFINED; File startfile = files.get(JobStatus.STARTED.toString()); if (startfile == null) { startfile = files.get(JobStatus.SUBMITTED.toString()); } if (startfile != null) { starttime = startfile.lastModified(); /* * String start = FileUtil.readFileToString(startfile); * starttime = Long.parseLong(start.trim()); */ } return starttime; } String getClusterJobID() { String clustjobId = ""; File jobid = files.get("JOBID"); try { if (jobid != null) { clustjobId = FileUtil.readFileToString(jobid); } } catch (IOException ioe) { ioe.printStackTrace(); // TODO LOG } return clustjobId.trim(); } long getFinishedTime() { long ftime = UNDEFINED; File finished = files.get(JobStatus.FINISHED.toString()); if (finished != null) { ftime = finished.lastModified(); /* * String start = FileUtil.readFileToString(finished); ftime = * Long.parseLong(start.trim()); */ // System.out.println("f " + ftime); } /* * } catch (IOException e) { log.log(Level.WARN, * "Cannot parse finished time: " + e.getMessage(), e); } catch * (NumberFormatException e) { log.log(Level.WARN, * "Cannot parse finished time: " + e.getMessage(), e); } */ return ftime; } String getWSName() { String name = jobdir.getName().split("#")[0]; if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) { assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1; name = name.substring(1); } if (name.startsWith("ClustalW")) { name = name.trim().substring(name.length() - 1); } return name; } Services getService() { return Services.getService(getWSName() + "WS"); } // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out // Probcons task:fasta.in result:alignment.out /* * TODO replace with Universal names for WS! */ long getResultSize() { String name = getWSName(); File f = null; if (name.equalsIgnoreCase("Probcons")) { f = files.get("alignment.out"); } f = files.get("fasta.out"); if (f != null) { return f.length(); } return UNDEFINED; } long getInputSize() { File input = files.get("fasta.in"); if (input != null) { return input.length(); } return UNDEFINED; } StatProcessor.JobStat getJobStat() { return new StatProcessor.JobStat(getService(), getClusterJobID(), jobdir.getName(), getStartTime(), getFinishedTime(), getInputSize(), getResultSize(), isCollected(), isCancelled()); } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((jobdir == null) ? 0 : jobdir.hashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; JobDirectory other = (JobDirectory) obj; if (jobdir == null) { if (other.jobdir != null) return false; } else if (!jobdir.equals(other.jobdir)) return false; return true; } } }