From: Jim Procter Date: Thu, 31 May 2018 12:48:14 +0000 (+0100) Subject: Merge branch 'spike/JAL-1950_hmmer3client' into features/mchmmer_merge_JAL-1950 X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=09b8644607fb5b9debbfe39a7177922b0e66aa9a;hp=31117f8593d9c51c6238be3d40771819da96939d;p=jalview.git Merge branch 'spike/JAL-1950_hmmer3client' into features/mchmmer_merge_JAL-1950 Conflicts: src/jalview/gui/AlignViewport.java --- diff --git a/examples/groovy/hmmertestimport.groovy b/examples/groovy/hmmertestimport.groovy new file mode 100644 index 0000000..88bd0e1 --- /dev/null +++ b/examples/groovy/hmmertestimport.groovy @@ -0,0 +1,6 @@ +// def alv = new jalview.io.FileLoader().LoadFileWaitTillLoaded("examples/testdata/hmmer3/alignment_res.fa.gz","File").getViewport(); +def alv = jalview.bin.Jalview.getCurrentAlignFrame().getViewport(); +def al = alv.getAlignment(); +def jproc = new jalview.ws.ebi.HmmerJSONProcessor(al) +jproc.parseFrom(new jalview.io.FileParse("/Users/jprocter/git/jalview/examples/testdata/hmmer3/hmmeresult.json.gz",jalview.io.DataSourceType.FILE)) +jproc.updateView(alv) \ No newline at end of file diff --git a/examples/testdata/hmmer3/alignment_frag.fa.gz b/examples/testdata/hmmer3/alignment_frag.fa.gz new file mode 100644 index 0000000..a4e79b2 Binary files /dev/null and b/examples/testdata/hmmer3/alignment_frag.fa.gz differ diff --git a/examples/testdata/hmmer3/alignment_res.fa.gz b/examples/testdata/hmmer3/alignment_res.fa.gz new file mode 100644 index 0000000..03206f4 Binary files /dev/null and b/examples/testdata/hmmer3/alignment_res.fa.gz differ diff --git a/examples/testdata/hmmer3/hit_fragment.json.gz b/examples/testdata/hmmer3/hit_fragment.json.gz new file mode 100644 index 0000000..bbc8405 Binary files /dev/null and b/examples/testdata/hmmer3/hit_fragment.json.gz differ diff --git a/examples/testdata/hmmer3/hmmeresult.json.gz b/examples/testdata/hmmer3/hmmeresult.json.gz new file mode 100644 index 0000000..c3fa9a2 Binary files /dev/null and b/examples/testdata/hmmer3/hmmeresult.json.gz differ diff --git a/src/jalview/datamodel/AlignmentAnnotation.java b/src/jalview/datamodel/AlignmentAnnotation.java index 0f7883e..0a569b5 100755 --- a/src/jalview/datamodel/AlignmentAnnotation.java +++ b/src/jalview/datamodel/AlignmentAnnotation.java @@ -925,6 +925,7 @@ public class AlignmentAnnotation * @param seqRef * @param startRes * @param alreadyMapped + * - annotation are at aligned columns */ public void createSequenceMapping(SequenceI seqRef, int startRes, boolean alreadyMapped) diff --git a/src/jalview/gui/AlignViewport.java b/src/jalview/gui/AlignViewport.java index 862c4fb..39ded7f 100644 --- a/src/jalview/gui/AlignViewport.java +++ b/src/jalview/gui/AlignViewport.java @@ -261,7 +261,11 @@ public class AlignViewport extends AlignmentViewport setFont(new Font(fontName, style, Integer.parseInt(fontSize)), true); AlignmentI al = getAlignment(); - al.setGapCharacter(Cache.getDefault("GAP_SYMBOL", "-").charAt(0)); + + if (Cache.getDefault("NORMALISE_GAPS", true)) + { + al.setGapCharacter(Cache.getDefault("GAP_SYMBOL", "-").charAt(0)); + } // We must set conservation and consensus before setting colour, // as Blosum and Clustal require this to be done diff --git a/src/jalview/gui/RestServiceEditorPane.java b/src/jalview/gui/RestServiceEditorPane.java index 2e2593b..06f4e06 100644 --- a/src/jalview/gui/RestServiceEditorPane.java +++ b/src/jalview/gui/RestServiceEditorPane.java @@ -471,10 +471,13 @@ public class RestServiceEditorPane extends GRestServiceEditorPane final Thread runner = Thread.currentThread(); JFrame df = new JFrame(); df.getContentPane().setLayout(new BorderLayout()); - df.getContentPane().add((nulserv = !nulserv) - ? new RestServiceEditorPane(jalview.ws.rest.RestClient - .makeShmmrRestClient().getRestDescription()) - : new RestServiceEditorPane(), BorderLayout.CENTER); + df.getContentPane().add( + (nulserv = !nulserv) ? new RestServiceEditorPane( + jalview.ws.rest.clientdefs.ShmrRestClient + .makeShmmrRestClient() + .getRestDescription()) + : new RestServiceEditorPane(), + BorderLayout.CENTER); df.setBounds(100, 100, 600, 400); df.addComponentListener(new ComponentListener() { diff --git a/src/jalview/schemes/AnnotationColourGradient.java b/src/jalview/schemes/AnnotationColourGradient.java index c28ea5f..3ed18bf 100755 --- a/src/jalview/schemes/AnnotationColourGradient.java +++ b/src/jalview/schemes/AnnotationColourGradient.java @@ -36,6 +36,16 @@ import java.util.Map; public class AnnotationColourGradient extends FollowerColourScheme { + /** + * map positional scores to transparency rather than colour + */ + boolean positionToTransparency = true; + + /** + * compute shade based on annotation row score + */ + boolean perLineScore = true; + public static final int NO_THRESHOLD = -1; public static final int BELOW_THRESHOLD = 0; @@ -93,6 +103,8 @@ public class AnnotationColourGradient extends FollowerColourScheme acg.predefinedColours = predefinedColours; acg.seqAssociated = seqAssociated; acg.noGradient = noGradient; + acg.positionToTransparency = positionToTransparency; + acg.perLineScore = perLineScore; return acg; } @@ -185,20 +197,22 @@ public class AnnotationColourGradient extends FollowerColourScheme } else { - seqannot = new IdentityHashMap(); + seqannot = new IdentityHashMap<>(); } // resolve the context containing all the annotation for the sequence AnnotatedCollectionI alcontext = alignment instanceof AlignmentI ? alignment : alignment.getContext(); - boolean f = true, rna = false; - for (AlignmentAnnotation alan : alcontext - .findAnnotation(annotation.getCalcId())) + boolean f = true, sf = true, rna = false; + long plcount = 0, ancount = 0; + for (AlignmentAnnotation alan : alcontext.findAnnotation(annotation + .getCalcId())) { if (alan.sequenceRef != null && (alan.label != null && annotation != null && alan.label.equals(annotation.label))) { + ancount++; if (!rna && alan.isRNA()) { rna = true; @@ -213,8 +227,26 @@ public class AnnotationColourGradient extends FollowerColourScheme aamin = alan.graphMin; } f = false; + if (alan.score == alan.score) + { + if (sf || alan.score < plmin) + { + plmin = alan.score; + } + if (sf || alan.score > plmax) + { + plmax = alan.score; + } + sf = false; + plcount++; + } } } + if (plcount > 0 && plcount == ancount) + { + perLineScore = plcount == ancount; + aamax=plmax; + } if (rna) { ColourSchemeProperty.initRnaHelicesShading(1 + (int) aamax); @@ -222,7 +254,15 @@ public class AnnotationColourGradient extends FollowerColourScheme } } - float aamin = 0f, aamax = 0f; + /** + * positional annotation max/min + */ + double aamin = 0.0, aamax = 0.0; + + /** + * per line score max/min + */ + double plmin = Double.NaN, plmax = Double.NaN; public AlignmentAnnotation getAnnotation() { @@ -435,11 +475,25 @@ public class AnnotationColourGradient extends FollowerColourScheme } } - int dr = (int) (redRange * range + redMin); - int dg = (int) (greenRange * range + greenMin); - int db = (int) (blueRange * range + blueMin); - - return new Color(dr, dg, db); + // midtr sets the ceiling for bleaching out the shading + int trans = 0, midtr = 239; + if (perLineScore) + { + trans = (int) ((1f - range) * midtr); + range = (float) ((annotation.score - plmin) / (plmax - aamin)); + } + int dr = (int) (redRange * range + redMin), + dg = (int) (greenRange * range + greenMin), + db = (int) (blueRange * range + blueMin); + if (annotation.score == annotation.score && positionToTransparency) + { + return new Color(Math.min(dr + trans, midtr), Math.min(dg + + trans, midtr), Math.min(db + trans, midtr)); + } + else + { + return new Color(dr, dg, db); + } } public boolean isPredefinedColours() diff --git a/src/jalview/util/Comparison.java b/src/jalview/util/Comparison.java index d4fc233..4afa12d 100644 --- a/src/jalview/util/Comparison.java +++ b/src/jalview/util/Comparison.java @@ -256,7 +256,7 @@ public class Comparison */ public static final boolean isGap(char c) { - return (c == GAP_DASH || c == GAP_DOT || c == GAP_SPACE) ? true : false; + return (c == GAP_DASH || c == GAP_DOT || c == GAP_SPACE); } /** diff --git a/src/jalview/util/HttpUtils.java b/src/jalview/util/HttpUtils.java index a5a9460..97c84bc 100644 --- a/src/jalview/util/HttpUtils.java +++ b/src/jalview/util/HttpUtils.java @@ -20,6 +20,9 @@ */ package jalview.util; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -64,4 +67,46 @@ public class HttpUtils return false; } + /** + * download from given URL and return a pointer to temporary file + */ + public static File fetchURLToTemp(String url) throws OutOfMemoryError, + IOException + { + long time = System.currentTimeMillis(); + URL rcall = new URL(url); + + InputStream is = new BufferedInputStream(rcall.openStream()); + File outFile = null; + try + { + outFile = File.createTempFile("jalview", ".xml"); + outFile.deleteOnExit(); + if (outFile.length() == 0) + { + outFile.delete(); + return null; + } + } catch (Exception ex) + { + } + + if (outFile != null) + { + FileOutputStream fio = new FileOutputStream(outFile); + byte[] bb = new byte[32 * 1024]; + int l; + while ((l = is.read(bb)) > 0) + { + fio.write(bb, 0, l); + } + fio.close(); + is.close(); + return outFile; + } + else + { + return null; + } + } } diff --git a/src/jalview/ws/ebi/HmmerJSONProcessor.java b/src/jalview/ws/ebi/HmmerJSONProcessor.java new file mode 100644 index 0000000..428c498 --- /dev/null +++ b/src/jalview/ws/ebi/HmmerJSONProcessor.java @@ -0,0 +1,331 @@ +package jalview.ws.ebi; + +import jalview.datamodel.AlignmentAnnotation; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.Annotation; +import jalview.datamodel.SequenceFeature; +import jalview.datamodel.SequenceGroup; +import jalview.datamodel.SequenceI; +import jalview.io.FileParse; +import jalview.viewmodel.AlignmentViewport; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +public class HmmerJSONProcessor +{ + /** + * result to be annotated. may not be null + */ + AlignmentI resultAl; + + /** + * viewport on the alignment. may be null at construction time + */ + AlignmentViewport viewAl = null; + + public HmmerJSONProcessor(AlignmentI searchResult) + { + resultAl = searchResult; + } + + public void parseFrom(FileParse jsonsource) throws IOException, + OutOfMemoryError + { + JSONParser hmmerResultParser = new JSONParser(); + Object jsonResults = null; + try + { + jsonResults = hmmerResultParser.parse(jsonsource.getReader()); + } catch (Exception p) + { + throw new IOException("While parsing from " + jsonsource.getInFile(), + p); + } + if (jsonResults == null) + { + throw new IOException("No data at" + jsonsource.getInFile()); + } + if (!(jsonResults instanceof JSONObject)) + { + throw new IOException("Unexpected JSON model at " + + jsonsource.getInFile()); + } + try + { + JSONObject hmmsearchr = (JSONObject) ((JSONObject) jsonResults) + .get("results"); + // now process the hits + addStatistics((JSONObject) hmmsearchr.get("stats")); + JSONArray jsonArray = (JSONArray) hmmsearchr.get("hits"); + long p = 1; + for (Object hit : jsonArray) + { + JSONObject hmmhit = (JSONObject) hit; + addHit(hmmhit, p++); + } + } catch (ClassCastException q) + { + throw new IOException("Unexpected JSON model content at " + + jsonsource.getInFile(), q); + } + } + + /** + * + * @param object + * - actually a JSONObject key value set of search statistics. + */ + public void addStatistics(JSONObject stats) + { + for (Object stat : stats.keySet()) + { + String key = (String) stat; + Object val = stats.get(key); + resultAl.setProperty(key, "" + val); + } + } + + // encodings for JSON keys + /** + * score becomes sequence associated AlignmentAnnotation + */ + private String[] score = { "aliId", "ali_IdCount", "bitscore", "ievalue", + "aliSim", "aliSimCount", "aliL", "aliSim", "ievalue", "cevalue" }; + + /** + * attrib becomes numeric or binary attribute for sequence with respect to + * this hmmsearch run + */ + private String[] attrib = { "bias", "oasc", "is_included", "is_reported" }; + + /** + * name of the hmmsearch query + */ + private String[] label = { "alihmmname" // (query label?)}, + }; + + /** + * integer attributes for each + */ + private String[] ipos = { "alihmmfrom", "alihmmto" }, pos_l = { + "alimline", "alimodel", "alirfline" }; + + /** + * positional quantitative annotation encoded as strings. + */ + private String[] pos_nscore = { "alippline" }; + + // + // mapping of keys to types of property on sequence + // + public void addHit(JSONObject hmmrhit, long p) + { + String sname = (String) hmmrhit.get("name"); + SequenceI[] hits = resultAl.findSequenceMatch(sname); + if (hits == null) + { + System.err.println("No seq for " + sname); + } + double pvalue = (Double) hmmrhit.get("pvalue"); + + double evalue = Double.valueOf("" + hmmrhit.get("evalue")); + for (Object domainhit : ((JSONArray) hmmrhit.get("domains"))) + { + JSONObject dhit = (JSONObject) domainhit; + // dhit.get(key) + + // alihmmfrom,alihmmto alimodel + long alihmmfrom = (long) dhit.get("alihmmfrom"), alihmmto = (long) dhit + .get("alihmmto"), alisqfrom = (long) dhit.get("alisqfrom"), alisqto = (long) dhit + .get("alisqto"); + + // alisqfrom,alisqto,aliaseq + + // alippline + String aliaseq = (String) dhit.get("aliaseq"), alimodel = (String) dhit + .get("alimodel"), ppline = (String) dhit.get("alippline"); + // + int found = 0; + SequenceI firsthit = null; + for (SequenceI hitseq : hits) + { + // match alisqfrom,alisqto,seq + if (hitseq.getStart() == alisqfrom && hitseq.getEnd() == alisqto) + { + if (found == 0) + { + firsthit = hitseq; + } + found++; // annotated a sequence + AlignmentAnnotation alipp = parsePosteriorProb(ppline); + AlignmentAnnotation pval = new AlignmentAnnotation("p-value", + "hmmer3 pvalue", pvalue); + AlignmentAnnotation eval = new AlignmentAnnotation("e-value", + "hmmer3 evalue", evalue); + pval.setCalcId("HMMER3"); + eval.setCalcId("HMMER3"); + alipp.setCalcId("HMMER3"); + hitseq.addAlignmentAnnotation(pval); + hitseq.addAlignmentAnnotation(eval); + alipp.createSequenceMapping(hitseq, hitseq.getStart(), false); + hitseq.addAlignmentAnnotation(alipp); + String arch; + hitseq.addSequenceFeature(new SequenceFeature( + "Pfam Domain Architecture", (hmmrhit.get("archindex")) + + " " + (arch = (String) hmmrhit.get("arch")), 0, + 0, + (hmmrhit.get("archScore") != null ? Integer + .valueOf((String) hmmrhit.get("archScore")) : 0f), + "HMMER3")); + addArchGroup(hitseq, arch); + alipp.setScore(Double.valueOf("" + dhit.get("bitscore"))); + alipp.adjustForAlignment(); + resultAl.addAnnotation(pval); + resultAl.addAnnotation(eval); + resultAl.addAnnotation(alipp); + alipp.validateRangeAndDisplay(); + } + } + // look for other sequences represented by this hit and create rep groups + // could be in "pdbs", or .. + addRedundantSeqGroup(firsthit, alisqfrom, alisqto, + (JSONArray) hmmrhit.get("seqs"), true); + } + } + + /** + * series of operations to perform for the viewpanel associated with the + * alignment + */ + private List viewOps = new ArrayList(); + + public void updateView(AlignmentViewport view) + { + viewAl = view; + for (Runnable op : viewOps) + { + op.run(); + } + } + + private void addRedundantSeqGroup(final SequenceI firsthit, + long alisqfrom, long alisqto, JSONArray others, boolean justDelete) + { + if (others != null) + { + final SequenceGroup repgroup = new SequenceGroup(); + repgroup.setSeqrep(firsthit); + repgroup.addOrRemove(firsthit, false); + repgroup.setStartRes(0); + repgroup.setEndRes(resultAl.getWidth() - 1); + for (Object otherseq : others.toArray(new JSONObject[0])) + { + String repseq = (String) ((JSONObject) otherseq).get("dn"); + SequenceI[] other = resultAl.findSequenceMatch(repseq); + if (other != null && other.length > 0) + { + if (justDelete) + { + for (SequenceI oth : other) + { + resultAl.deleteSequence(oth); + } + ; + } + else + { + int ofound = 0; + for (SequenceI oth : other) + { + if (oth.getStart() == alisqfrom && oth.getEnd() == alisqto) + { + ofound++; + repgroup.addSequence(oth, false); + } + } + if (ofound == 0) + { + System.err.println("Warn - no match for redundant hit " + + repseq + "/" + alisqfrom + "-" + alisqto); + } + if (ofound > 1) + { + System.err + .println("Warn - multiple matches for redundant hit " + + repseq + "/" + alisqfrom + "-" + alisqto); + } + } + } + } + if (repgroup.getSequences().size() > 1) + { + // queue a hide operation + final HmmerJSONProcessor me = this; + viewOps.add(new Runnable() + { + @Override + public void run() + { + me.viewAl.hideRepSequences(firsthit, repgroup); + } + }); + } + } + } + + Map groups = new HashMap(); + + private void addArchGroup(SequenceI seqToAdd, String groupNam) + { + SequenceGroup sg = groups.get(groupNam); + if (sg == null) + { + sg = new SequenceGroup(); + sg.setName(groupNam); + sg.addSequence(seqToAdd, false); + sg.setStartRes(0); + sg.setEndRes(resultAl.getWidth() - 1); + groups.put(groupNam, sg); + resultAl.addGroup(sg); + } + else + { + sg.addSequence(seqToAdd, false); + } + } + + private AlignmentAnnotation parsePosteriorProb(String ppline) + { + Annotation[] ae = new Annotation[ppline.length()]; + int spos = 0; + for (int i = 0, iSize = ppline.length(); i < iSize; i++) + { + char pp = ppline.charAt(i); + if (pp == '*') + { + ae[spos++] = new Annotation(10f); + } + else + { + if (pp >= '0' && pp <= '9') + { + ae[spos++] = new Annotation(Integer.valueOf("" + pp)); + } + } + } + AlignmentAnnotation pprob = new AlignmentAnnotation( + "Posterior Probability", + "Likelihood of HMM fit at each hit position.", ae); + pprob.graph = AlignmentAnnotation.BAR_GRAPH; + pprob.visible = false; + return pprob; + } +} diff --git a/src/jalview/ws/ebi/hmmerClient.java b/src/jalview/ws/ebi/hmmerClient.java new file mode 100644 index 0000000..41cd0d8 --- /dev/null +++ b/src/jalview/ws/ebi/hmmerClient.java @@ -0,0 +1,319 @@ +package jalview.ws.ebi; + +import jalview.datamodel.AlignmentI; +import jalview.io.AppletFormatAdapter; +import jalview.io.DataSourceType; +import jalview.io.FileFormat; +import jalview.io.FileParse; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Matcher; + +import org.apache.axis.transport.http.HTTPConstants; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONObject; + +import compbio.util.FileUtil; + +public class hmmerClient +{ + /** + * URLs for ebi api + */ + static String baseUrl = "http://www.ebi.ac.uk/Tools/hmmer", + jackH = "/search/jackhmmer", phmmer = "/search/phmmer", + hmmscan = "/search/hmmscan", hmmsearch = "/search/hmmsearch"; + + static String edseq = ">2abl_A mol:protein length:163 ABL TYROSINE KINASE\nMGPSENDPNLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPSNYITPVNSLEKHS\nWYHGPVSRNAAEYLLSSGINGSFLVRESESSPGQRSISLRYEGRVYHYRINTASDGKLYVSSESRFNTLAE\nLVHHHSTVADGLITTLHYPAP"; + + public static void main(String[] args) + { + String instr = edseq; + if (args.length > 0) + { + try + { + instr = FileUtil.readFileToString(new File(args[0])); + } catch (Exception f) + { + f.printStackTrace(); + return; + } + } + String res = new hmmerClient().submitJackhmmerSearch(instr, + "jackhmmer", "pdb", 5); + if (res == null) + { + throw new Error("Failed."); + } + System.out.println("Result\n" + res); + return; + } + + /** + * + * @param input + * - fasta or other formatted sequence or alignment + * @param algo + * - jackhmmer + * @param db + * - pdb, uniprot, etc. + * @param niter + * number of iterations + * @return job id + */ + String submitJackhmmerSearch(String input, String algo, String db, + int niter) + { + JSONObject inparam = new JSONObject(); + HttpPost jackhp = new HttpPost(baseUrl + jackH); + String lastiter = null; + try + { + inparam.put("algo", algo); + inparam.put("seq", input); + inparam.put("seqdb", db); + inparam.put("iterations", niter); + // #Now POST the request and generate the search job. + // dumb json post service + jackhp.setHeader("content-type", "application/json"); + jackhp.setEntity(new StringEntity(inparam.toString())); + } catch (Exception f) + { + f.printStackTrace(); + return null; + } + HttpResponse r = null; + try + { + DefaultHttpClient httpCl = new DefaultHttpClient(); + + r = httpCl.execute(jackhp); + + } catch (Exception x) + { + System.err.println("Submit failed."); + x.printStackTrace(); + } + if (r.getStatusLine().getStatusCode() != 201) + { + throw new Error(r.toString()); + } + // get uid for job + String jobid = null, redir = null; + try + { + JSONObject res = new JSONObject(EntityUtils.toString(r.getEntity())); + jobid = res.getString("job_id"); + + Header[] loc; + if ((loc = r.getHeaders(HTTPConstants.HEADER_LOCATION)) != null + && loc.length > 0) + { + if (loc.length > 1) + { + System.err + .println("Ignoring additional " + + (loc.length - 1) + + " location(s) provided in response header ( next one is '" + + loc[1].getValue() + "' )"); + } + redir = loc[0].getValue(); + } + } catch (Exception x) + { + System.err.println("job id extraction failed."); + x.printStackTrace(); + } + int tries = 0; + boolean finished = false; + JSONObject jobstate = null; + do + { + try + { + DefaultHttpClient httpCl = new DefaultHttpClient(); + + HttpGet jackcheck = new HttpGet(redir); + jackcheck.setHeader("content-type", "application/json"); + r = httpCl.execute(jackcheck); + switch (r.getStatusLine().getStatusCode()) + { + case 200: + jobstate = new JSONObject(EntityUtils.toString(r.getEntity())); + String st = jobstate.getString("status"); + if ("DONE".equals(st)) + { + finished = true; + } + if ("ERROR".equals(st)) + { + System.err.println("Error"); + finished = true; + } + if ("PEND".equals(st) || "RUN".equals("st")) + { + JSONArray iters = jobstate.getJSONArray("result"); + lastiter = iters.getJSONObject(iters.length() - 1).getString( + "uuid"); + if (lastiter.length() > 0) + { + java.util.regex.Pattern p = java.util.regex.Pattern + .compile(".+(\\d+)"); + Matcher m = p.matcher(lastiter); + if (m.matches()) + { + System.out.println("On iteration " + m.group(1)); + } + } + } + break; + + default: + tries++; + Thread.sleep(2000); + } + } catch (Exception q) + { + q.printStackTrace(); + return null; + } + } while (!finished && tries < 50); + + if (!finished) + { + System.err.println("Giving up with job " + jobid + " at " + redir); + return null; + } + // get results + // http://www.ebi.ac.uk/Tools/hmmer/download/60048B38-7CEC-11E5-A230-CED6D26C98AD.5/score?format=csv + // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 1 2 4.4e-46 2.1e-43 + // 151.758316040039 0.04 11 151 3 139 1 150 0.94 GROWTH FACTOR BOUND PROTEIN + // 2 1cj1_J 1gri_B + // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 2 2 1.6e-17 7.9e-15 + // 58.8796501159668 0.01 7 66 157 215 153 216 0.95 GROWTH FACTOR BOUND + // PROTEIN 2 1cj1_J 1gri_B + // 4h1o_A 4h1o_A 560 jackhmmer - 163 2.1e-57 197.3 0.0 1 2 7.5e-28 3.6e-25 + // 92.4921493530273 0.00 65 161 20 122 17 124 0.95 Tyrosine-protein + // phosphatase non-receptor typ 4h1o_A + // + // 4h1o_A 4h1o_A 560 jackhmmer - 163 2.1e-57 197.3 0.0 2 2 7.6e-31 3.7e-28 + // 102.219146728516 0.03 66 161 127 236 124 238 0.94 Tyrosine-protein + // phosphatase non-receptor typ 4h1o_A + // + // $ua->get( $rootUrl."/results/".$lastIteration->{uuid} . "/score" + return lastiter; + /* + * * #Job should have finished, but we may have converged, so get the last + * job. my $results = $json->decode( $response->content ); my $lastIteration + * = pop( @{ $results->{result} } ); #Now fetch the results of the last + * iteration my $searchResult = $ua->get( $rootUrl."/results/" . + * $lastIteration->{uuid} . "/score", 'Accept' => 'application/json' ); + * unless( $searchResult->status_line eq "200 OK"){ die + * "Failed to get search results\n"; } + * + * #Decode the content of the full set of results $results = $json->decode( + * $searchResult->content ); print + * "Matched ".$results->{'results'}->{'stats'}->{'nincluded'}." sequences + * ($lastIteration->{uuid})!\n"; #Now do something more interesting with the + * results...... + */ + } + + /** + * retrieve an alignment annotated with scores from JackHmmer + * + * @param jobid + * @param dataset + * @return + */ + AlignmentI retrieveJackhmmerResult(String jobid, AlignmentI dataset) + throws OutOfMemoryError, IOException + { + AlignmentI searchResult = null; + + // get results + + searchResult = new AppletFormatAdapter().readFile(baseUrl + + "/download/" + jobid + "/score?format=afa&t=.gz", + DataSourceType.URL, FileFormat.Fasta); + + // TODO extract gapped columns as '.' - inserts to query profile + + // TODO match up jackhammer results to dataset. + + // do scores + FileParse jsonsource = new FileParse(baseUrl + "/download/" + jobid + + "/score?format=json", DataSourceType.URL); + if (!jsonsource.isValid()) + { + throw new IOException("Couldn't access scores for Jackhammer results"); + } + readJackhmmerScores(searchResult, jsonsource); + return searchResult; + } + + /** + * // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 1 2 4.4e-46 2.1e-43 + * + * // 151.758316040039 0.04 11 151 3 139 1 150 0.94 GROWTH FACTOR BOUND + * PROTEIN // 2 1cj1_J 1gri_B // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 + * 212.4 0.1 2 2 1.6e-17 7.9e-15 // 58.8796501159668 0.01 7 66 157 215 153 216 + * 0.95 GROWTH FACTOR BOUND // PROTEIN 2 1cj1_J 1gri_B // 4h1o_A 4h1o_A 560 + * jackhmmer - 163 2.1e-57 197.3 0.0 1 2 7.5e-28 3.6e-25 // 92.4921493530273 + * 0.00 65 161 20 122 17 124 0.95 Tyrosine-protein // phosphatase non-receptor + * typ 4h1o_A + */ + private static String[] _hmmsearchcols = new String[] { "acc", "name", "" }; + + private void readJackhmmerScores(AlignmentI searchResult, + FileParse jsonsource) throws IOException, OutOfMemoryError + { + HmmerJSONProcessor hjp = new HmmerJSONProcessor(searchResult); + hjp.parseFrom(jsonsource); + + // http://www.ebi.ac.uk/Tools/hmmer/download/60048B38-7CEC-11E5-A230-CED6D26C98AD.5/score?format=csv + // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 1 2 4.4e-46 2.1e-43 + // 151.758316040039 0.04 11 151 3 139 1 150 0.94 GROWTH FACTOR BOUND PROTEIN + // 2 1cj1_J 1gri_B + // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 2 2 1.6e-17 7.9e-15 + // 58.8796501159668 0.01 7 66 157 215 153 216 0.95 GROWTH FACTOR BOUND + // PROTEIN 2 1cj1_J 1gri_B + // 4h1o_A 4h1o_A 560 jackhmmer - 163 2.1e-57 197.3 0.0 1 2 7.5e-28 3.6e-25 + // 92.4921493530273 0.00 65 161 20 122 17 124 0.95 Tyrosine-protein + // phosphatase non-receptor typ 4h1o_A + // each line scores a fragment + // so for a combined score ? + + /** + * for a sequence q sort any t against q according to overallScore(q,t) + * maxFragment(q,t) in sequence features parlance: for alignment + * s.getFeature("overallScore",q) -> range on q and range on s + * + * + */ + + // 151.758316040039 0.04 11 151 3 139 1 150 0.94 GROWTH FACTOR BOUND PROTEIN + // 2 1cj1_J 1gri_B + // 1gri_A 1gri_A 217 jackhmmer - 163 4.7e-62 212.4 0.1 2 2 1.6e-17 7.9e-15 + // 58.8796501159668 0.01 7 66 157 215 153 216 0.95 GROWTH FACTOR BOUND + // PROTEIN 2 1cj1_J 1gri_B + // 4h1o_A 4h1o_A 560 jackhmmer - 163 2.1e-57 197.3 0.0 1 2 7.5e-28 3.6e-25 + // 92.4921493530273 0.00 65 161 20 122 17 124 0.95 Tyrosine-protein + // phosphatase non-receptor typ 4h1o_A + // + // 4h1o_A 4h1o_A 560 jackhmmer - 163 2.1e-57 197.3 0.0 2 2 7.6e-31 3.7e-28 + // 102.219146728516 0.03 66 161 127 236 124 238 0.94 Tyrosine-protein + // phosphatase non-receptor typ 4h1o_A + + } + +} diff --git a/src/jalview/ws/rest/RestClient.java b/src/jalview/ws/rest/RestClient.java index a71b70d..b04030d 100644 --- a/src/jalview/ws/rest/RestClient.java +++ b/src/jalview/ws/rest/RestClient.java @@ -28,15 +28,14 @@ import jalview.gui.AlignmentPanel; import jalview.gui.Desktop; import jalview.gui.JvOptionPane; import jalview.gui.WebserviceInfo; -import jalview.io.packed.DataProvider.JvDataType; import jalview.util.MessageManager; import jalview.ws.WSClient; import jalview.ws.WSClientI; import jalview.ws.WSMenuEntryProviderI; +import jalview.ws.rest.clientdefs.ShmrRestClient; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.util.Hashtable; import java.util.Vector; import javax.swing.JMenu; @@ -339,46 +338,6 @@ public class RestClient extends WSClient } } - public static RestClient makeShmmrRestClient() - { - String action = "Analysis", - description = "Sequence Harmony and Multi-Relief (Brandt et al. 2010)", - name = MessageManager.getString("label.multiharmony"); - Hashtable iparams = new Hashtable(); - jalview.ws.rest.params.JobConstant toolp; - // toolp = new jalview.ws.rest.JobConstant("tool","jalview"); - // iparams.put(toolp.token, toolp); - // toolp = new jalview.ws.rest.params.JobConstant("mbjob[method]","shmr"); - // iparams.put(toolp.token, toolp); - // toolp = new - // jalview.ws.rest.params.JobConstant("mbjob[description]","step 1"); - // iparams.put(toolp.token, toolp); - // toolp = new jalview.ws.rest.params.JobConstant("start_search","1"); - // iparams.put(toolp.token, toolp); - // toolp = new jalview.ws.rest.params.JobConstant("blast","0"); - // iparams.put(toolp.token, toolp); - - jalview.ws.rest.params.Alignment aliinput = new jalview.ws.rest.params.Alignment(); - // SHMR server has a 65K limit for content pasted into the 'ali' parameter, - // so we always upload our files. - aliinput.token = "ali_file"; - aliinput.writeAsFile = true; - iparams.put(aliinput.token, aliinput); - jalview.ws.rest.params.SeqGroupIndexVector sgroups = new jalview.ws.rest.params.SeqGroupIndexVector(); - sgroups.setMinsize(2); - sgroups.min = 2;// need at least two group defined to make a partition - iparams.put("groups", sgroups); - sgroups.token = "groups"; - sgroups.sep = " "; - RestServiceDescription shmrService = new RestServiceDescription(action, - description, name, - "http://zeus.few.vu.nl/programs/shmrwww/index.php?tool=jalview", // ?tool=jalview&mbjob[method]=shmr&mbjob[description]=step1", - "?tool=jalview", iparams, true, false, '-'); - // a priori knowledge of the data returned from the service - shmrService.addResultDatatype(JvDataType.ANNOTATION); - return new RestClient(shmrService); - } - public AlignmentPanel recoverAlignPanelForView() { AlignmentPanel[] aps = Desktop @@ -411,9 +370,9 @@ public class RestClient extends WSClient try { for (RestServiceDescription descr : RestServiceDescription - .parseDescriptions( - jalview.bin.Cache.getDefault(RSBS_SERVICES, - makeShmmrRestClient().service.toString()))) + .parseDescriptions(jalview.bin.Cache.getDefault( + RSBS_SERVICES, + ShmrRestClient.makeShmmrRestClient().service.toString()))) { services.add(descr.toString()); } diff --git a/src/jalview/ws/rest/RestJobThread.java b/src/jalview/ws/rest/RestJobThread.java index acb7904..e397c79 100644 --- a/src/jalview/ws/rest/RestJobThread.java +++ b/src/jalview/ws/rest/RestJobThread.java @@ -59,8 +59,6 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; public class RestJobThread extends AWSThread @@ -216,9 +214,6 @@ public class RestJobThread extends AWSThread protected void doHttpReq(Stage stg, RestJob rj, String postUrl) throws Exception { - StringBuffer respText = new StringBuffer(); - // con.setContentHandlerFactory(new - // jalview.ws.io.mime.HttpContentHandler()); HttpRequestBase request = null; String messages = ""; if (stg == Stage.SUBMIT) @@ -253,7 +248,6 @@ public class RestJobThread extends AWSThread { DefaultHttpClient httpclient = new DefaultHttpClient(); - HttpContext localContext = new BasicHttpContext(); HttpResponse response = null; try { @@ -287,39 +281,21 @@ public class RestJobThread extends AWSThread Cache.log.debug("Processing result set."); processResultSet(rj, response, request); break; + case 202: - rj.statMessage = "
Job submitted successfully. Results available at this URL:\n" - + "" + rj.getJobId() - + "
"; - rj.running = true; + markJobAsRunning(rj); break; + + case 201: + // Created - redirect may be present. Fallthrough to 302 case 302: - Header[] loc; - if (!rj.isSubmitted() - && (loc = response - .getHeaders(HTTPConstants.HEADER_LOCATION)) != null - && loc.length > 0) - { - if (loc.length > 1) - { - Cache.log.warn("Ignoring additional " + (loc.length - 1) - + " location(s) provided in response header ( next one is '" - + loc[1].getValue() + "' )"); - } - rj.setJobId(loc[0].getValue()); - rj.setSubmitted(true); - } + extractJobId(rj, response); completeStatus(rj, response); break; case 500: - // Failed. - rj.setSubmitted(true); - rj.setAllowedServerExceptions(0); - rj.setSubjobComplete(true); - rj.error = true; - rj.running = false; - completeStatus(rj, response, - "" + getStage(stg) + "failed. Reason below:\n"); + markAsFailed(rj, response); + completeStatus(rj, response, "" + getStage(stg) + + "failed. Reason below:\n"); break; default: // Some other response. Probably need to pop up the content in a window. @@ -346,6 +322,64 @@ public class RestJobThread extends AWSThread } } + private void markAsFailed(RestJob rj, HttpResponse response) + { + // Failed. + rj.setSubmitted(true); + rj.setAllowedServerExceptions(0); + rj.setSubjobComplete(true); + rj.error = true; + rj.running = false; + } + + /** + * set the jobRunning flag and post a link to the physical result page encoded + * in rj.getJobId() + * + * @param rj + */ + private void markJobAsRunning(RestJob rj) + { + rj.statMessage = "
Job submitted successfully. Results available at this URL:\n" + + "" + + rj.getJobId() + + "
"; + rj.running = true; + } + + /** + * extract the job ID URL from the redirect page. Does nothing if job is + * already running. + * + * @param rj + * @param response + */ + private void extractJobId(RestJob rj, HttpResponse response) + { + Header[] loc; + if (!rj.isSubmitted()) + { + + // redirect URL - typical for IBIVU type jobs. + if ((loc = response.getHeaders(HTTPConstants.HEADER_LOCATION)) != null + && loc.length > 0) + { + if (loc.length > 1) + { + Cache.log + .warn("Ignoring additional " + + (loc.length - 1) + + " location(s) provided in response header ( next one is '" + + loc[1].getValue() + "' )"); + } + rj.setJobId(loc[0].getValue()); + rj.setSubmitted(true); + } + } + } + /** * job has completed. Something valid should be available from con * diff --git a/src/jalview/ws/rest/clientdefs/ShmrRestClient.java b/src/jalview/ws/rest/clientdefs/ShmrRestClient.java new file mode 100644 index 0000000..a10931b --- /dev/null +++ b/src/jalview/ws/rest/clientdefs/ShmrRestClient.java @@ -0,0 +1,58 @@ +package jalview.ws.rest.clientdefs; + +import jalview.io.packed.DataProvider.JvDataType; +import jalview.util.MessageManager; +import jalview.ws.rest.InputType; +import jalview.ws.rest.RestClient; +import jalview.ws.rest.RestServiceDescription; +import jalview.ws.rest.params.Alignment; +import jalview.ws.rest.params.JobConstant; +import jalview.ws.rest.params.SeqGroupIndexVector; + +import java.util.Hashtable; + +public class ShmrRestClient +{ + + public static RestClient makeShmmrRestClient() + { + String action = "Analysis", description = "Sequence Harmony and Multi-Relief (Brandt et al. 2010)", name = MessageManager + .getString("label.multiharmony"); + Hashtable iparams = new Hashtable(); + jalview.ws.rest.params.JobConstant toolp; + // toolp = new jalview.ws.rest.JobConstant("tool","jalview"); + // iparams.put(toolp.token, toolp); + // toolp = new jalview.ws.rest.params.JobConstant("mbjob[method]","shmr"); + // iparams.put(toolp.token, toolp); + // toolp = new + // jalview.ws.rest.params.JobConstant("mbjob[description]","step 1"); + // iparams.put(toolp.token, toolp); + // toolp = new jalview.ws.rest.params.JobConstant("start_search","1"); + // iparams.put(toolp.token, toolp); + // toolp = new jalview.ws.rest.params.JobConstant("blast","0"); + // iparams.put(toolp.token, toolp); + + jalview.ws.rest.params.Alignment aliinput = new jalview.ws.rest.params.Alignment(); + // SHMR server has a 65K limit for content pasted into the 'ali' parameter, + // so we always upload our files. + aliinput.token = "ali_file"; + aliinput.writeAsFile = true; + iparams.put(aliinput.token, aliinput); + jalview.ws.rest.params.SeqGroupIndexVector sgroups = new jalview.ws.rest.params.SeqGroupIndexVector(); + sgroups.setMinsize(2); + sgroups.min = 2;// need at least two group defined to make a partition + iparams.put("groups", sgroups); + sgroups.token = "groups"; + sgroups.sep = " "; + RestServiceDescription shmrService = new RestServiceDescription( + action, + description, + name, + "http://zeus.few.vu.nl/programs/shmrwww/index.php?tool=jalview",// ?tool=jalview&mbjob[method]=shmr&mbjob[description]=step1", + "?tool=jalview", iparams, true, false, '-'); + // a priori knowledge of the data returned from the service + shmrService.addResultDatatype(JvDataType.ANNOTATION); + return new RestClient(shmrService); + } + +} diff --git a/test/jalview/ws/ebi/HmmerJSONProcessTest.java b/test/jalview/ws/ebi/HmmerJSONProcessTest.java new file mode 100644 index 0000000..bf68906 --- /dev/null +++ b/test/jalview/ws/ebi/HmmerJSONProcessTest.java @@ -0,0 +1,122 @@ +package jalview.ws.ebi; + +import jalview.datamodel.AlignmentAnnotation; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.SequenceI; +import jalview.io.DataSourceType; +import jalview.io.FileFormat; +import jalview.io.FileParse; +import jalview.io.FormatAdapter; + +import java.io.File; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class HmmerJSONProcessTest { + public static File alignmentFragFile = new File( + "examples/testdata/hmmer3/alignment_frag.fa.gz"); + + private AlignmentI getSearchResultFragmentAlignment() throws Exception + { + AlignmentI alf = new FormatAdapter().readFile( + alignmentFragFile.getAbsolutePath(), DataSourceType.FILE, + FileFormat.Fasta); + + return alf; + } + + public static File alignmentResultFile = new File( + "examples/testdata/hmmer3/alignment_res.fa.gz"); + + private AlignmentI getSearchResultAlignment() throws Exception + { + AlignmentI alf = new FormatAdapter().readFile( + alignmentResultFile.getAbsolutePath(), DataSourceType.FILE, + FileFormat.Fasta); + + return alf; + } + + public static String hitTestFile = "examples/testdata/hmmer3/hit_fragment.json.gz", + hmmerResultFile = "examples/testdata/hmmer3/hmmeresult.json.gz"; + + + @Test(groups = { "Functional" }) + public void parseHitTest() throws Exception + { + + Assert.assertTrue(new File(hitTestFile).exists(), + "Can't find test data.\n" + + hitTestFile); + JSONParser jp = new JSONParser(); + // read JSON in same way - via fileparse + Object hitfragment = jp.parse(new FileParse(hitTestFile, + DataSourceType.FILE).getReader()); + Assert.assertTrue((hitfragment instanceof JSONObject), + "Didn't find a JSON object map in " + hitTestFile); + AlignmentI searchResult = getSearchResultFragmentAlignment(); + + Assert.assertTrue(searchResult != null && searchResult.getHeight() > 0, + "Didn't read search result alignment from " + alignmentFragFile); + + HmmerJSONProcessor hjsp = new HmmerJSONProcessor(searchResult); + hjsp.addHit((JSONObject) hitfragment, 1); + // check that + // scores, posterior probabilities and stuff exist. + } + + @Test(groups = { "Functional" }) + public void parseJsonResultTest() throws Exception + { + + Assert.assertTrue(new File(hmmerResultFile).exists(), + "Can't find test data.\n" + hmmerResultFile); + + AlignmentI searchResult = getSearchResultAlignment(); + + Assert.assertTrue(searchResult != null && searchResult.getHeight() > 0, + "Didn't read search result alignment from " + alignmentFragFile); + + HmmerJSONProcessor hjsp = new HmmerJSONProcessor(searchResult); + hjsp.parseFrom(new FileParse(hmmerResultFile, DataSourceType.FILE)); + AlignmentAnnotation[] aa = searchResult.getSequenceAt(5) + .getAnnotation(); + Assert.assertNotNull(aa); + Assert.assertEquals(aa.length, 3, + "didn't get expected set of annotation.\n"); + // DPTSERWFHGHLSGKEAEKLLTeKGKHGSFLVRESQSHPGDFVLSVRTgddkgesndgKSKVTHVMIR-CQELKYDVGGGERFDSLTDLVEHYKKNPmvet + // LGTVLQLKQP + // 5789*****************9799***********************999998888888********.99**************************9999 + // 899999*999 + // AlignmentAnnotation + // 101 == 8 + String seq = "tLGT"; + SequenceI s5 = searchResult.getSequenceAt(5); + Assert.assertEquals( + s5.getSubSequence(s5.findIndex(225), s5.findIndex(229)) + .getSequenceAsString(), + seq); + int pos = s5.findIndex(226); + for (AlignmentAnnotation an : aa) + { + if (an.label.startsWith("Posterior")) + { + Assert.assertEquals(an.annotations[pos].value, 8f); + + } + } + ; + // check that + // scores, posterior probabilities and stuff exist. + } + // Groovy test + // def al = Jalview.getAlignFrames()[0].getViewport().getAlignment() + // def jproc = new jalview.ws.ebi.HmmerJSONProcessor(al) + // jproc.parseFrom(new + // jalview.io.FileParse("examples/testdata/hmmer3/hmmeresult.json.gz","File")) + // jproc.updateView(Jalview.getAlignFrames()[0].getViewport()) + +} diff --git a/test/jalview/ws/rest/ShmmrRSBSService.java b/test/jalview/ws/rest/ShmmrRSBSService.java index 709f2c5..61ad91f 100644 --- a/test/jalview/ws/rest/ShmmrRSBSService.java +++ b/test/jalview/ws/rest/ShmmrRSBSService.java @@ -25,6 +25,7 @@ import static org.testng.AssertJUnit.assertTrue; import jalview.gui.AlignFrame; import jalview.gui.JvOptionPane; +import jalview.ws.rest.clientdefs.ShmrRestClient; import java.util.Map; @@ -52,13 +53,13 @@ public class ShmmrRSBSService assertTrue( "Test Rsd Exchange using using default Shmmr service failed.", testRsdExchange("Test using default Shmmr service", - RestClient.makeShmmrRestClient().service)); + ShmrRestClient.makeShmmrRestClient().service)); } @Test(groups = { "Functional" }) public void testShmmrServiceDataprep() throws Exception { - RestClient _rc = RestClient.makeShmmrRestClient(); + RestClient _rc = ShmrRestClient.makeShmmrRestClient(); assertNotNull(_rc); AlignFrame alf = new jalview.io.FileLoader(false) .LoadFileWaitTillLoaded("examples/testdata/smad.fa",