From: jprocter Date: Fri, 17 Dec 2010 16:53:32 +0000 (+0000) Subject: JAL-715 and JAL-724 X-Git-Tag: Release_2_7~333 X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=d57c42c2998cafdf8f6874a3b85aa61823819f7d;p=jalview.git JAL-715 and JAL-724 --- diff --git a/src/jalview/ws/rest/RestJobThread.java b/src/jalview/ws/rest/RestJobThread.java index c100a72..581af9a 100644 --- a/src/jalview/ws/rest/RestJobThread.java +++ b/src/jalview/ws/rest/RestJobThread.java @@ -1,26 +1,32 @@ package jalview.ws.rest; import jalview.bin.Cache; +import jalview.datamodel.Alignment; +import jalview.datamodel.AlignmentAnnotation; import jalview.datamodel.AlignmentI; +import jalview.datamodel.AlignmentOrder; +import jalview.datamodel.Annotation; import jalview.datamodel.ColumnSelection; +import jalview.datamodel.SequenceGroup; +import jalview.datamodel.SequenceI; import jalview.gui.AlignFrame; +import jalview.gui.Desktop; import jalview.gui.WebserviceInfo; -import jalview.io.packed.DataProvider; +import jalview.io.NewickFile; import jalview.io.packed.JalviewDataset; import jalview.io.packed.JalviewDataset.AlignmentSet; -import jalview.io.packed.ParsePackedSet; -import jalview.io.packed.SimpleDataProvider; -import jalview.io.packed.DataProvider.JvDataType; import jalview.ws.AWSThread; import jalview.ws.AWsJob; -import java.awt.Desktop; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.io.IOException; import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Hashtable; import java.util.List; import java.util.Map.Entry; +import java.util.Vector; import org.apache.axis.transport.http.HTTPConstants; import org.apache.http.Header; @@ -491,6 +497,27 @@ public class RestJobThread extends AWSThread } } + /** + * instructions for whether to create new alignment view on current alignment + * set, add to current set, or create new alignFrame + */ + private enum AddDataTo + { + /** + * add annotation, trees etc to current view + */ + currentView, + /** + * create a new view derived from current view and add data to that + */ + newView, + /** + * create a new alignment frame for the result set and add annotation to + * that. + */ + newAlignment + }; + public void realiseResults(boolean merge) { /* @@ -499,134 +526,658 @@ public class RestJobThread extends AWSThread * annotation/features. */ /** - * alignment panels derived from each alignment set returned by service. + * alignment panels derived from each alignment set returned by service. */ ArrayList destPanels = new ArrayList(); /** - * current pane being worked with + * list of instructions for how to process each distinct alignment set + * returned by the job set */ - jalview.gui.AlignmentPanel destPanel; + ArrayList resultDest = new ArrayList(); /** * when false, zeroth pane is panel derived from input deta. */ boolean newAlignment = false; - if (merge) - { - if (!restClient.isAlignmentModified()) - { - destPanel = restClient.recoverAlignPanelForView(); - if (restClient.isShowResultsInNewView()) - { - destPanel = destPanel.alignFrame.newView(false); - } - // add the destination panel to frame zero of result panel set - destPanels.add(destPanel); - } - } - if (destPanels.size()==0) - { - Object[] idat = input.getAlignmentAndColumnSelection(restClient.av - .getGapCharacter()); - AlignFrame af = new AlignFrame((AlignmentI) idat[0], - (ColumnSelection) idat[1], AlignFrame.DEFAULT_WIDTH, - AlignFrame.DEFAULT_HEIGHT); - jalview.gui.Desktop.addInternalFrame(af, - "Results for " + restClient.service.details.Name + " " - + restClient.service.details.Action + " on " - + restClient.af.getTitle(), AlignFrame.DEFAULT_WIDTH, - AlignFrame.DEFAULT_HEIGHT); - destPanel = af.alignPanel; - // create totally new alignment from stashed data/results - newAlignment = true; - } - // Now process results, adding/creating new views as necessary. + /** + * gap character to be used for alignment reconstruction + */ + char gapCharacter = restClient.av.getGapCharacter(); + // Now, iterate over all alignment sets returned from all jobs: + // first inspect jobs and collate results data in order to count alignments + // and other results + // then assemble results from decomposed (v followed by h-separated) jobs + // finally, new views and alignments will be created and displayed as + // necessary. + boolean hsepjobs = restClient.service.isHseparable(); + boolean vsepjobs = restClient.service.isVseparable(); + // total number of distinct alignment sets generated by job set. + int numAlSets = 0, als = 0; + List destAls = new ArrayList(); + List destColsel = new ArrayList(); + List> trees = new ArrayList>(); + + do { - boolean hsepjobs = restClient.service.isHseparable(); - boolean vsepjobs = restClient.service.isVseparable(); - // total number of distinct alignment sets generated by job set. - int totalSets = 0, numAlSets = 0; + // Step 1. + // iterate over each alignment set returned from each subjob. Treating + // each one returned in parallel with others. + // Result collation arrays + + /** + * mapping between index of sequence in alignment that was submitted to + * server and index of sequence in the input alignment + */ + int[][] ordermap = new int[jobs.length][]; + SequenceI[][] rseqs = new SequenceI[jobs.length][]; + AlignmentOrder[] orders = new AlignmentOrder[jobs.length]; + AlignmentAnnotation[][] alan = new AlignmentAnnotation[jobs.length][]; + SequenceGroup[][] sgrp = new SequenceGroup[jobs.length][]; + // Now collect all data for alignment Set als from job array for (int j = 0; j < jobs.length; j++) { RestJob rj = (RestJob) jobs[j]; if (rj.hasResults()) { JalviewDataset rset = rj.context; - numAlSets = rset.hasAlignments() ? 0 : rset.getAl().size(); - if (numAlSets > 0) + if (rset.hasAlignments()) + { + if (numAlSets < rset.getAl().size()) + { + numAlSets = rset.getAl().size(); + } + if (als < rset.getAl().size() + && rset.getAl().get(als).isModified()) + { + // Collate result data + // TODO: decide if all alignmentI should be collected rather than + // specific alignment data containers + // for moment, we just extract content, but this means any + // alignment properties may be lost. + AlignmentSet alset = rset.getAl().get(als); + alan[j] = alset.al.getAlignmentAnnotation(); + if (alset.al.getGroups() != null) + { + sgrp[j] = new SequenceGroup[alset.al.getGroups().size()]; + alset.al.getGroups().toArray(sgrp[j]); + } + else + { + sgrp[j] = null; + } + orders[j] = new AlignmentOrder(alset.al); + rseqs[j] = alset.al.getSequencesArray(); + ordermap[j] = rj.getOrderMap(); + // if (rj.isInputUniquified()) { + // jalview.analysis.AlignmentSorter.recoverOrder(rseqs[als]); + // } + + if (alset.trees != null) + { + trees.add(new ArrayList(alset.trees)); + } + else + { + trees.add(new ArrayList()); + } + } + } + } + } + // Now aggregate and present results from this frame of alignment data. + int nvertsep = 0, nvertseps = 1; + if (vsepjobs) + { + // Jobs relate to different rows of input alignment. + // Jobs are subdivided by rows before columns, + // so there will now be a number of subjobs according tohsep for each + // vertsep + // TODO: get vertical separation intervals for each job and set + // nvertseps + // TODO: merge data from each group/sequence onto whole + // alignment + } + /** + * index into rest jobs subdivided vertically + */ + int vrestjob = 0; + // Destination alignments for all result data. + ArrayList visgrps = new ArrayList(); + Hashtable groupNames = new Hashtable(); + ArrayList visAlAn = null; + for (nvertsep = 0; nvertsep < nvertseps; nvertsep++) + { + // TODO: set scope w.r.t. original alignment view for vertical + // separation. + { + // results for a job exclude hidden columns of input data, so map + // back on to all visible regions + /** + * rest job result we are working with + */ + int nrj = vrestjob; + + RestJob rj = (RestJob) jobs[nrj]; + int contigs[] = input.getVisibleContigs(); + AlignmentI destAl = null; + jalview.datamodel.ColumnSelection destCs = null; + // Resolve destAl for this data. + if (als == 0 && rj.isInputContextModified()) + { + // special case: transfer features, annotation, groups, etc, + // from input + // context to align panel derived from input data + if (destAls.size() > als) + { + destAl = destAls.get(als); + } + else + { + if (!restClient.isAlignmentModified() && merge) + { + destAl = restClient.av.getAlignment(); + destCs = restClient.av.getColumnSelection(); + resultDest + .add(restClient.isShowResultsInNewView() ? AddDataTo.newView + : AddDataTo.currentView); + destPanels.add(restClient.recoverAlignPanelForView()); + } + else + { + newAlignment = true; + // recreate the input alignment data + Object[] idat = input + .getAlignmentAndColumnSelection(gapCharacter); + destAl = new Alignment((SequenceI[]) idat[0]); + destCs = (ColumnSelection) idat[1]; + resultDest.add(AddDataTo.newAlignment); + // but do not add to the alignment panel list - since we need to + // create a whole new alignFrame set. + } + destAls.add(destAl); + destColsel.add(destCs); + } + } + else { - for (int als = 0; als < numAlSets; als++) + // alignment(s) returned by service is to be re-integrated and + // displayed + if (destAls.size() > als) { - // gather data from context - if (vsepjobs) + if (!vsepjobs) { - // todo: merge data from each group/sequence onto whole - // alignment + // TODO: decide if multiple multiple alignments returned by + // non-vseparable services are allowed. + Cache.log + .warn("dealing with multiple alignment products returned by non-vertically separable service."); + } + // recover reference to last alignment created for this rest frame + // ready for extension + destAl = destAls.get(als); + destCs = destColsel.get(als); + } + else + { + Object[] newview; + + if (!hsepjobs) + { + // single alignment for any job that gets mapped back on to + // input data. Reconstruct by interleaving parts of returned + // alignment with hidden parts of input data. + SequenceI[][] nsq = splitSeqsOnVisibleContigs(rseqs[nrj], + contigs, gapCharacter); + AlignmentOrder alo[] = new AlignmentOrder[nsq.length]; + for (int no = 0; no < alo.length; no++) + { + alo[no] = new AlignmentOrder(orders[nrj].getOrder()); + } + newview = input.getUpdatedView(nsq, orders, gapCharacter); } else { - if (hsepjobs) + // each job maps to a single visible contig, and all need to be + // stitched back together. + // reconstruct using sub-region based MSA alignment construction + // mechanism + newview = input.getUpdatedView(rseqs, orders, gapCharacter); + } + destAl = new Alignment((SequenceI[]) newview[0]); + destCs = (ColumnSelection) newview[1]; + newAlignment = true; + // TODO create alignment from result data with propagated + // references. + destAls.add(destAl); + destColsel.add(destCs); + resultDest.add(AddDataTo.newAlignment); + throw new Error("Impl. Error! TODO: "); + } + } + /** + * save initial job in this set in case alignment is h-separable + */ + int initnrj = nrj; + // Now add in groups + for (int ncnt = 0; ncnt < contigs.length; ncnt += 2) + { + if (!hsepjobs) + { + // single alignment for any job that gets mapped back on to input + // data. + } + else + { + // each job maps to a single visible contig, and all need to be + // stitched back together. + if (ncnt > 0) + { + nrj++; + } + // TODO: apply options for group merging and annotation merging. + // If merging not supported, then either clear hashtables now or + // use them to rename the new annotation/groups for each contig if + // a conflict occurs. + } + if (sgrp[nrj] != null) + { + for (SequenceGroup sg : sgrp[nrj]) + { + boolean recovered = false; + SequenceGroup exsg = null; + if (sg.getName() != null) + { + exsg = groupNames.get(sg.getName()); + recovered = true; + } + if (exsg == null) { - // map single result back on to all visible region of original alignment - if (als==0 && rj.isInputContextModified()) + exsg = new SequenceGroup(sg); + groupNames.put(exsg.getName(), exsg); + visgrps.add(exsg); + exsg.setStartRes(sg.getStartRes() + contigs[ncnt]); + exsg.setEndRes(sg.getEndRes() + contigs[ncnt]); + } + // now replace any references from the result set with + // corresponding refs from alignment input set. + + // TODO: cope with recovering hidden sequences from + // resultContext + { + Vector sqs = sg.getSequences(null); + for (int sqsi = 0, iSize = sqs.size(); sqsi < iSize; sqsi++) + { + SequenceI oseq = (SequenceI) sqs.get(sqsi); + SequenceI nseq = getNewSeq(oseq, rseqs[nrj], + ordermap[nrj], destAl); + if (nseq != null) + { + if (!recovered) + { + exsg.deleteSequence(oseq, false); + } + exsg.addSequence(nseq, false); + } + else + { + Cache.log + .warn("Couldn't resolve original sequence for new sequence."); + } + } + if (sg.hasSeqrep()) { - // transfer features, annotation, groups, etc, from input context to align panel derived from input data - new jalview.datamodel.Alignment(new jalview.datamodel.SequenceI[] {null}).getAlignmentAnnotation(); + if (exsg.getSeqrep() == sg.getSeqrep()) + { + // lift over sequence rep reference too + SequenceI oseq = sg.getSeqrep(); + SequenceI nseq = getNewSeq(oseq, rseqs[nrj], + ordermap[nrj], destAl); + if (nseq != null) + { + exsg.setSeqrep(nseq); + } + } } - } - else + if (recovered) { - // map result onto visible contigs. - AlignmentSet alset = rset.getAl().get(als); - if (als>0 ) - if (als == 0) + // adjust boundaries of recovered group w.r.t. new group being + // merged on to original alignment. + int start = sg.getStartRes() + contigs[ncnt], end = sg + .getEndRes() + contigs[ncnt]; + if (start < exsg.getStartRes()) + { + exsg.setStartRes(start); + } + if (end > exsg.getEndRes()) { - - // alignment is added straight to + exsg.setEndRes(end); } } } } } + // reset job counter + nrj = initnrj; + // and finally add in annotation and any trees for each job + for (int ncnt = 0; ncnt < contigs.length; ncnt += 2) + { + if (!hsepjobs) + { + // single alignment for any job that gets mapped back on to input + // data. + } + else + { + // each job maps to a single visible contig, and all need to be + // stitched back together. + if (ncnt > 0) + { + nrj++; + } + } + + // merge alignmentAnnotation into one row + if (alan[nrj] != null) + { + for (int an = 0; an < alan[nrj].length; an++) + { + SequenceI sqass = null; + SequenceGroup grass = null; + if (alan[nrj][an].sequenceRef != null) + { + // TODO: ensure this relocates sequence reference to local + // context. + sqass = getNewSeq(alan[nrj][an].sequenceRef, rseqs[nrj], + ordermap[nrj], destAl); + } + if (alan[nrj][an].groupRef != null) + { + // TODO: verify relocate group reference to local context + // works correctly + grass = groupNames.get(alan[nrj][an].groupRef.getName()); + if (grass == null) + { + Cache.log + .error("Couldn't relocate group referemce for group " + + alan[nrj][an].groupRef.getName()); + } + } + if (visAlAn == null) + { + visAlAn = new ArrayList(); + } + AlignmentAnnotation visan = null; + for (AlignmentAnnotation v : visAlAn) + { + if (v.label != null + && v.label.equals(alan[nrj][an].label)) + { + visan = v; + } + } + if (visan == null) + { + visan = new AlignmentAnnotation(alan[nrj][an]); + // copy annotations, and wipe out/update refs. + visan.annotations = new Annotation[input.getWidth()]; + visan.groupRef = grass; + visan.sequenceRef = sqass; + visAlAn.add(visan); + } + // now copy local annotation data into correct position + System.arraycopy(alan[nrj][an].annotations, 0, + visan.annotations, contigs[ncnt], + alan[nrj][an].annotations.length); + + } + } + // Trees + if (trees.get(nrj).size() > 0) + { + for (NewickFile nf : trees.get(nrj)) + { + // TODO: process each newick file, lifting over sequence refs to + // current alignment, if necessary. + Cache.log + .error("Tree recovery from restjob not yet implemented."); + } + } + } + } + } // end of vseps loops. + if (visAlAn != null) + { + for (AlignmentAnnotation v : visAlAn) + { + destAls.get(als).addAnnotation(v); } - // transfer results onto panel + } + if (visgrps != null) + { + for (SequenceGroup sg : visgrps) + { + destAls.get(als).addGroup(sg); + } + } + } while (++als < numAlSets); + // Finally, assemble each new alignment, and create new gui components to + // present it. + /** + * current AlignFrame where results will go. + */ + AlignFrame destaf = restClient.recoverAlignFrameForView(); + /** + * current pane being worked with + */ + jalview.gui.AlignmentPanel destPanel = restClient + .recoverAlignPanelForView(); + als = 0; + for (AddDataTo action : resultDest) + { + AlignmentI destal; + ColumnSelection destcs; + String alTitle = restClient.service.details.Action + " using " + + restClient.service.details.Name + " on " + + " whatever you clicked on." + "(set " + als + ")"; + switch (action) + { + case newAlignment: + destal = destAls.get(als); + destcs = destColsel.get(als); + destaf = new AlignFrame(destal, destcs, AlignFrame.DEFAULT_WIDTH, + AlignFrame.DEFAULT_HEIGHT); + // todo transfer any feature settings and colouring + /* + * destaf.getFeatureRenderer().transferSettings(this.featureSettings); + * // update orders if (alorders.size() > 0) { if (alorders.size() == 1) + * { af.addSortByOrderMenuItem(WebServiceName + " Ordering", + * (AlignmentOrder) alorders.get(0)); } else { // construct a + * non-redundant ordering set Vector names = new Vector(); for (int i = + * 0, l = alorders.size(); i < l; i++) { String orderName = new + * String(" Region " + i); int j = i + 1; + * + * while (j < l) { if (((AlignmentOrder) alorders.get(i)) + * .equals(((AlignmentOrder) alorders.get(j)))) { alorders.remove(j); + * l--; orderName += "," + j; } else { j++; } } + * + * if (i == 0 && j == 1) { names.add(new String("")); } else { + * names.add(orderName); } } for (int i = 0, l = alorders.size(); i < l; + * i++) { af.addSortByOrderMenuItem( WebServiceName + ((String) + * names.get(i)) + " Ordering", (AlignmentOrder) alorders.get(i)); } } } + */ + Desktop.addInternalFrame(destaf, alTitle, AlignFrame.DEFAULT_WIDTH, + AlignFrame.DEFAULT_HEIGHT); + break; + case newView: + + break; + case currentView: + break; } - /** - * alignments. New alignments are added to dataset, and subsequently - * annotated/visualised accordingly. 1. New alignment frame created for - * new alignment. Decide if any vis settings should be inherited from old - * alignment frame (e.g. sequence ordering ?). 2. Subsequent data added to - * alignment as below: - */ - /** - * annotation update to original/newly created context alignment: 1. - * identify alignment where annotation is to be loaded onto. 2. Add - * annotation, excluding any duplicates. 3. Ensure annotation is visible - * on alignment - honouring ordering given by file. - */ - /** - * features updated to original or newly created context alignment: 1. - * Features are(or were already) added to dataset. 2. Feature settings - * modified to ensure all features are displayed - honouring any ordering - * given by result file. Consider merging action with the code used by the - * DAS fetcher to update alignment views with new info. - */ - /** - * Seq associated data files (PDB files). 1. locate seq association in - * current dataset/alignment context and add file as normal - keep handle - * of any created ref objects. 2. decide if new data should be displayed : - * PDB display: if alignment has PDB display already, should new pdb files - * be aligned to it ? - * - */ + } + if (!newAlignment) + { + if (restClient.isShowResultsInNewView()) + { + // destPanel = destPanel.alignFrame.newView(false); + } + } + else + { } + /* + * if (als) // add the destination panel to frame zero of result panel set } + * } if (destPanels.size()==0) { AlignFrame af = new AlignFrame((AlignmentI) + * idat[0], (ColumnSelection) idat[1], AlignFrame.DEFAULT_WIDTH, + * AlignFrame.DEFAULT_HEIGHT); + * + * jalview.gui.Desktop.addInternalFrame(af, "Results for " + + * restClient.service.details.Name + " " + restClient.service.details.Action + * + " on " + restClient.af.getTitle(), AlignFrame.DEFAULT_WIDTH, + * AlignFrame.DEFAULT_HEIGHT); destPanel = af.alignPanel; // create totally + * new alignment from stashed data/results + */ + + /* + */ + + /** + * alignments. New alignments are added to dataset, and subsequently + * annotated/visualised accordingly. 1. New alignment frame created for new + * alignment. Decide if any vis settings should be inherited from old + * alignment frame (e.g. sequence ordering ?). 2. Subsequent data added to + * alignment as below: + */ + /** + * annotation update to original/newly created context alignment: 1. + * identify alignment where annotation is to be loaded onto. 2. Add + * annotation, excluding any duplicates. 3. Ensure annotation is visible on + * alignment - honouring ordering given by file. + */ + /** + * features updated to original or newly created context alignment: 1. + * Features are(or were already) added to dataset. 2. Feature settings + * modified to ensure all features are displayed - honouring any ordering + * given by result file. Consider merging action with the code used by the + * DAS fetcher to update alignment views with new info. + */ + /** + * Seq associated data files (PDB files). 1. locate seq association in + * current dataset/alignment context and add file as normal - keep handle of + * any created ref objects. 2. decide if new data should be displayed : PDB + * display: if alignment has PDB display already, should new pdb files be + * aligned to it ? + * + */ // destPanel.adjustAnnotationHeight(); } /** + * split the given array of sequences into blocks of subsequences + * corresponding to each visible contig + * + * @param sequenceIs + * @param contigs + * @param gapChar + * padding character for ragged ends of visible contig region. + * @return + */ + private SequenceI[][] splitSeqsOnVisibleContigs(SequenceI[] sequenceIs, + int[] contigs, char gapChar) + { + int nvc = contigs == null ? 1 : contigs.length / 2; + SequenceI[][] blocks = new SequenceI[nvc][]; + if (contigs == null) + { + blocks[0] = new SequenceI[sequenceIs.length]; + System.arraycopy(sequenceIs, 0, blocks[0], 0, sequenceIs.length); + } + else + { + // deja vu - have I written this before ?? propagateGaps does this in a + // way + char[] gapset = null; + int start = 0, width = 0; + for (int c = 0; c < nvc; c++) + { + width = contigs[c * 2 + 1] - contigs[c * 2] + 1; + for (int s = 0; s < sequenceIs.length; s++) + { + int end = sequenceIs[s].getLength(); + if (start < end) + { + if (start + width < end) + { + blocks[c][s] = sequenceIs[s].getSubSequence(start, start + + width); + } + else + { + blocks[c][s] = sequenceIs[s].getSubSequence(start, end); + String sq = blocks[c][s].getSequenceAsString(); + for (int n = start + width; n > end; n--) + { + sq += gapChar; + } + } + } + else + { + if (gapset == null || gapset.length < width) + { + char ng[] = new char[width]; + int gs = 0; + if (gapset != null) + { + System.arraycopy(gapset, 0, ng, 0, gs = gapset.length); + } + while (gs < ng.length) + { + ng[gs++] = gapChar; + } + } + blocks[c][s] = sequenceIs[s].getSubSequence(end, end); + blocks[c][s].setSequence(gapset.toString().substring(0, width)); + } + } + if (c > 0) + { + // adjust window for next visible segnment + start += contigs[c * 2 + 1] - contigs[c * 2]; + } + } + } + return blocks; + } + + /** + * recover corresponding sequence from original input data corresponding to + * sequence in a specific job's input data. + * + * @param oseq + * @param sequenceIs + * @param is + * @param destAl + * @return + */ + private SequenceI getNewSeq(SequenceI oseq, SequenceI[] sequenceIs, + int[] is, AlignmentI destAl) + { + int p = 0; + while (p < sequenceIs.length && sequenceIs[p] != oseq) + { + p++; + } + if (p < sequenceIs.length && p < destAl.getHeight()) + { + return destAl.getSequenceAt(is[p]); + } + return null; + } + + /** * * @return true if the run method is safe to call */