From 12513d239162ffe5d3e8ece01545634c09bd5050 Mon Sep 17 00:00:00 2001 From: Mateusz Warowny Date: Fri, 24 Sep 2021 15:49:24 +0200 Subject: [PATCH] JAL-3878 Display results of completed alignment job --- src/jalview/ws2/operations/AlignmentOperation.java | 583 ++++++++++++++++---- 1 file changed, 480 insertions(+), 103 deletions(-) diff --git a/src/jalview/ws2/operations/AlignmentOperation.java b/src/jalview/ws2/operations/AlignmentOperation.java index e3560bf..fc92f88 100644 --- a/src/jalview/ws2/operations/AlignmentOperation.java +++ b/src/jalview/ws2/operations/AlignmentOperation.java @@ -8,25 +8,31 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Hashtable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import javax.swing.JMenu; import javax.swing.JMenuItem; import javax.swing.ToolTipManager; -import org.codehaus.groovy.ast.GenericsType.GenericsTypeName; - +import jalview.analysis.AlignSeq; +import jalview.analysis.AlignmentSorter; +import jalview.analysis.SeqsetUtils; import jalview.bin.Cache; +import jalview.datamodel.AlignedCodonFrame; +import jalview.datamodel.Alignment; import jalview.datamodel.AlignmentI; import jalview.datamodel.AlignmentOrder; import jalview.datamodel.AlignmentView; +import jalview.datamodel.HiddenColumns; import jalview.datamodel.SequenceI; +import jalview.datamodel.Sequence; import jalview.gui.AlignFrame; +import jalview.gui.AlignViewport; import jalview.gui.JvSwingUtils; import jalview.gui.WebserviceInfo; import jalview.gui.WsJobParameters; @@ -52,58 +58,95 @@ import jalview.ws2.utils.WSJobList; public class AlignmentOperation implements Operation { final WebServiceI service; + final ResultSupplier supplier; - public AlignmentOperation(WebServiceI service, ResultSupplier supplier) { + public AlignmentOperation(WebServiceI service, + ResultSupplier supplier) + { this.service = service; this.supplier = supplier; } - @Override public int getMinSequences() { return 2; } - @Override public int getMaxSequences() { return Integer.MAX_VALUE; } - @Override public boolean isProteinOperation() { return true; } - @Override public boolean isNucleotideOperation() { return true; } - @Override public boolean canSubmitGaps() { + @Override + public int getMinSequences() + { + return 2; + } + + @Override + public int getMaxSequences() + { + return Integer.MAX_VALUE; + } + + @Override + public boolean isProteinOperation() + { + return true; + } + + @Override + public boolean isNucleotideOperation() + { + return true; + } + + @Override + public boolean canSubmitGaps() + { // hack copied from original jabaws code, don't blame me return service.getName().contains("lustal"); } - @Override public MenuEntryProviderI getMenuBuilder() { return this::buildMenu; } - protected void buildMenu(JMenu parent, AlignFrame frame) { - if (canSubmitGaps()) { + @Override + public MenuEntryProviderI getMenuBuilder() + { + return this::buildMenu; + } + + protected void buildMenu(JMenu parent, AlignFrame frame) + { + if (canSubmitGaps()) + { var alignSubmenu = new JMenu(service.getName()); buildMenu(alignSubmenu, frame, false); parent.add(alignSubmenu); var realignSubmenu = new JMenu(MessageManager.formatMessage( "label.realign_with_params", service.getName())); - realignSubmenu.setToolTipText(MessageManager.getString( - "label.align_sequences_to_existing_alignment")); + realignSubmenu.setToolTipText(MessageManager + .getString("label.align_sequences_to_existing_alignment")); buildMenu(realignSubmenu, frame, true); parent.add(realignSubmenu); } - else { + else + { buildMenu(parent, frame, false); } } - protected void buildMenu(JMenu parent, AlignFrame frame, boolean submitGaps) { + protected void buildMenu(JMenu parent, AlignFrame frame, + boolean submitGaps) + { final String action = submitGaps ? "Align" : "Realign"; final var calcName = service.getName(); final AlignmentView msa = frame.gatherSequencesForAlignment(); - final AlignmentI dataset = frame.getViewport().getAlignment().getDataset(); + final AlignViewport viewport = frame.getViewport(); + final AlignmentI alignment = frame.getViewport().getAlignment(); String title = frame.getTitle(); WebServiceExecutor executor = frame.getViewport().getWSExecutor(); { var item = new JMenuItem(MessageManager.formatMessage( - "label.calcname_with_default_settings", calcName)); - item.setToolTipText(MessageManager.formatMessage( - "label.action_with_default_settings", action)); + "label.calcname_with_default_settings", calcName)); + item.setToolTipText(MessageManager + .formatMessage("label.action_with_default_settings", action)); item.addActionListener((event) -> { if (msa != null) { - WebServiceWorkerI worker = new AlignmentWorker( - msa, Collections.emptyList(), title, submitGaps, true, dataset); + WebServiceWorkerI worker = new AlignmentWorker(msa, + Collections.emptyList(), title, submitGaps, true, + alignment, viewport); executor.submit(worker); } }); @@ -112,20 +155,23 @@ public class AlignmentOperation implements Operation if (service.hasParameters()) { - var item = new JMenuItem(MessageManager.getString("label.edit_settings_and_run")); + var item = new JMenuItem( + MessageManager.getString("label.edit_settings_and_run")); item.setToolTipText(MessageManager.getString( - "label.view_and_change_parameters_before_alignment")); + "label.view_and_change_parameters_before_alignment")); item.addActionListener((event) -> { if (msa != null) { - openEditParamsDialog(service, null, null).thenAcceptAsync((arguments) ->{ - if (arguments != null) - { - WebServiceWorkerI worker = new AlignmentWorker( - msa, arguments, title, submitGaps, true, dataset); - executor.submit(worker); - } - }); + openEditParamsDialog(service, null, null) + .thenAcceptAsync((arguments) -> { + if (arguments != null) + { + WebServiceWorkerI worker = new AlignmentWorker(msa, + arguments, title, submitGaps, true, alignment, + viewport); + executor.submit(worker); + } + }); } }); parent.add(item); @@ -134,35 +180,41 @@ public class AlignmentOperation implements Operation var presets = service.getParamStore().getPresets(); if (presets != null && presets.size() > 0) { - final var presetList = new JMenu(MessageManager.formatMessage( - "label.run_with_preset_params", calcName)); - final var showToolTipFor = ToolTipManager.sharedInstance().getDismissDelay(); + final var presetList = new JMenu(MessageManager + .formatMessage("label.run_with_preset_params", calcName)); + final var showToolTipFor = ToolTipManager.sharedInstance() + .getDismissDelay(); for (final var preset : presets) { var item = new JMenuItem(preset.getName()); final int QUICK_TOOLTIP = 1500; item.addMouseListener(new MouseAdapter() { - @Override public void mouseEntered(MouseEvent e) + @Override + public void mouseEntered(MouseEvent e) { ToolTipManager.sharedInstance().setDismissDelay(QUICK_TOOLTIP); } - @Override public void mouseExited(MouseEvent e) + + @Override + public void mouseExited(MouseEvent e) { ToolTipManager.sharedInstance().setDismissDelay(showToolTipFor); } }); - String tooltip = JvSwingUtils.wrapTooltip(true, format( - "%s
%s", - MessageManager.getString(preset.isModifiable() ? - "label.user_preset" : "label.service_preset"), - preset.getDescription())); + String tooltip = JvSwingUtils.wrapTooltip(true, + format("%s
%s", + MessageManager.getString( + preset.isModifiable() ? "label.user_preset" + : "label.service_preset"), + preset.getDescription())); item.setToolTipText(tooltip); item.addActionListener((event) -> { if (msa != null) { - WebServiceWorkerI worker = new AlignmentWorker( - msa, preset.getArguments(), title, submitGaps, true, dataset); + WebServiceWorkerI worker = new AlignmentWorker(msa, + preset.getArguments(), title, submitGaps, true, + alignment, viewport); executor.submit(worker); } }); @@ -172,26 +224,32 @@ public class AlignmentOperation implements Operation } } - - private CompletionStage> openEditParamsDialog(WebServiceI service, - WsParamSetI preset, List arguments) + private CompletionStage> openEditParamsDialog( + WebServiceI service, WsParamSetI preset, + List arguments) { WsJobParameters jobParams; if (preset == null && arguments != null && arguments.size() > 0) - jobParams = new WsJobParameters(service.getParamStore(), preset, arguments); + jobParams = new WsJobParameters(service.getParamStore(), preset, + arguments); else - jobParams = new WsJobParameters(service.getParamStore(), preset, null); + jobParams = new WsJobParameters(service.getParamStore(), preset, + null); var stage = jobParams.showRunDialog(); return stage.thenApply((startJob) -> { - if (startJob) { - if (jobParams.getPreset() == null) { + if (startJob) + { + if (jobParams.getPreset() == null) + { return jobParams.getJobParams(); } - else { + else + { return jobParams.getPreset().getArguments(); } } - else { + else + { return null; } }); @@ -204,29 +262,51 @@ public class AlignmentOperation implements Operation * @author mmwarowny * */ - private class AlignmentWorker implements WebServiceWorkerI { + private class AlignmentWorker implements WebServiceWorkerI + { private long uid = MathUtils.getUID(); + private final AlignmentView msa; - private final AlignmentI seqdata; + + private final AlignmentI dataset; + + private final List codonFrame = new ArrayList<>(); + private List args = Collections.emptyList(); + private String alnTitle = ""; + private boolean submitGaps = false; + private boolean preserveOrder = false; + + private char gapCharacter; + private WSJobList jobs = new WSJobList(); + + private Map inputs = new LinkedHashMap<>(); + private WebserviceInfo wsInfo; + private Map exceptionCount = new HashMap<>(); + private final int MAX_RETRY = 5; - AlignmentWorker(AlignmentView msa, List args, String alnTitle, - boolean submitGaps, boolean preserveOrder, AlignmentI seqdata) + AlignmentWorker(AlignmentView msa, List args, + String alnTitle, boolean submitGaps, boolean preserveOrder, + AlignmentI alignment, AlignViewport viewport) { this.msa = msa; - this.seqdata = seqdata; + this.dataset = alignment.getDataset(); + List cf = Objects.requireNonNullElse( + alignment.getCodonFrames(), Collections.emptyList()); + this.codonFrame.addAll(cf); this.args = args; this.alnTitle = alnTitle; this.submitGaps = submitGaps; this.preserveOrder = preserveOrder; + this.gapCharacter = viewport.getGapCharacter(); String panelInfo = String.format("%s using service hosted at %s%n%s", service.getName(), service.getHostName(), @@ -234,9 +314,17 @@ public class AlignmentOperation implements Operation wsInfo = new WebserviceInfo(service.getName(), panelInfo, false); } + @Override + public long getUID() + { + return uid; + } @Override - public long getUID() { return uid; } + public WebServiceI getWebService() + { + return service; + } @Override public List getJobs() @@ -250,20 +338,76 @@ public class AlignmentOperation implements Operation String outputHeader = String.format("%s of %s%nJob details%n", submitGaps ? "Re-alignment" : "Alignment", alnTitle); SequenceI[][] conmsa = msa.getVisibleContigs('-'); - if (conmsa == null) { + if (conmsa == null) + { return; } WebServiceInfoUpdater updater = new WebServiceInfoUpdater(wsInfo); updater.setOutputHeader(outputHeader); - for (int i = 0; i < conmsa.length; i++) { - WSJob job = service.submit(List.of(conmsa[i]), args); + int numValid = 0; + for (int i = 0; i < conmsa.length; i++) + { + JobInput input = JobInput.create(conmsa[i], 2, submitGaps); + WSJob job = new WSJob(service.getProviderName(), service.getName(), + service.getHostName()); job.setJobNum(wsInfo.addJobPane()); + if (conmsa.length > 0) + { + wsInfo.setProgressName(String.format("region %d", i), + job.getJobNum()); + } + wsInfo.setProgressText(job.getJobNum(), outputHeader); job.addPropertyChangeListener(updater); + inputs.put(job.getUid(), input); jobs.add(job); - if (conmsa.length > 0) { - wsInfo.setProgressName(String.format("region %d", i), job.getJobNum()); + if (input.isInputValid()) + { + int count; + String jobId = null; + do + { + count = exceptionCount.getOrDefault(job.getUid(), MAX_RETRY); + try + { + jobId = service.submit(input.inputSequences, args); + Cache.log.debug((format("Job %s submitted", job))); + exceptionCount.remove(job.getUid()); + } catch (IOException e) + { + exceptionCount.put(job.getUid(), --count); + } + } while (jobId == null && count > 0); + if (jobId != null) + { + job.setJobId(jobId); + job.setStatus(WSJobStatus.SUBMITTED); + numValid++; + } + else + { + job.setStatus(WSJobStatus.SERVER_ERROR); + } } - wsInfo.setProgressText(job.getJobNum(), outputHeader); + else + { + job.setStatus(WSJobStatus.INVALID); + job.setErrorLog( + MessageManager.getString("label.empty_alignment_job")); + } + } + if (numValid > 0) + { + // wsInfo.setThisService() should happen here + wsInfo.setVisible(true); + } + else + { + wsInfo.setVisible(false); + // TODO show notification dialog. + // JvOptionPane.showMessageDialog(frame, + // MessageManager.getString("info.invalid_msa_input_mininfo"), + // MessageManager.getString("info.invalid_msa_notenough"), + // JvOptionPane.INFORMATION_MESSAGE); } } @@ -271,19 +415,38 @@ public class AlignmentOperation implements Operation public boolean pollJobs() { boolean done = true; - for (WSJob job : getJobs()) { - if (!job.getStatus().isDone() ) { - try { + for (WSJob job : getJobs()) + { + if (!job.getStatus().isDone()) + { + Cache.log.debug(format("Polling job %s.", job)); + try + { service.updateProgress(job); exceptionCount.remove(job.getUid()); - } - catch (IOException e) { - int count = exceptionCount.getOrDefault(job.getUid(), MAX_RETRY); - if (--count <= 0) { + } catch (IOException e) + { + Cache.log.error(format("Polling job %s failed.", job), e); + wsInfo.appendProgressText(job.getJobNum(), + MessageManager.formatMessage("info.server_exception", + service.getName(), e.getMessage())); + int count = exceptionCount.getOrDefault(job.getUid(), + MAX_RETRY); + if (--count <= 0) + { job.setStatus(WSJobStatus.SERVER_ERROR); + Cache.log.warn(format( + "Attempts limit exceeded. Droping job %s.", job)); } exceptionCount.put(job.getUid(), count); + } catch (OutOfMemoryError e) + { + job.setStatus(WSJobStatus.BROKEN); + Cache.log.error( + format("Out of memory when retrieving job %s", job), e); } + Cache.log.debug( + format("Job %s status is %s", job, job.getStatus())); } done &= job.getStatus().isDone(); } @@ -291,42 +454,55 @@ public class AlignmentOperation implements Operation return done; } - private void updateWSInfoGlobalStatus() { - if (jobs.countRunning() > 0) { + private void updateWSInfoGlobalStatus() + { + if (jobs.countRunning() > 0) + { wsInfo.setStatus(WebserviceInfo.STATE_RUNNING); } - else if (jobs.countQueuing() > 0 || jobs.countSubmitted() < jobs.size()) { + else if (jobs.countQueuing() > 0 + || jobs.countSubmitted() < jobs.size()) + { wsInfo.setStatus(WebserviceInfo.STATE_QUEUING); } - else { - if (jobs.countSuccessful() > 0) { + else + { + if (jobs.countSuccessful() > 0) + { wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_OK); } - else if (jobs.countCancelled() > 0) { + else if (jobs.countCancelled() > 0) + { wsInfo.setStatus(WebserviceInfo.STATE_CANCELLED_OK); } - else if (jobs.countFailed() > 0) { + else if (jobs.countFailed() > 0) + { wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_ERROR); } } } @Override - public void done() { + public void done() + { long progbarId = MathUtils.getUID(); wsInfo.setProgressBar( MessageManager.getString("status.collecting_job_results"), progbarId); Map results = new LinkedHashMap<>(); - for (WSJob job : getJobs()) { - try { + for (WSJob job : getJobs()) + { + try + { AlignmentI alignment = supplier.getResult(job); - if (alignment != null) { + if (alignment != null) + { results.put(job.getUid(), alignment); } - } - catch (Exception e) { - if (!service.handleCollectionError(job, e)) { + } catch (Exception e) + { + if (!service.handleCollectionError(job, e)) + { Cache.log.error("Couldn't get alignment for job.", e); // TODO: Increment exception count and retry. job.setStatus(WSJobStatus.SERVER_ERROR); @@ -334,12 +510,14 @@ public class AlignmentOperation implements Operation } } updateWSInfoGlobalStatus(); - if (results.size() > 0) { - wsInfo.showResultsNewFrame.addActionListener( - evt -> displayResults(results)); + if (results.size() > 0) + { + wsInfo.showResultsNewFrame + .addActionListener(evt -> displayResults(results)); wsInfo.setResultsReady(); } - else { + else + { wsInfo.setFinishedNoResults(); } wsInfo.removeProgressBar(progbarId); @@ -350,31 +528,230 @@ public class AlignmentOperation implements Operation List alorders = new ArrayList<>(); SequenceI[][] results = new SequenceI[jobs.size()][]; AlignmentOrder[] orders = new AlignmentOrder[jobs.size()]; - for (int i = 0; i < jobs.size(); i++) { + for (int i = 0; i < jobs.size(); i++) + { WSJob job = jobs.get(i); AlignmentI aln = alignments.get(job.getUid()); - if (aln != null) { - /* - * Get the alignment including any empty sequences in the original - * order with original ids. - */ + if (aln != null) + { // equivalent of job.hasResults() + /* Get the alignment including any empty sequences in the original + * order with original ids. */ + JobInput input = inputs.get(job.getUid()); char gapChar = aln.getGapCharacter(); - int alSeqLen = aln.getSequences().size(); - SequenceI[] alSeqs = new SequenceI[alSeqLen]; - alSeqs = aln.getSequences().toArray(alSeqs); + List emptySeqs = input.emptySequences; + List alnSeqs = aln.getSequences(); + // find the width of the longest sequence + int width = 0; + for (var seq : alnSeqs) + width = Integer.max(width, seq.getLength()); + for (var emptySeq : emptySeqs) + width = Integer.max(width, emptySeq.getLength()); + // pad shorter sequences with gaps + String gapSeq = String.join("", + Collections.nCopies(width, Character.toString(gapChar))); + List seqs = new ArrayList<>( + alnSeqs.size() + emptySeqs.size()); + seqs.addAll(alnSeqs); + seqs.addAll(emptySeqs); + for (var seq : seqs) + { + if (seq.getLength() < width) + seq.setSequence(seq.getSequenceAsString() + + gapSeq.substring(seq.getLength())); + } + SequenceI[] result = seqs.toArray(new SequenceI[0]); + AlignmentOrder msaOrder = new AlignmentOrder(result); + AlignmentSorter.recoverOrder(result); + // temporary workaround for deuniquify + @SuppressWarnings({ "rawtypes", "unchecked" }) + Hashtable names = new Hashtable(input.sequenceNames); + SeqsetUtils.deuniquify(names, result); + alorders.add(msaOrder); + results[i] = result; + orders[i] = msaOrder; } - else { + else + { results[i] = null; } } + + Object[] newView = msa.getUpdatedView(results, orders, gapCharacter); + // free references to original data + for (int i = 0; i < jobs.size(); i++) + { + results[i] = null; + orders[i] = null; + } + SequenceI[] alignment = (SequenceI[]) newView[0]; + HiddenColumns hidden = (HiddenColumns) newView[1]; + Alignment aln = new Alignment(alignment); + aln.setProperty("Alignment Program", service.getName()); + if (dataset != null) + aln.setDataset(dataset); + + propagateDatasetMappings(aln); + + displayNewFrame(aln, alorders, hidden); } - @Override - public WebServiceI getWebService() + /* + * conserves dataset references to sequence objects returned from web + * services. propagate codon frame data to alignment. + */ + private void propagateDatasetMappings(Alignment aln) { - return service; + if (codonFrame != null) + { + SequenceI[] alignment = aln.getSequencesArray(); + for (SequenceI seq : alignment) + { + for (AlignedCodonFrame acf : codonFrame) + { + if (acf != null && acf.involvesSequence(seq)) + { + aln.addCodonFrame(acf); + break; + } + } + } + } + } + + private void displayNewFrame(AlignmentI aln, + List alorders, HiddenColumns hidden) + { + AlignFrame frame = new AlignFrame(aln, hidden, + AlignFrame.DEFAULT_WIDTH, AlignFrame.DEFAULT_HEIGHT); + // TODO store feature renderer settings in worker object + // frame.getFeatureRenderer().transferSettings(featureSettings); + var regions = sortOrders(alorders); + if (alorders.size() == 1) + { + frame.addSortByOrderMenuItem( + format("%s Ordering", service.getName()), alorders.get(0)); + } + else + { + for (int i = 0; i < alorders.size(); i++) + { + final int j = i; + Iterable iter = () -> regions.get(j).stream() + .map(it -> Integer.toString(it)).iterator(); + var orderName = format("%s Region %s Ordering", service.getName(), + String.join(",", iter)); + frame.addSortByOrderMenuItem(orderName, alorders.get(i)); + } + } + + /* TODO + * If alignment was requested from one half of a SplitFrame, show in a + * SplitFrame with the other pane similarly aligned. + */ } + private List> sortOrders(List alorders) + { + List> regions = new ArrayList<>(); + for (int i = 0; i < alorders.size(); i++) + { + List regs = new ArrayList<>(); + regs.add(i); + int j = i + 1; + while (j < alorders.size()) + { + if (alorders.get(i).equals(alorders.get(j))) + { + alorders.remove(j); + regs.add(j); + } + else + { + j++; + } + } + regions.add(regs); + } + return regions; + } + } + + private static class JobInput + { + final List inputSequences; + + final List emptySequences; + + @SuppressWarnings("rawtypes") + final Map sequenceNames; + + private JobInput(int numSequences, List inputSequences, + List emptySequences, + @SuppressWarnings("rawtypes") Map names) + { + this.inputSequences = Collections.unmodifiableList(inputSequences); + this.emptySequences = Collections.unmodifiableList(emptySequences); + this.sequenceNames = names; + } + + boolean isInputValid() + { + return inputSequences.size() >= 2; + } + + static JobInput create(SequenceI[] sequences, int minLength, + boolean submitGaps) + { + assert minLength >= 0 : MessageManager.getString( + "error.implementation_error_minlen_must_be_greater_zero"); + int numSeq = 0; + for (SequenceI seq : sequences) + { + if (seq.getEnd() - seq.getStart() >= minLength) + { + numSeq++; + } + } + + List inputSequences = new ArrayList<>(); + List emptySequences = new ArrayList<>(); + @SuppressWarnings("rawtypes") + Map names = new LinkedHashMap<>(); + for (int i = 0; i < sequences.length; i++) + { + SequenceI seq = sequences[i]; + String newName = SeqsetUtils.unique_name(i); + @SuppressWarnings("rawtypes") + Hashtable hash = SeqsetUtils.SeqCharacterHash(seq); + names.put(newName, hash); + if (numSeq > 1 && seq.getEnd() - seq.getStart() >= minLength) + { + String seqString = seq.getSequenceAsString(); + if (!submitGaps) + { + seqString = AlignSeq.extractGaps( + jalview.util.Comparison.GapChars, seqString); + } + inputSequences.add(new Sequence(newName, seqString)); + } + else + { + String seqString = null; + if (seq.getEnd() >= seq.getStart()) // is it ever false? + { + seqString = seq.getSequenceAsString(); + if (!submitGaps) + { + seqString = AlignSeq.extractGaps( + jalview.util.Comparison.GapChars, seqString); + } + } + emptySequences.add(new Sequence(newName, seqString)); + } + } + + return new JobInput(numSeq, inputSequences, emptySequences, names); + } } } -- 1.7.10.2