From: Mateusz Warowny Date: Mon, 18 Oct 2021 16:04:53 +0000 (+0200) Subject: JAL-3878 Refactoring SeqAnnotationServiceCalcWorker. X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=7385bc1418b68162cc670cc35f89033f34db2e86;p=jalview.git JAL-3878 Refactoring SeqAnnotationServiceCalcWorker. --- diff --git a/src/jalview/ws2/WSJobStatus.java b/src/jalview/ws2/WSJobStatus.java index 0a3872d..9172f7c 100755 --- a/src/jalview/ws2/WSJobStatus.java +++ b/src/jalview/ws2/WSJobStatus.java @@ -97,4 +97,9 @@ public enum WSJobStatus { return this == WSJobStatus.SUBMITTED || this == WSJobStatus.QUEUED; } + + public boolean isCompleted() + { + return this == WSJobStatus.FINISHED; + } } \ No newline at end of file diff --git a/src/jalview/ws2/gui/ProgressBarUpdater.java b/src/jalview/ws2/gui/ProgressBarUpdater.java new file mode 100644 index 0000000..476308c --- /dev/null +++ b/src/jalview/ws2/gui/ProgressBarUpdater.java @@ -0,0 +1,47 @@ +package jalview.ws2.gui; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; + +import jalview.gui.AlignFrame; +import jalview.gui.IProgressIndicator; +import jalview.ws2.WSJob; +import jalview.ws2.WSJobStatus; + +public class ProgressBarUpdater implements PropertyChangeListener +{ + private IProgressIndicator progressIndicator; + + public ProgressBarUpdater(IProgressIndicator progressIndicator) + { + this.progressIndicator = progressIndicator; + } + + @Override + public void propertyChange(PropertyChangeEvent evt) + { + switch (evt.getPropertyName()) + { + case "status": + statusChanged(evt); + break; + } + } + + private void statusChanged(PropertyChangeEvent evt) + { + var job = (WSJob) evt.getSource(); + var oldStatus = (WSJobStatus) evt.getOldValue(); + var newStatus = (WSJobStatus) evt.getNewValue(); + boolean wasRunning = oldStatus.isRunning() || oldStatus.isSubmitted(); + boolean isRunning = newStatus.isRunning() || newStatus.isSubmitted(); + if (!wasRunning && isRunning) + { + progressIndicator.setProgressBar(job.getServiceName(), job.getUid()); + } + else if (wasRunning && !isRunning) + { + progressIndicator.removeProgressBar(job.getUid()); + } + } +} diff --git a/src/jalview/ws2/operations/AlignmentOperation.java b/src/jalview/ws2/operations/AlignmentOperation.java index 5b2f554..89aeb79 100644 --- a/src/jalview/ws2/operations/AlignmentOperation.java +++ b/src/jalview/ws2/operations/AlignmentOperation.java @@ -110,6 +110,12 @@ public class AlignmentOperation implements Operation { return true; } + + @Override + public boolean isAlignmentAnalysis() + { + return false; + } @Override public boolean canSubmitGaps() @@ -123,6 +129,18 @@ public class AlignmentOperation implements Operation { return false; } + + @Override + public boolean getFilterNonStandardSymbols() + { + return true; + } + + @Override + public boolean getNeedsAlignedSequences() + { + return false; + } @Override public MenuEntryProviderI getMenuBuilder() diff --git a/src/jalview/ws2/operations/AnnotationOperation.java b/src/jalview/ws2/operations/AnnotationOperation.java index b940f3b..1eb5da0 100644 --- a/src/jalview/ws2/operations/AnnotationOperation.java +++ b/src/jalview/ws2/operations/AnnotationOperation.java @@ -10,14 +10,17 @@ import java.util.concurrent.CompletionStage; import javax.swing.JMenu; import javax.swing.JMenuItem; +import jalview.api.AlignmentViewPanel; import jalview.bin.Cache; import jalview.datamodel.AlignmentAnnotation; import jalview.gui.AlignFrame; +import jalview.gui.AlignViewport; import jalview.gui.WsJobParameters; import jalview.io.AnnotationFile; import jalview.io.FeaturesFile; import jalview.util.MathUtils; import jalview.util.MessageManager; +import jalview.workers.AlignCalcManager2; import jalview.ws.params.ArgumentI; import jalview.ws.params.WsParamSetI; import jalview.ws2.MenuEntryProviderI; @@ -111,6 +114,24 @@ public class AnnotationOperation implements Operation } @Override + public boolean isAlignmentAnalysis() + { + return false; + } + + @Override + public boolean getFilterNonStandardSymbols() + { + return false; + } + + @Override + public boolean getNeedsAlignedSequences() + { + return false; + } + + @Override public MenuEntryProviderI getMenuBuilder() { return this::buildMenu; @@ -120,12 +141,18 @@ public class AnnotationOperation implements Operation { final var calcName = service.getName(); PollingTaskExecutor wsExecutor = frame.getViewport().getWSExecutor(); + final var calcManager = frame.getViewport().getCalcManager(); { var item = new JMenuItem(MessageManager.formatMessage( "label.calcname_with_default_settings", calcName)); item.addActionListener((event) -> { - WebServiceWorkerI worker = new AnnotationWorker(); - wsExecutor.submit(worker); + AlignViewport viewport = frame.getCurrentView(); + AlignmentViewPanel alignPanel = frame.alignPanel; + var worker = new AnnotationServiceWorker(this, service, + Collections.emptyList(), viewport, alignPanel, frame, frame, + calcManager); + calcManager.startWorker(worker); + // TODO create and submit AnnotataionServiceWorker }); parent.add(item); } @@ -140,7 +167,12 @@ public class AnnotationOperation implements Operation .thenAcceptAsync((arguments) -> { if (arguments != null) { - + AlignViewport viewport = frame.getCurrentView(); + AlignmentViewPanel alignPanel = frame.alignPanel; + var worker = new AnnotationServiceWorker( + AnnotationOperation.this, service, arguments, viewport, + alignPanel, frame, frame, calcManager); + calcManager.startWorker(worker); } }); }); @@ -178,85 +210,4 @@ public class AnnotationOperation implements Operation }); } - private class AnnotationWorker implements WebServiceWorkerI - { - private long uid = MathUtils.getUID(); - - private WSJobList jobs = new WSJobList(); - - private HashMap exceptionCount = new HashMap<>(); - - private static final int MAX_RETRY = 5; - - @Override - public long getUID() - { - return uid; - } - - @Override - public WebServiceI getWebService() - { - return service; - } - - @Override - public List getJobs() - { - return Collections.unmodifiableList(jobs); - } - - @Override - public void start() throws IOException - { - - } - - @Override - public boolean poll() throws IOException - { - boolean done = true; - for (WSJob job : getJobs()) - { - if (!job.getStatus().isDone() && !job.getStatus().isFailed()) - { - Cache.log.debug(format("Polling job %s", job)); - try - { - service.updateProgress(job); - exceptionCount.remove(job.getUid()); - } catch (IOException e) - { - Cache.log.error(format("Polling job %s failed.", job), e); - int count = exceptionCount.getOrDefault(job.getUid(), - MAX_RETRY); - if (--count <= 0) - { - job.setStatus(WSJobStatus.SERVER_ERROR); - Cache.log.warn(format( - "Attempts limit exceeded. Droping job %s.", job)); - } - exceptionCount.put(job.getUid(), count); - } catch (OutOfMemoryError e) - { - job.setStatus(WSJobStatus.BROKEN); - Cache.log.error( - format("Out of memory when retrieving job %s", job), e); - } - Cache.log.debug( - format("Job %s status is %s", job, job.getStatus())); - } - done &= job.getStatus().isDone() || job.getStatus().isFailed(); - } - return done; - } - - @Override - public void done() - { - // TODO Auto-generated method stub - - } - - } } diff --git a/src/jalview/ws2/operations/AnnotationServiceWorker.java b/src/jalview/ws2/operations/AnnotationServiceWorker.java new file mode 100644 index 0000000..d59fbdf --- /dev/null +++ b/src/jalview/ws2/operations/AnnotationServiceWorker.java @@ -0,0 +1,569 @@ +package jalview.ws2.operations; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import jalview.analysis.AlignSeq; +import jalview.analysis.AlignmentAnnotationUtils; +import jalview.analysis.SeqsetUtils; +import jalview.api.AlignCalcManagerI2; +import jalview.api.AlignViewportI; +import jalview.api.AlignmentViewPanel; +import jalview.api.FeatureColourI; +import jalview.api.PollableAlignCalcWorkerI; +import jalview.bin.Cache; +import jalview.datamodel.Alignment; +import jalview.datamodel.AlignmentAnnotation; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.AnnotatedCollectionI; +import jalview.datamodel.Annotation; +import jalview.datamodel.ContiguousI; +import jalview.datamodel.Mapping; +import jalview.datamodel.Sequence; +import jalview.datamodel.SequenceI; +import jalview.datamodel.features.FeatureMatcherSetI; +import jalview.gui.AlignFrame; +import jalview.gui.AlignViewport; +import jalview.gui.IProgressIndicator; +import jalview.gui.IProgressIndicatorHandler; +import jalview.io.FeaturesFile; +import jalview.schemes.FeatureSettingsAdapter; +import jalview.schemes.ResidueProperties; +import jalview.util.MapList; +import jalview.workers.AlignCalcManager2; +import jalview.ws.params.ArgumentI; +import jalview.ws2.WSJob; +import jalview.ws2.WSJobStatus; +import jalview.ws2.WebServiceI; +import jalview.ws2.gui.ProgressBarUpdater; + +import static java.lang.String.format; + +public class AnnotationServiceWorker implements PollableAlignCalcWorkerI +{ + private AnnotationOperation operation; + private WebServiceI service; + private List args; + private AlignViewport viewport; + private AlignmentViewPanel alignPanel; + List sequences; + private IProgressIndicator progressIndicator; + private AlignFrame frame; + private final AlignCalcManagerI2 calcMan; + private Map seqNames; + boolean[] gapMap = new boolean[0]; + int start, end; + boolean transferSequenceFeatures = false; + private WSJob job; + private List ourAnnots; + + private int exceptionCount = MAX_RETRY; + private static final int MAX_RETRY = 5; + + AnnotationServiceWorker(AnnotationOperation operation, WebServiceI service, + List args, AlignViewport viewport, AlignmentViewPanel alignPanel, + IProgressIndicator progressIndicator, AlignFrame frame, AlignCalcManagerI2 calcMan) + { + this.operation = operation; + this.service = service; + this.args = args; + this.viewport = viewport; + this.alignPanel = alignPanel; + this.progressIndicator = progressIndicator; + this.frame = frame; + this.calcMan = calcMan; + } + + @Override + public String getCalcName() + { + return service.getName(); + } + + @Override + public boolean involves(AlignmentAnnotation annot) + { + return ourAnnots != null && ourAnnots.contains(annot); + } + + @Override + public void updateAnnotation() + { + if (!calcMan.isWorking(this) && job != null && !job.getStatus().isCompleted()) + { + updateResultAnnotation(ourAnnots); + } + } + + @Override + public void removeAnnotation() + { + if (ourAnnots != null && viewport != null) + { + AlignmentI alignment = viewport.getAlignment(); + synchronized (ourAnnots) + { + for (AlignmentAnnotation aa : ourAnnots) + { + alignment.deleteAnnotation(aa, true); + } + } + } + } + + @Override + public boolean isDeletable() + { + return false; + } + + @Override + public void startUp() throws IOException + { + if (viewport.isClosed()) + { + return; + } + var bySequence = !operation.isAlignmentAnalysis(); + sequences = prepareInput(viewport.getAlignment(), + bySequence ? viewport.getSelectionGroup() : null); + if (sequences == null) + { + Cache.log.info("Sequences for analysis service were null"); + return; + } + if (!checkInputSequencesValid(sequences)) + { + Cache.log.info("Sequences for analysis service were not valid"); + } + Cache.log.debug(format("submitting %d sequences to %s", sequences.size(), + service.getName())); + job = new WSJob(service.getProviderName(), service.getName(), + service.getHostName()); + if (progressIndicator != null) + { + job.addPropertyChangeListener("status", new ProgressBarUpdater(progressIndicator)); + progressIndicator.registerHandler(job.getUid(), new IProgressIndicatorHandler() + { + @Override + public boolean cancelActivity(long id) + { + calcMan.cancelWorker(AnnotationServiceWorker.this); + return true; + } + + @Override + public boolean canCancel() + { + return isDeletable(); + } + }); + } + String jobId = service.submit(sequences, args); + job.setJobId(jobId); + Cache.log.debug(format("Service %s: submitted job id %s", + service.getHostName(), jobId)); + } + + private List prepareInput(AlignmentI alignment, + AnnotatedCollectionI inputSeqs) + { + if (alignment == null || alignment.getWidth() <= 0 || + alignment.getSequences() == null) + return null; + if (alignment.isNucleotide() && !operation.isNucleotideOperation()) + return null; + if (!alignment.isNucleotide() && !operation.isProteinOperation()) + return null; + if (inputSeqs == null || inputSeqs.getWidth() <= 0 || + inputSeqs.getSequences() == null || inputSeqs.getSequences().size() < 1) + inputSeqs = alignment; + + List seqs = new ArrayList<>(); + final boolean submitGaps = operation.isAlignmentAnalysis(); + final int minlen = 10; + int ln = -1; + // FIXME don't return values by class parameters + if (!operation.isAlignmentAnalysis()) + seqNames = new HashMap<>(); + start = inputSeqs.getStartRes(); + end = inputSeqs.getEndRes(); + // TODO: URGENT! unify with JPred / MSA code to handle hidden regions + // correctly + // TODO: push attributes into WsJob instance (so they can be safely + // persisted/restored + for (SequenceI sq : inputSeqs.getSequences()) + { + int sqlen; + if (!operation.isAlignmentAnalysis()) + sqlen = sq.findPosition(end + 1) - sq.findPosition(start + 1); + else + sqlen = sq.getEnd() - sq.getStart(); + if (sqlen >= minlen) + { + String newName = SeqsetUtils.unique_name(seqs.size()); + if (seqNames != null) + { + seqNames.put(newName, sq); + } + SequenceI seq; + if (submitGaps) + { + seq = new Sequence(newName, sq.getSequenceAsString()); + seqs.add(seq); + if (gapMap == null || gapMap.length < seq.getLength()) + { + boolean[] tg = gapMap; + gapMap = new boolean[seq.getLength()]; + System.arraycopy(tg, 0, gapMap, 0, tg.length); + for (int p = tg.length; p < gapMap.length; p++) + { + gapMap[p] = false; // init as a gap + } + } + for (int apos : sq.gapMap()) + { + char sqc = sq.getCharAt(apos); + boolean isStandard = sq.isProtein() ? ResidueProperties.aaIndex[sqc] < 20 + : ResidueProperties.nucleotideIndex[sqc] < 5; + if (!operation.getFilterNonStandardSymbols() || isStandard) + { + gapMap[apos] = true; + } + } + } + else + { + // TODO: add ability to exclude hidden regions + String sqstring = sq.getSequenceAsString(start, end + 1); + seq = new Sequence(newName, + AlignSeq.extractGaps(jalview.util.Comparison.GapChars, sqstring)); + // for annotation need to also record map to sequence start/end + // position in range + // then transfer back to original sequence on return. + } + ln = Integer.max(seq.getLength(), ln); + } + } + if (operation.getNeedsAlignedSequences() && submitGaps) + { + int realw = 0; + for (int i = 0; i < gapMap.length; i++) + { + if (gapMap[i]) + { + realw++; + } + } + // try real hard to return something submittable + // TODO: some of AAcon measures need a minimum of two or three amino + // acids at each position, and AAcon doesn't gracefully degrade. + for (int p = 0; p < seqs.size(); p++) + { + SequenceI sq = seqs.get(p); + // strip gapped columns + char[] padded = new char[realw]; + char[] orig = sq.getSequence(); + for (int i = 0, pp = 0; i < realw; pp++) + { + if (gapMap[pp]) + { + if (orig.length > pp) + { + padded[i++] = orig[pp]; + } + else + { + padded[i++] = '-'; + } + } + } + seqs.set(p, new Sequence(sq.getName(), new String(padded))); + } + } + return seqs; + } + + private boolean checkInputSequencesValid(List sequences) + { + int nvalid = 0; + boolean allowProtein = operation.isProteinOperation(), + allowNucleotides = operation.isNucleotideOperation(); + for (SequenceI sq : sequences) + { + if (sq.getStart() <= sq.getEnd() && + (sq.isProtein() ? allowProtein : allowNucleotides)) + { + nvalid++; + } + } + return nvalid >= operation.getMinSequences(); + } + + @Override + public boolean poll() throws IOException + { + if (!job.getStatus().isDone() && !job.getStatus().isFailed()) + { + Cache.log.debug(format("Polling job %s", job)); + try + { + service.updateProgress(job); + exceptionCount = MAX_RETRY; + } catch (IOException e) + { + Cache.log.error(format("Polling job %s failed.", job), e); + if (--exceptionCount <= 0) + { + job.setStatus(WSJobStatus.SERVER_ERROR); + Cache.log.warn(format("Attempts limit exceeded. Dropping job %s.", job)); + } + } catch (OutOfMemoryError e) + { + job.setStatus(WSJobStatus.BROKEN); + Cache.log.error(format("Out of memory when retrieving job %s", job), e); + } + } + return job.getStatus().isDone() || job.getStatus().isFailed(); + } + + @Override + public void cancel() + { + try + { + service.cancel(job); + } catch (IOException e) + { + Cache.log.error(format("Failed to cancel job %s.", job), e); + } + } + + @Override + public void done() + { + Cache.log.debug(format("Polling loop exited, job %s is %s", job, job.getStatus())); + if (!job.getStatus().isCompleted()) + { + return; + } + List outputAnnotations = null; + try + { + outputAnnotations = operation.annotationSupplier + .getResult(job, sequences, viewport); + } catch (IOException e) + { + Cache.log.error(format("Couldn't retrieve features for job %s.", job), e); + } + if (outputAnnotations != null) + Cache.log.debug(format("Obtained %d annotation rows.", outputAnnotations.size())); + else + Cache.log.debug("Obtained no annotations."); + Map featureColours = new HashMap<>(); + Map featureFilters = new HashMap<>(); + FeaturesFile featuresFile; + try + { + featuresFile = operation.featuresSupplier.getResult(job, sequences, viewport); + if (featuresFile != null) + { + Alignment aln = new Alignment(sequences.toArray(new SequenceI[0])); + featuresFile.parse(aln, featureColours, true); + } + } catch (IOException e) + { + Cache.log.error(format("Couldn't retrieve features for job %s", job), e); + } + Cache.log.debug(format("There are %d feature colours and %d filters.", + featureColours.size(), featureFilters.size())); + if (outputAnnotations != null) + { + for (AlignmentAnnotation aa : outputAnnotations) + { + if (aa.getCalcId() == null || aa.getCalcId().equals("")) + { + aa.setCalcId(service.getName()); + } + aa.autoCalculated = operation.isAlignmentAnalysis() && operation.isInteractive(); + } + updateResultAnnotation(outputAnnotations); + if (transferSequenceFeatures) + { + Cache.log.debug(format("Updating feature display settings and transferring" + + "features fron job %s at %s", job, service.getHostName())); + viewport.applyFeaturesStyle(new FeatureSettingsAdapter() + { + @Override + public FeatureColourI getFeatureColour(String type) + { + return featureColours.get(type); + } + + @Override + public FeatureMatcherSetI getFeatureFilters(String type) + { + return featureFilters.get(type); + } + + @Override + public boolean isFeatureDisplayed(String type) + { + return featureColours.containsKey(type); + } + }); + if (frame.alignPanel == alignPanel) + { + viewport.setShowSequenceFeatures(true); + frame.setMenusForViewport(); + } + } + } + Cache.log.debug("Annotation service task finished."); + } + + private void updateResultAnnotation(List annotations) + { + var currentAnnotations = Objects.requireNonNullElse( + viewport.getAlignment().getAlignmentAnnotation(), + new AlignmentAnnotation[0]); + List newAnnots = new ArrayList<>(); + int graphGroup = 1; + for (AlignmentAnnotation alna : currentAnnotations) + { + graphGroup = Integer.max(graphGroup, alna.graphGroup); + } + for (AlignmentAnnotation ala : annotations) + { + if (ala.graphGroup > 0) + { + ala.graphGroup += graphGroup; + } + + SequenceI aseq = null; + if (ala.sequenceRef != null) + { + SequenceI seq = seqNames.get(ala.sequenceRef.getName()); + aseq = seq; + while (seq.getDatasetSequence() != null) + { + seq = seq.getDatasetSequence(); + } + } + Annotation[] resAnnot = ala.annotations; + Annotation[] gappedAnnot = new Annotation[Math + .max(viewport.getAlignment().getWidth(), gapMap.length)]; + for (int p = 0, ap = start; ap < gappedAnnot.length; ap++) + { + if (gapMap != null && gapMap.length > ap && !gapMap[ap]) + { + gappedAnnot[ap] = new Annotation("", "", ' ', Float.NaN); + } + else if (p < resAnnot.length) + { + gappedAnnot[ap] = resAnnot[p++]; + } + } + ala.sequenceRef = aseq; + ala.annotations = gappedAnnot; + AlignmentAnnotation newAnnot = viewport.getAlignment() + .updateFromOrCopyAnnotation(ala); + if (aseq != null) + { + aseq.addAlignmentAnnotation(newAnnot); + newAnnot.adjustForAlignment(); + AlignmentAnnotationUtils.replaceAnnotationOnAlignmentWith(newAnnot, + newAnnot.label, newAnnot.getCalcId()); + } + newAnnots.add(newAnnot); + } + + for (SequenceI sq : sequences) + { + if (!sq.getFeatures().hasFeatures() && + (sq.getDBRefs() == null || sq.getDBRefs().size() == 0)) + { + continue; + } + transferSequenceFeatures = true; + SequenceI seq = seqNames.get(sq.getName()); + SequenceI dseq; + ContiguousI seqRange = seq.findPositions(start, end); + + while ((dseq = seq).getDatasetSequence() != null) + { + seq = seq.getDatasetSequence(); + } + List sourceRange = new ArrayList<>(); + if (gapMap != null && gapMap.length > end) + { + int lastcol = start, col = start; + do + { + if (col == end || !gapMap[col]) + { + if (lastcol <= col - 1) + { + seqRange = seq.findPositions(lastcol, col); + sourceRange.add(seqRange); + } + lastcol = col + 1; + } + } while (++col < end); + } + else + { + sourceRange.add(seq.findPositions(start, end)); + } + int i = 0; + int sourceStartEnd[] = new int[sourceRange.size() * 2]; + for (ContiguousI range : sourceRange) + { + sourceStartEnd[i++] = range.getBegin(); + sourceStartEnd[i++] = range.getEnd(); + } + Mapping mp = new Mapping(new MapList(sourceStartEnd, + new int[] { seq.getStart(), seq.getEnd() }, 1, 1)); + dseq.transferAnnotation(sq, mp); + } + updateOurAnnots(newAnnots); + } + + protected void updateOurAnnots(List annots) + { + List our = ourAnnots; + ourAnnots = Collections.synchronizedList(annots); + AlignmentI alignment = viewport.getAlignment(); + if (our != null) + { + if (our.size() > 0) + { + for (AlignmentAnnotation an : our) + { + if (!ourAnnots.contains(an)) + { + // remove the old annotation + alignment.deleteAnnotation(an); + } + } + } + our.clear(); + } + // validate rows and update Alignment state + synchronized (ourAnnots) + { + for (AlignmentAnnotation an : ourAnnots) + { + viewport.getAlignment().validateAnnotation(an); + } + } + // TODO: may need a menu refresh after this + // af.setMenusForViewport(); + alignPanel.adjustAnnotationHeight(); + } +} diff --git a/src/jalview/ws2/operations/Operation.java b/src/jalview/ws2/operations/Operation.java index e244afa..ff559d4 100644 --- a/src/jalview/ws2/operations/Operation.java +++ b/src/jalview/ws2/operations/Operation.java @@ -23,4 +23,10 @@ public interface Operation public boolean isInteractive(); public MenuEntryProviderI getMenuBuilder(); + + public boolean isAlignmentAnalysis(); + + public boolean getFilterNonStandardSymbols(); + + public boolean getNeedsAlignedSequences(); }