package jalview.ws2.operations; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import jalview.analysis.AlignSeq; import jalview.analysis.AlignmentAnnotationUtils; import jalview.analysis.SeqsetUtils; import jalview.api.AlignCalcManagerI2; import jalview.api.AlignViewportI; import jalview.api.AlignmentViewPanel; import jalview.api.FeatureColourI; import jalview.api.PollableAlignCalcWorkerI; import jalview.bin.Cache; import jalview.datamodel.Alignment; import jalview.datamodel.AlignmentAnnotation; import jalview.datamodel.AlignmentI; import jalview.datamodel.AnnotatedCollectionI; import jalview.datamodel.Annotation; import jalview.datamodel.ContiguousI; import jalview.datamodel.Mapping; import jalview.datamodel.Sequence; import jalview.datamodel.SequenceI; import jalview.datamodel.features.FeatureMatcherSetI; import jalview.gui.AlignFrame; import jalview.gui.AlignViewport; import jalview.gui.IProgressIndicator; import jalview.gui.IProgressIndicatorHandler; import jalview.io.FeaturesFile; import jalview.schemes.FeatureSettingsAdapter; import jalview.schemes.ResidueProperties; import jalview.util.MapList; import jalview.workers.AlignCalcManager2; import jalview.ws.params.ArgumentI; import jalview.ws2.WSJob; import jalview.ws2.WSJobStatus; import jalview.ws2.WebServiceI; import jalview.ws2.gui.ProgressBarUpdater; import static java.lang.String.format; public class AnnotationServiceWorker implements PollableAlignCalcWorkerI { private AnnotationOperation operation; private WebServiceI service; private List args; private AlignViewport viewport; private AlignmentViewPanel alignPanel; List sequences; private IProgressIndicator progressIndicator; private AlignFrame frame; private final AlignCalcManagerI2 calcMan; private Map seqNames; /** * indicates columns consisting of gaps only */ boolean[] gapMap = new boolean[0]; int start, end; boolean transferSequenceFeatures = false; private WSJob job; private List ourAnnots; private int exceptionCount = MAX_RETRY; private static final int MAX_RETRY = 5; public AnnotationServiceWorker(AnnotationOperation operation, List args, AlignViewport viewport, AlignmentViewPanel alignPanel, IProgressIndicator progressIndicator, AlignFrame frame, AlignCalcManagerI2 calcMan) { this.operation = operation; this.service = operation.service; this.args = args; this.viewport = viewport; this.alignPanel = alignPanel; this.progressIndicator = progressIndicator; this.frame = frame; this.calcMan = calcMan; } @Override public String getCalcName() { return service.getName(); } @Override public boolean involves(AlignmentAnnotation annot) { return ourAnnots != null && ourAnnots.contains(annot); } @Override public void updateAnnotation() { if (!calcMan.isWorking(this) && job != null && !job.getStatus().isCompleted()) { // is it correct to store annotations in a field and use them here? updateResultAnnotation(ourAnnots); } } @Override public void removeAnnotation() { if (ourAnnots != null && viewport != null) { AlignmentI alignment = viewport.getAlignment(); synchronized (ourAnnots) { for (AlignmentAnnotation aa : ourAnnots) { alignment.deleteAnnotation(aa, true); } } } } @Override public boolean isDeletable() { return true; } @Override public void startUp() throws IOException { if (viewport.isClosed()) { return; } /* What "bySequence" means in this context and * what is the SelectionGroup and why is it only relevant when * not dealing with alignment analysis? */ var bySequence = !operation.isAlignmentAnalysis(); sequences = prepareInput(viewport.getAlignment(), bySequence ? viewport.getSelectionGroup() : null); if (sequences == null) { Cache.log.info("Sequences for analysis service were null"); return; } if (!checkInputSequencesValid(sequences)) { Cache.log.info("Sequences for analysis service were not valid"); } Cache.log.debug(format("submitting %d sequences to %s", sequences.size(), service.getName())); job = new WSJob(service.getProviderName(), service.getName(), service.getHostName()); // Should this part be moved out of this class to one of the gui // classes? if (progressIndicator != null) { job.addPropertyChangeListener("status", new ProgressBarUpdater(progressIndicator)); progressIndicator.registerHandler(job.getUid(), new IProgressIndicatorHandler() { @Override public boolean cancelActivity(long id) { calcMan.cancelWorker(AnnotationServiceWorker.this); return true; } @Override public boolean canCancel() { return isDeletable(); } }); } String jobId = service.submit(sequences, args); job.setJobId(jobId); Cache.log.debug(format("Service %s: submitted job id %s", service.getHostName(), jobId)); } private List prepareInput(AlignmentI alignment, AnnotatedCollectionI inputSeqs) { if (alignment == null || alignment.getWidth() <= 0 || alignment.getSequences() == null) return null; if (alignment.isNucleotide() && !operation.isNucleotideOperation()) return null; if (!alignment.isNucleotide() && !operation.isProteinOperation()) return null; if (inputSeqs == null || inputSeqs.getWidth() <= 0 || inputSeqs.getSequences() == null || inputSeqs.getSequences().size() < 1) inputSeqs = alignment; List seqs = new ArrayList<>(); final boolean submitGaps = operation.isAlignmentAnalysis(); final int minlen = 10; int ln = -1; // I think this variable is redundant if (!operation.isAlignmentAnalysis()) seqNames = new HashMap<>(); start = inputSeqs.getStartRes(); end = inputSeqs.getEndRes(); // TODO: URGENT! unify with JPred / MSA code to handle hidden regions // correctly // TODO: push attributes into WsJob instance (so they can be safely // persisted/restored for (SequenceI sq : inputSeqs.getSequences()) { int sqlen; // is it trying to find the length of a sequence excluding gaps? if (!operation.isAlignmentAnalysis()) // why starting at positions to the right from the end/start? sqlen = sq.findPosition(end + 1) - sq.findPosition(start + 1); else sqlen = sq.getEnd() - sq.getStart(); if (sqlen >= minlen) { String newName = SeqsetUtils.unique_name(seqs.size()); if (seqNames != null) { seqNames.put(newName, sq); } SequenceI seq; if (submitGaps) { seq = new Sequence(newName, sq.getSequenceAsString()); seqs.add(seq); if (gapMap == null || gapMap.length < seq.getLength()) { boolean[] tg = gapMap; gapMap = new boolean[seq.getLength()]; System.arraycopy(tg, 0, gapMap, 0, tg.length); for (int p = tg.length; p < gapMap.length; p++) { gapMap[p] = false; // init as a gap } } for (int apos : sq.gapMap()) { char sqc = sq.getCharAt(apos); boolean isStandard = sq.isProtein() ? ResidueProperties.aaIndex[sqc] < 20 : ResidueProperties.nucleotideIndex[sqc] < 5; if (!operation.getFilterNonStandardSymbols() || isStandard) { gapMap[apos] = true; } } } else { // TODO: add ability to exclude hidden regions String sqstring = sq.getSequenceAsString(start, end + 1); seq = new Sequence(newName, AlignSeq.extractGaps(jalview.util.Comparison.GapChars, sqstring)); seqs.add(seq); // for annotation need to also record map to sequence start/end // position in range // then transfer back to original sequence on return. } ln = Integer.max(seq.getLength(), ln); } } if (operation.getNeedsAlignedSequences() && submitGaps) { int realw = 0; for (int i = 0; i < gapMap.length; i++) { if (gapMap[i]) { realw++; } } // try real hard to return something submittable // TODO: some of AAcon measures need a minimum of two or three amino // acids at each position, and AAcon doesn't gracefully degrade. for (int p = 0; p < seqs.size(); p++) { SequenceI sq = seqs.get(p); // strip gapped columns char[] padded = new char[realw]; char[] orig = sq.getSequence(); for (int i = 0, pp = 0; i < realw; pp++) { if (gapMap[pp]) { if (orig.length > pp) { padded[i++] = orig[pp]; } else { padded[i++] = '-'; } } } seqs.set(p, new Sequence(sq.getName(), new String(padded))); } } return seqs; } private boolean checkInputSequencesValid(List sequences) { int nvalid = 0; boolean allowProtein = operation.isProteinOperation(), allowNucleotides = operation.isNucleotideOperation(); for (SequenceI sq : sequences) { if (sq.getStart() <= sq.getEnd() && (sq.isProtein() ? allowProtein : allowNucleotides)) { nvalid++; } } return nvalid >= operation.getMinSequences(); } @Override public boolean poll() throws IOException { if (!job.getStatus().isDone() && !job.getStatus().isFailed()) { Cache.log.debug(format("Polling job %s", job)); try { service.updateProgress(job); exceptionCount = MAX_RETRY; } catch (IOException e) { Cache.log.error(format("Polling job %s failed.", job), e); if (--exceptionCount <= 0) { job.setStatus(WSJobStatus.SERVER_ERROR); Cache.log.warn(format("Attempts limit exceeded. Dropping job %s.", job)); } } catch (OutOfMemoryError e) { job.setStatus(WSJobStatus.BROKEN); Cache.log.error(format("Out of memory when retrieving job %s", job), e); } } return job.getStatus().isDone() || job.getStatus().isFailed(); } @Override public void cancel() { try { service.cancel(job); } catch (IOException e) { Cache.log.error(format("Failed to cancel job %s.", job), e); } } @Override public void done() { Cache.log.debug(format("Polling loop exited, job %s is %s", job, job.getStatus())); if (!job.getStatus().isCompleted()) { return; } List outputAnnotations = null; try { outputAnnotations = operation.annotationSupplier .getResult(job, sequences, viewport); } catch (IOException e) { Cache.log.error(format("Couldn't retrieve features for job %s.", job), e); } if (outputAnnotations != null) Cache.log.debug(format("Obtained %d annotation rows.", outputAnnotations.size())); else Cache.log.debug("Obtained no annotations."); Map featureColours = new HashMap<>(); Map featureFilters = new HashMap<>(); FeaturesFile featuresFile; try { // I think there should be a better way for obtaining features // Are the features added to the sequences here? featuresFile = operation.featuresSupplier.getResult(job, sequences, viewport); if (featuresFile != null) { Alignment aln = new Alignment(sequences.toArray(new SequenceI[0])); // I do nothing with the featureFilters object featuresFile.parse(aln, featureColours, true); } } catch (IOException e) { Cache.log.error(format("Couldn't retrieve features for job %s", job), e); } Cache.log.debug(format("There are %d feature colours and %d filters.", featureColours.size(), featureFilters.size())); if (outputAnnotations != null) { for (AlignmentAnnotation aa : outputAnnotations) { if (aa.getCalcId() == null || aa.getCalcId().equals("")) { aa.setCalcId(service.getName()); } // Can't services other than alignment analysis be interactive? // What's the point of storing that information in the annotation? aa.autoCalculated = operation.isAlignmentAnalysis() && operation.isInteractive(); } updateResultAnnotation(outputAnnotations); if (transferSequenceFeatures) { Cache.log.debug(format("Updating feature display settings and transferring" + "features fron job %s at %s", job, service.getHostName())); viewport.applyFeaturesStyle(new FeatureSettingsAdapter() { @Override public FeatureColourI getFeatureColour(String type) { return featureColours.get(type); } @Override public FeatureMatcherSetI getFeatureFilters(String type) { return featureFilters.get(type); } @Override public boolean isFeatureDisplayed(String type) { return featureColours.containsKey(type); } }); if (frame.alignPanel == alignPanel) { viewport.setShowSequenceFeatures(true); frame.setMenusForViewport(); } } } Cache.log.debug("Annotation service task finished."); } // What is the purpose of this method? // When is it called (apart from the above)? private void updateResultAnnotation(List annotations) { var currentAnnotations = Objects.requireNonNullElse( viewport.getAlignment().getAlignmentAnnotation(), new AlignmentAnnotation[0]); List newAnnots = new ArrayList<>(); // what is the graph group and why starting from 1? int graphGroup = 1; for (AlignmentAnnotation alna : currentAnnotations) { graphGroup = Integer.max(graphGroup, alna.graphGroup); } for (AlignmentAnnotation ala : annotations) { if (ala.graphGroup > 0) { ala.graphGroup += graphGroup; } // stores original sequence, in what case it ends up as null? SequenceI aseq = null; if (ala.sequenceRef != null) { SequenceI seq = seqNames.get(ala.sequenceRef.getName()); aseq = seq; while (seq.getDatasetSequence() != null) { seq = seq.getDatasetSequence(); } } Annotation[] resAnnot = ala.annotations; Annotation[] gappedAnnot = new Annotation[Math .max(viewport.getAlignment().getWidth(), gapMap.length)]; // is it adding gaps which were previously removed to the annotation? for (int p = 0, ap = start; ap < gappedAnnot.length; ap++) { if (gapMap != null && gapMap.length > ap && !gapMap[ap]) { gappedAnnot[ap] = new Annotation("", "", ' ', Float.NaN); } else if (p < resAnnot.length) { gappedAnnot[ap] = resAnnot[p++]; } } // replacing sequence with the original one? ala.sequenceRef = aseq; ala.annotations = gappedAnnot; AlignmentAnnotation newAnnot = viewport.getAlignment() .updateFromOrCopyAnnotation(ala); if (aseq != null) { aseq.addAlignmentAnnotation(newAnnot); newAnnot.adjustForAlignment(); AlignmentAnnotationUtils.replaceAnnotationOnAlignmentWith(newAnnot, newAnnot.label, newAnnot.getCalcId()); } newAnnots.add(newAnnot); } for (SequenceI sq : sequences) { // what are DBRefs? why are they relevant here? if (!sq.getFeatures().hasFeatures() && (sq.getDBRefs() == null || sq.getDBRefs().size() == 0)) { continue; } transferSequenceFeatures = true; SequenceI seq = seqNames.get(sq.getName()); SequenceI dseq; ContiguousI seqRange = seq.findPositions(start, end); while ((dseq = seq).getDatasetSequence() != null) { seq = seq.getDatasetSequence(); } List sourceRange = new ArrayList<>(); if (gapMap != null && gapMap.length > end) { int lastcol = start, col = start; do { if (col == end || !gapMap[col]) { if (lastcol <= col - 1) { seqRange = seq.findPositions(lastcol, col); sourceRange.add(seqRange); } lastcol = col + 1; } } while (++col < end); } else { sourceRange.add(seq.findPositions(start, end)); } int i = 0; int sourceStartEnd[] = new int[sourceRange.size() * 2]; for (ContiguousI range : sourceRange) { sourceStartEnd[i++] = range.getBegin(); sourceStartEnd[i++] = range.getEnd(); } Mapping mp = new Mapping(new MapList(sourceStartEnd, new int[] { seq.getStart(), seq.getEnd() }, 1, 1)); dseq.transferAnnotation(sq, mp); } updateOurAnnots(newAnnots); } protected void updateOurAnnots(List annots) { List our = ourAnnots; ourAnnots = Collections.synchronizedList(annots); AlignmentI alignment = viewport.getAlignment(); if (our != null) { if (our.size() > 0) { for (AlignmentAnnotation an : our) { if (!ourAnnots.contains(an)) { // remove the old annotation alignment.deleteAnnotation(an); } } } our.clear(); } // validate rows and update Alignment state synchronized (ourAnnots) { for (AlignmentAnnotation an : ourAnnots) { viewport.getAlignment().validateAnnotation(an); } } // TODO: may need a menu refresh after this // af.setMenusForViewport(); alignPanel.adjustAnnotationHeight(); } }