1 package jalview.ws2.operations;
3 import java.io.IOException;
4 import java.util.ArrayList;
5 import java.util.Collections;
6 import java.util.HashMap;
7 import java.util.LinkedHashMap;
10 import java.util.Objects;
11 import java.util.function.Consumer;
13 import jalview.analysis.AlignSeq;
14 import jalview.analysis.AlignmentSorter;
15 import jalview.analysis.SeqsetUtils;
16 import jalview.analysis.SeqsetUtils.SequenceInfo;
17 import jalview.bin.Cache;
18 import jalview.datamodel.AlignedCodonFrame;
19 import jalview.datamodel.Alignment;
20 import jalview.datamodel.AlignmentI;
21 import jalview.datamodel.AlignmentOrder;
22 import jalview.datamodel.AlignmentView;
23 import jalview.datamodel.HiddenColumns;
24 import jalview.datamodel.Sequence;
25 import jalview.datamodel.SequenceI;
26 import jalview.gui.AlignViewport;
27 import jalview.util.MessageManager;
28 import jalview.ws.params.ArgumentI;
29 import jalview.ws2.WSJob;
30 import jalview.ws2.WSJobStatus;
32 public class AlignmentWorker extends AbstractPollableWorker
34 private AlignmentOperation operation;
36 private Consumer<AlignmentResult> resultConsumer;
38 private AlignmentView msa;
40 private AlignmentI dataset;
42 private AlignViewport viewport;
44 private List<AlignedCodonFrame> codonFrame = new ArrayList<>();
46 private List<ArgumentI> args = Collections.emptyList();
48 private boolean submitGaps = false;
50 private boolean preserveOrder = false;
52 private char gapCharacter;
54 private WSJobList<AlignmentJob> jobs = new WSJobList<>();
56 private Map<Long, Integer> exceptionCount = new HashMap<>();
58 private static final int MAX_RETRY = 5;
60 private static class JobInput
62 final List<SequenceI> inputSequences;
64 final List<SequenceI> emptySequences;
66 final Map<String, SequenceInfo> sequenceNames;
68 private JobInput(List<SequenceI> inputSequences,
69 List<SequenceI> emptySequences,
70 Map<String, SequenceInfo> names)
72 this.inputSequences = Collections.unmodifiableList(inputSequences);
73 this.emptySequences = Collections.unmodifiableList(emptySequences);
74 this.sequenceNames = names;
77 boolean isInputValid()
79 return inputSequences.size() >= 2;
83 public class AlignmentJob extends WSJob
85 private List<SequenceI> inputSequences;
86 private List<SequenceI> emptySequences;
87 private Map<String, SequenceInfo> sequenceNames;
89 private AlignmentJob(String serviceProvider, String serviceName,
92 super(serviceProvider, serviceName, hostName);
95 private void setInput(JobInput input) {
96 inputSequences = input.inputSequences;
97 emptySequences = input.emptySequences;
98 sequenceNames = input.sequenceNames;
102 public class AlignmentResult
106 List<AlignmentOrder> alorders;
108 HiddenColumns hidden;
110 AlignmentResult(AlignmentI aln, List<AlignmentOrder> alorders,
111 HiddenColumns hidden)
114 this.alorders = alorders;
115 this.hidden = hidden;
118 public AlignmentI getAlignment()
123 public List<AlignmentOrder> getAlignmentOrders()
128 public HiddenColumns getHiddenColumns()
134 public AlignmentWorker(AlignmentOperation operation,
135 AlignmentView msa, List<ArgumentI> args,
136 boolean submitGaps, boolean preserveOrder, AlignViewport viewport)
138 this.operation = operation;
140 this.dataset = viewport.getAlignment().getDataset();
141 List<AlignedCodonFrame> cf = Objects.requireNonNullElse(
142 viewport.getAlignment().getCodonFrames(), Collections.emptyList());
143 this.codonFrame.addAll(cf);
145 this.submitGaps = submitGaps;
146 this.preserveOrder = preserveOrder;
147 this.viewport = viewport;
148 this.gapCharacter = viewport.getGapCharacter();
152 public Operation getOperation()
158 public WSJobList<? extends WSJob> getJobs()
163 public void setResultConsumer(Consumer<AlignmentResult> consumer)
165 this.resultConsumer = consumer;
169 public void start() throws IOException
171 Cache.log.info(String.format("Starting new %s job.", operation.getName()));
172 SequenceI[][] conmsa = msa.getVisibleContigs('-');
178 for (int i = 0; i < conmsa.length; i++)
180 JobInput input = prepareInputData(conmsa[i], 2, submitGaps);
181 AlignmentJob job = new AlignmentJob(operation.service.getProviderName(),
182 operation.getName(), operation.getHostName());
186 listeners.fireJobCreated(job);
187 if (input.isInputValid())
193 count = exceptionCount.getOrDefault(job.getUid(), MAX_RETRY);
196 jobId = operation.service.submit(input.inputSequences, args);
197 Cache.log.debug((String.format("Job %s submitted", job)));
198 exceptionCount.remove(job.getUid());
199 } catch (IOException e)
201 exceptionCount.put(job.getUid(), --count);
203 } while (jobId == null && count > 0);
207 job.setStatus(WSJobStatus.SUBMITTED);
212 job.setStatus(WSJobStatus.SERVER_ERROR);
217 job.setStatus(WSJobStatus.INVALID);
219 MessageManager.getString("label.empty_alignment_job"));
224 listeners.fireWorkerStarted();
228 listeners.fireWorkerNotStarted();
233 private static JobInput prepareInputData(SequenceI[] sequences,
234 int minLength, boolean submitGaps)
236 assert minLength >= 0 : MessageManager.getString(
237 "error.implementation_error_minlen_must_be_greater_zero");
239 for (SequenceI seq : sequences)
241 if (seq.getEnd() - seq.getStart() >= minLength)
247 List<SequenceI> inputSequences = new ArrayList<>();
248 List<SequenceI> emptySequences = new ArrayList<>();
249 Map<String, SequenceInfo> names = new LinkedHashMap<>();
250 for (int i = 0; i < sequences.length; i++)
252 SequenceI seq = sequences[i];
253 String newName = SeqsetUtils.unique_name(i);
254 var hash = SeqsetUtils.SeqCharacterHash(seq);
255 names.put(newName, hash);
256 if (numSeq > 1 && seq.getEnd() - seq.getStart() >= minLength)
258 String seqString = seq.getSequenceAsString();
261 seqString = AlignSeq.extractGaps(
262 jalview.util.Comparison.GapChars, seqString);
264 inputSequences.add(new Sequence(newName, seqString));
268 String seqString = "";
269 if (seq.getEnd() >= seq.getStart()) // true if gaps only
271 seqString = seq.getSequenceAsString();
274 seqString = AlignSeq.extractGaps(
275 jalview.util.Comparison.GapChars, seqString);
278 emptySequences.add(new Sequence(newName, seqString));
282 return new JobInput(inputSequences, emptySequences, names);
288 listeners.fireWorkerCompleting();
289 Map<Long, AlignmentI> results = new LinkedHashMap<>();
290 for (WSJob job : getJobs())
292 if (job.getStatus().isFailed())
296 AlignmentI alignment = operation.getAlignmentSupplier().getAlignment(job);
297 if (alignment != null)
299 results.put(job.getUid(), alignment);
301 } catch (Exception e)
303 if (!operation.getWebService().handleCollectionError(job, e))
305 Cache.log.error("Couldn't get alignment for job.", e);
306 // TODO: Increment exception count and retry.
307 job.setStatus(WSJobStatus.SERVER_ERROR);
311 if (results.size() > 0)
313 AlignmentResult out = prepareResult(results);
314 resultConsumer.accept(out);
318 resultConsumer.accept(null);
320 listeners.fireWorkerCompleted();
323 private AlignmentResult prepareResult(Map<Long, AlignmentI> alignments)
325 List<AlignmentOrder> alorders = new ArrayList<>();
326 SequenceI[][] results = new SequenceI[jobs.size()][];
327 AlignmentOrder[] orders = new AlignmentOrder[jobs.size()];
328 for (int i = 0; i < jobs.size(); i++)
330 AlignmentJob job = jobs.get(i);
331 AlignmentI aln = alignments.get(job.getUid());
332 if (aln != null) // equivalent of job.hasResults()
334 /* Get the alignment including any empty sequences in the original
335 * order with original ids. */
336 char gapChar = aln.getGapCharacter();
337 List<SequenceI> emptySeqs = job.emptySequences;
338 List<SequenceI> alnSeqs = aln.getSequences();
339 // find the width of the longest sequence
341 for (var seq : alnSeqs)
342 width = Integer.max(width, seq.getLength());
343 for (var emptySeq : emptySeqs)
344 width = Integer.max(width, emptySeq.getLength());
345 // pad shorter sequences with gaps
346 String gapSeq = String.join("",
347 Collections.nCopies(width, Character.toString(gapChar)));
348 List<SequenceI> seqs = new ArrayList<>(
349 alnSeqs.size() + emptySeqs.size());
350 seqs.addAll(alnSeqs);
351 seqs.addAll(emptySeqs);
354 if (seq.getLength() < width)
355 seq.setSequence(seq.getSequenceAsString()
356 + gapSeq.substring(seq.getLength()));
358 SequenceI[] result = seqs.toArray(new SequenceI[0]);
359 AlignmentOrder msaOrder = new AlignmentOrder(result);
360 AlignmentSorter.recoverOrder(result);
361 Map<String, SequenceInfo> names = new HashMap<>(job.sequenceNames);
362 // FIXME first call to deuniquify alters original alignment
363 SeqsetUtils.deuniquify(names, result);
364 alorders.add(msaOrder);
366 orders[i] = msaOrder;
374 Object[] newView = msa.getUpdatedView(results, orders, gapCharacter);
375 // free references to original data
376 for (int i = 0; i < jobs.size(); i++)
381 SequenceI[] alignment = (SequenceI[]) newView[0];
382 HiddenColumns hidden = (HiddenColumns) newView[1];
383 Alignment aln = new Alignment(alignment);
384 aln.setProperty("Alignment Program", operation.getName());
386 aln.setDataset(dataset);
388 propagateDatasetMappings(aln);
389 return new AlignmentResult(aln, alorders, hidden);
390 // displayNewFrame(aln, alorders, hidden);
394 * conserves dataset references to sequence objects returned from web
395 * services. propagate codon frame data to alignment.
397 private void propagateDatasetMappings(Alignment aln)
399 if (codonFrame != null)
401 SequenceI[] alignment = aln.getSequencesArray();
402 for (SequenceI seq : alignment)
404 for (AlignedCodonFrame acf : codonFrame)
406 if (acf != null && acf.involvesSequence(seq))
408 aln.addCodonFrame(acf);