2 // Getdown - application installer, patcher and launcher
3 // Copyright (C) 2004-2018 Getdown authors
4 // https://github.com/threerings/getdown/blob/master/LICENSE
6 package com.threerings.getdown.net;
9 import java.io.IOException;
11 import java.util.Collection;
12 import java.util.HashMap;
14 import java.util.concurrent.ExecutorService;
15 import java.util.concurrent.Executors;
16 import java.util.concurrent.TimeUnit;
18 import com.threerings.getdown.data.Resource;
20 import static com.threerings.getdown.Log.log;
23 * Handles the download of a collection of files, first issuing HTTP head requests to obtain size
24 * information and then downloading the files individually, reporting progress back via protected
25 * callback methods. <em>Note:</em> these methods are all called arbitrary download threads, so
26 * implementors must take care to only execute thread-safe code or simply pass a message to the AWT
27 * thread, for example.
29 public abstract class Downloader
32 * Start the downloading process.
33 * @param resources the resources to download.
34 * @param maxConcurrent the maximum number of concurrent downloads allowed.
35 * @return true if the download completed, false if it was aborted (via {@link #abort}).
37 public boolean download (Collection<Resource> resources, int maxConcurrent)
39 // first compute the total size of our download
41 for (Resource rsrc : resources) {
43 _sizes.put(rsrc, Math.max(checkSize(rsrc), 0L));
44 } catch (IOException ioe) {
45 downloadFailed(rsrc, ioe);
49 long totalSize = sum(_sizes.values());
50 log.info("Downloading " + resources.size() + " resources",
51 "totalBytes", totalSize, "maxConcurrent", maxConcurrent);
53 // make a note of the time at which we started the download
54 _start = System.currentTimeMillis();
56 // start the downloads
57 ExecutorService exec = Executors.newFixedThreadPool(maxConcurrent);
58 for (final Resource rsrc : resources) {
59 // make sure the resource's target directory exists
60 File parent = new File(rsrc.getLocal().getParent());
61 if (!parent.exists() && !parent.mkdirs()) {
62 log.warning("Failed to create target directory for resource '" + rsrc + "'.");
65 exec.execute(new Runnable() {
66 @Override public void run () {
68 if (_state != State.ABORTED) {
71 } catch (IOException ioe) {
72 _state = State.FAILED;
73 downloadFailed(rsrc, ioe);
80 // wait for the downloads to complete
82 exec.awaitTermination(10, TimeUnit.DAYS);
84 // report download completion if we did not already do so via our final resource
85 if (_state == State.DOWNLOADING) {
86 downloadProgress(100, 0);
89 } catch (InterruptedException ie) {
91 downloadFailed(null, ie);
94 return _state != State.ABORTED;
98 * Aborts the in-progress download.
100 public void abort () {
101 _state = State.ABORTED;
105 * Called before the downloader begins the series of HTTP head requests to determine the
106 * size of the files it needs to download.
108 protected void resolvingDownloads () {}
111 * Reports ongoing progress toward completion of the overall downloading task. One call is
112 * guaranteed to be made reporting 100% completion if the download is not aborted and no
115 * @param percent the percent completion of the complete download process (based on total bytes
116 * downloaded versus total byte size of all resources).
117 * @param remaining the estimated download time remaining in seconds, or {@code -1} if the time
118 * can not yet be determined.
120 protected void downloadProgress (int percent, long remaining) {}
123 * Called if a failure occurs while downloading a resource. No progress will be reported after
124 * a download fails, but additional download failures may be reported.
126 * @param rsrc the resource that failed to download, or null if the download failed due to
127 * thread interruption.
128 * @param cause the exception detailing the failure.
130 protected void downloadFailed (Resource rsrc, Exception cause) {}
133 * Performs the protocol-specific portion of checking download size.
135 protected abstract long checkSize (Resource rsrc) throws IOException;
138 * Periodically called by the protocol-specific downloaders to update their progress. This
139 * should be called at least once for each resource to be downloaded, with the total downloaded
140 * size for that resource. It can also be called periodically along the way for each resource
141 * to communicate incremental progress.
143 * @param rsrc the resource currently being downloaded.
144 * @param currentSize the number of bytes currently downloaded for said resource.
145 * @param actualSize the size reported for this resource now that we're actually downloading
146 * it. Some web servers lie about Content-length when doing a HEAD request, so by reporting
147 * updated sizes here we can recover from receiving bogus information in the earlier
148 * {@link #checkSize} phase.
150 protected synchronized void reportProgress (Resource rsrc, long currentSize, long actualSize)
152 // update the actual size for this resource (but don't let it shrink)
153 _sizes.put(rsrc, actualSize = Math.max(actualSize, _sizes.get(rsrc)));
155 // update the current downloaded size for said resource; don't allow the downloaded bytes
156 // to exceed the original claimed size of the resource, otherwise our progress will get
157 // booched and we'll end up back on the Daily WTF: http://tinyurl.com/29wt4oq
158 _downloaded.put(rsrc, Math.min(actualSize, currentSize));
160 // notify the observer if it's been sufficiently long since our last notification
161 long now = System.currentTimeMillis();
162 if ((now - _lastUpdate) >= UPDATE_DELAY) {
165 // total up our current and total bytes
166 long downloaded = sum(_downloaded.values());
167 long totalSize = sum(_sizes.values());
169 // compute our bytes per second
170 long secs = (now - _start) / 1000L;
171 long bps = (secs == 0) ? 0 : (downloaded / secs);
173 // compute our percentage completion
174 int pctdone = (totalSize == 0) ? 0 : (int)((downloaded * 100f) / totalSize);
176 // estimate our time remaining
177 long remaining = (bps <= 0 || totalSize == 0) ? -1 : (totalSize - downloaded) / bps;
179 // if we're complete or failed, when we don't want to report again
180 if (_state == State.DOWNLOADING) {
181 if (pctdone == 100) _state = State.COMPLETE;
182 downloadProgress(pctdone, remaining);
188 * Sums the supplied values.
190 protected static long sum (Iterable<Long> values)
193 for (Long value : values) {
199 protected enum State { DOWNLOADING, COMPLETE, FAILED, ABORTED }
202 * Accomplishes the copying of the resource from remote location to local location using
203 * protocol-specific code. This method should periodically check whether {@code _state} is set
204 * to aborted and abort any in-progress download if so.
206 protected abstract void download (Resource rsrc) throws IOException;
208 /** The reported sizes of our resources. */
209 protected Map<Resource, Long> _sizes = new HashMap<>();
211 /** The bytes downloaded for each resource. */
212 protected Map<Resource, Long> _downloaded = new HashMap<>();
214 /** The time at which the file transfer began. */
215 protected long _start;
217 /** The current transfer rate in bytes per second. */
218 protected long _bytesPerSecond;
220 /** The time at which the last progress update was posted to the progress observer. */
221 protected long _lastUpdate;
223 /** A wee state machine to ensure we call our callbacks sanely. */
224 protected volatile State _state = State.DOWNLOADING;
226 /** The delay in milliseconds between notifying progress observers of file download
228 protected static final long UPDATE_DELAY = 500L;