Merge pull request #338 from Kong/progres

Add download progress monitor

Author: ryber

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.6.00 (pending)
+* issue #336 Add ProgressMonitor for file downloads.
+
 ## 3.5.00
 * Re-package the object mapper sub-modules to work with Java 11 per issue #324.
 * Update Jackson to 2.10.2

build/suppressions.xml

@@ -6,6 +6,7 @@
 <suppressions>
     <suppress checks="MethodCount" files="Config.java"/>
     <suppress checks="MethodCount" files="HttpRequest.java"/>
+    <suppress checks="MethodCount" files="BaseRequest.java"/>
     <suppress checks="MethodCount" files="JSONArray.java"/>
     <suppress checks="MethodCount" files="JSONObject.java"/>
     <suppress checks="FileLength" files="JSONObject.java"/>

docs/index.md

@@ -394,6 +394,17 @@ File result = Unirest.get("http://some.file.location/file.zip")
                 .getBody();
 ```
 
+### Download Progress Monitoring
+If you are uploading large files you might want to provide some time of progress bar to a user. You can monitor this progress by providing a ProgresMonitor.
+
+```java
+          Unirest.get("http://httpbin.org")
+                .downLoadMonitor((b, fileName, bytesWritten, totalBytes) -> {
+                    updateProgressBarWithBytesLeft(totalBytes - bytesWritten);
+                })
+                .asFile("/disk/location/file.zip");
+```
+
 ## JSON responses
 Unirest offers a lightweight JSON response type when you don't need a full Object Mapper.
 

unirest/src/main/java/kong/unirest/BaseRequest.java

@@ -45,6 +45,7 @@ abstract class BaseRequest<R extends HttpRequest> implements HttpRequest<R> {
     private Integer socketTimeout;
     private Integer connectTimeout;
     private Proxy proxy;
+    private ProgressMonitor downloadMonitor;
 
     BaseRequest(BaseRequest httpRequest) {
         this.config = httpRequest.config;
@@ -170,6 +171,12 @@ public R withObjectMapper(ObjectMapper mapper) {
         return (R)this;
     }
 
+    @Override
+    public R downloadMonitor(ProgressMonitor monitor) {
+        this.downloadMonitor = monitor;
+        return (R)this;
+    }
+
     @Override
     public HttpResponse asEmpty() {
         return config.getClient().request(this, EmptyResponse::new);
@@ -298,17 +305,17 @@ public void thenConsumeAsync(Consumer<RawResponse> consumer) {
 
     @Override
     public HttpResponse<File> asFile(String path) {
-        return config.getClient().request(this, r -> new FileResponse(r, path));
+        return config.getClient().request(this, r -> new FileResponse(r, path, downloadMonitor));
     }
 
     @Override
     public CompletableFuture<HttpResponse<File>> asFileAsync(String path) {
-        return config.getAsyncClient().request(this, r -> new FileResponse(r, path), new CompletableFuture<>());
+        return config.getAsyncClient().request(this, r -> new FileResponse(r, path, downloadMonitor), new CompletableFuture<>());
     }
 
     @Override
     public CompletableFuture<HttpResponse<File>> asFileAsync(String path, Callback<File> callback) {
-        return config.getAsyncClient().request(this, r -> new FileResponse(r, path), wrap(callback));
+        return config.getAsyncClient().request(this, r -> new FileResponse(r, path, downloadMonitor), wrap(callback));
     }
 
     @Override

unirest/src/main/java/kong/unirest/FileResponse.java

@@ -27,24 +27,33 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 public class FileResponse extends BaseResponse<File> {
     private File body;
 
-    public FileResponse(RawResponse r, String path) {
+    public FileResponse(RawResponse r, String path, ProgressMonitor downloadMonitor) {
         super(r);
         try {
             Path target = Paths.get(path);
-            Files.copy(r.getContent(), target);
+            InputStream content = getContent(r, downloadMonitor, target);
+            Files.copy(content, target);
             body = target.toFile();
         } catch (IOException e) {
             throw new UnirestException(e);
         }
     }
 
+    private InputStream getContent(RawResponse r, ProgressMonitor downloadMonitor, Path target) {
+        if(downloadMonitor == null){
+            return r.getContent();
+        }
+        return new MonitoringInputStream(r.getContent(), downloadMonitor, target, r);
+    }
+
     @Override
     public File getBody() {
         return body;

unirest/src/main/java/kong/unirest/HttpRequest.java

@@ -178,6 +178,13 @@ public interface HttpRequest<R extends HttpRequest>  {
      */
     R proxy(String host, int port);
 
+    /**
+     * sets a download monitor for monitoring the response. this could be used for drawing a progress bar
+     * @param monitor a ProgressMonitor
+     * @return
+     */
+    R downloadMonitor(ProgressMonitor monitor);
+
     /**
      * Executes the request and returns the response with the body mapped into a String
      * @return response

unirest/src/main/java/kong/unirest/MonitoringInputStream.java

@@ -0,0 +1,107 @@
+/**
+ * The MIT License
+ *
+ * Copyright for portions of unirest-java are held by Kong Inc (c) 2013.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package kong.unirest;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+
+class MonitoringInputStream extends InputStream {
+    private final InputStream content;
+    private final ProgressMonitor downloadMonitor;
+    private long totalSize;
+    private long byteCount = 0;
+    private String fileName;
+
+    MonitoringInputStream(InputStream content, ProgressMonitor downloadMonitor, Path target, RawResponse contentSize) {
+        this.content = content;
+        this.downloadMonitor = downloadMonitor;
+        this.fileName = target.getFileName().toString();
+        this.totalSize = getBodySize(contentSize);
+    }
+
+    private Long getBodySize(RawResponse r) {
+        String header = r.getHeaders().getFirst("Content-Length");
+        if (header != null && header.length() > 0) {
+            return Long.valueOf(header);
+        }
+        return 0L;
+    }
+
+    @Override
+    public int read() throws IOException {
+        return content.read();
+    }
+
+    @Override
+    public int read(byte[] b) throws IOException {
+        int read = super.read(b);
+        monitor(read);
+        return read;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int read = super.read(b, off, len);
+        monitor(read);
+        return read;
+    }
+
+    private void monitor(int bytesRead) {
+        byteCount = byteCount + bytesRead;
+        downloadMonitor.accept("body", fileName, byteCount, totalSize);
+    }
+
+    @Override
+    public void close() throws IOException {
+        content.close();
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+        return  content.skip(n);
+    }
+
+    @Override
+    public int available() throws IOException {
+        return content.available();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+        content.mark(readlimit);
+    }
+
+    @Override
+    public boolean markSupported() {
+        return content.markSupported();
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+        content.reset();
+    }
+}

unirest/src/main/java/kong/unirest/ProgressMonitor.java

@@ -27,21 +27,24 @@
 
 /**
  * A ProgressMonitor is a functional interface which can be passed to unirest for the purposes of
- * monitoring file uploads. A common use case is for drawing upload progress bars.
- * If the upload contains multiple files each one is called individually and the file name is provided.
+ * monitoring file uploads and downloads. A common use case is for drawing progress bars.
+ *
+ * If an upload contains multiple files each one is called individually and the file name is provided.
  *
  * note that you will not receive a total for ALL files together at once.
- * If you wanted this you could keep track of the total bytes of files you planned to upload and then
+ * If you wanted this you can keep track of the total bytes of files you planned to upload and then
  * have your ProgressMonitor aggregate the results.
  */
 @FunctionalInterface
 public interface ProgressMonitor {
     /**
      * Accept stats about the current file upload chunk for a file.
-     * @param field the field name
+     * @param field the field name, or 'body' on file downloads
      * @param fileName the name of the file in question if available (InputStreams and byte arrays may not have file names)
-     * @param bytesWritten the number of bytes that have been uploaded so far
-     * @param totalBytes the total bytes that will be uploaded. Note this this may be an estimate if an InputStream was used
+     * @param bytesWritten the number of bytes that have been uploaded or downloaded so far
+     * @param totalBytes the total bytes that will be uploaded or downloaded.
+     *                    On downloads this depends on the Content-Length header be returned
+     *                    On uploads this this may be an estimate if an InputStream was used
      * */
     void accept(String field, String fileName, Long bytesWritten, Long totalBytes);
 }

unirest/src/test/java/BehaviorTests/DownloadProgressTest.java

@@ -32,23 +32,22 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 
 import static kong.unirest.TestUtil.rezFile;
+import static org.junit.Assert.assertTrue;
 
 public class DownloadProgressTest extends BddTest {
     @Rule
     public TemporaryFolder disk = new TemporaryFolder();
     private File targeFolder;
 
     private TestMonitor monitor;
-    private File spidey;
 
     @Override
     public void setUp() {
         super.setUp();
         this.monitor = new TestMonitor();
-        spidey = rezFile("/spidey.jpg");
-
         try {
             targeFolder = disk.newFolder("test");
         } catch (IOException e) {
@@ -58,11 +57,21 @@ public void setUp() {
 
     @Test
     public void canAddUploadProgress() {
-        Unirest.post(MockServer.BINARYFILE)
-                //.uploadMonitor(monitor)
-                .asFile(targeFolder.getPath());
+        Unirest.get(MockServer.BINARYFILE)
+                .downloadMonitor(monitor)
+                .asFile(targeFolder.getPath() + "/spidey.jpg");
+
+        monitor.assertSpideyFileDownload("spidey.jpg");
+    }
+
+    @Test
+    public void canAddUploadProgressAsync() throws Exception {
+        Unirest.get(MockServer.BINARYFILE)
+                .downloadMonitor(monitor)
+                .asFileAsync(targeFolder.getPath() + "/spidey.jpg")
+                .get();
 
-        //assertSpideyFileUpload("spidey.jpg");
+        monitor.assertSpideyFileDownload("spidey.jpg");
     }
 
 }

unirest/src/test/java/BehaviorTests/MockServer.java

@@ -168,6 +168,7 @@ private static Object file(Request request, Response response) throws Exception
 		 File f = TestUtil.rezFile("/spidey.jpg");
 		 response.raw().setContentType("application/octet-stream");
 		 response.raw().setHeader("Content-Disposition", "attachment;filename=image.jpg");
+		 response.raw().setHeader("Content-Length", String.valueOf(f.length()));
 		 response.status(200);
 		 final ServletOutputStream out = response.raw().getOutputStream();
 		 final FileInputStream in = new FileInputStream(f);

unirest/src/test/java/BehaviorTests/TestMonitor.java

@@ -74,10 +74,25 @@ public void assertSpideyFileUpload() {
 
     public void assertSpideyFileUpload(String name) {
         Stats stat = get(name);
+        assertEquals("Wrong Expected Length", spidey.length(), stat.total);
         assertEquals(12, stat.timesCalled);
-        assertEquals(asList(4096L, 8192L, 12288L, 16384L, 20480L, 24576L, 28672L,
-                32768L, 36864L, 40960L, 45056L, 46246L), stat.progress);
-        assertEquals(spidey.length(), stat.total);
+        assertEquals(asList(
+                4096L, 8192L, 12288L, 16384L,
+                20480L, 24576L, 28672L, 32768L,
+                36864L, 40960L, 45056L, 46246L),
+                stat.progress);
+    }
+
+    public void assertSpideyFileDownload(String name) {
+        Stats stat = get(name);
+        assertEquals("Wrong Expected Length", spidey.length(), stat.total);
+        assertEquals(14, stat.timesCalled);
+        assertEquals(asList(
+                8192L, 16384L, 24576L, 32768L,
+                40960L, 49152L, 57344L, 65536L,
+                73728L, 81920L, 87206L, 92492L,
+                92491L, 92490L),
+                stat.progress);
     }
 
     static class Stats {