Report percentual download progress in repository rules

If the HTTP response to a download request contains a `Content-Length` header, download progress is now reported as `10.1 MiB (20.2%)` instead of `10.1 MiB (10,590,000B)`.

Closes #18450.

PiperOrigin-RevId: 534035444
Change-Id: I1c5144555eda1890652b4d3f62b414292ba909d5

Author: fmeum

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadProgressEvent.java

@@ -17,6 +17,10 @@
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
 import com.google.devtools.build.lib.remote.util.Utils;
 import java.net.URL;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+import java.util.OptionalLong;
 
 /**
  * Postable event reporting on progress made downloading an URL. It can be used to report the URL
@@ -26,17 +30,20 @@ public class DownloadProgressEvent implements ExtendedEventHandler.FetchProgress
   private final URL originalUrl;
   private final URL actualUrl;
   private final long bytesRead;
+  private final OptionalLong totalBytes;
   private final boolean downloadFinished;
 
-  public DownloadProgressEvent(URL originalUrl, URL actualUrl, long bytesRead, boolean finished) {
+  public DownloadProgressEvent(
+      URL originalUrl, URL actualUrl, long bytesRead, OptionalLong totalBytes, boolean finished) {
     this.originalUrl = originalUrl;
     this.actualUrl = actualUrl;
     this.bytesRead = bytesRead;
+    this.totalBytes = totalBytes;
     this.downloadFinished = finished;
   }
 
   public DownloadProgressEvent(URL originalUrl, long bytesRead, boolean finished) {
-    this(originalUrl, null, bytesRead, finished);
+    this(originalUrl, null, bytesRead, OptionalLong.empty(), finished);
   }
 
   public DownloadProgressEvent(URL url, long bytesRead) {
@@ -69,10 +76,22 @@ public long getBytesRead() {
     return bytesRead;
   }
 
+  private static final DecimalFormat PERCENTAGE_FORMAT =
+      new DecimalFormat("0.0%", new DecimalFormatSymbols(Locale.US));
+
   @Override
   public String getProgress() {
     if (bytesRead > 0) {
-      return String.format("%s (%,dB)", Utils.bytesCountToDisplayString(bytesRead), bytesRead);
+      if (totalBytes.isPresent()) {
+        double totalBytesDouble = this.totalBytes.getAsLong();
+        double ratio = totalBytesDouble != 0 ? bytesRead / totalBytesDouble : 1;
+        // 10.1 MiB (20.2%)
+        return String.format(
+            "%s (%s)", Utils.bytesCountToDisplayString(bytesRead), PERCENTAGE_FORMAT.format(ratio));
+      } else {
+        // 10.1 MiB (10,590,000B)
+        return String.format("%s (%,dB)", Utils.bytesCountToDisplayString(bytesRead), bytesRead);
+      }
     } else {
       return "";
     }

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java

@@ -30,6 +30,7 @@
 import java.io.SequenceInputStream;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.OptionalLong;
 import java.util.zip.GZIPInputStream;
 import javax.annotation.WillCloseWhenClosed;
 
@@ -87,17 +88,19 @@ HttpStream create(
           stream = retrier;
         }
 
+        OptionalLong totalBytes = OptionalLong.empty();
         try {
           String contentLength = connection.getHeaderField("Content-Length");
           if (contentLength != null) {
-            long expectedSize = Long.parseLong(contentLength);
-            stream = new CheckContentLengthInputStream(stream, expectedSize);
+            totalBytes = OptionalLong.of(Long.parseUnsignedLong(contentLength));
+            stream = new CheckContentLengthInputStream(stream, totalBytes.getAsLong());
           }
         } catch (NumberFormatException ignored) {
           // ignored
         }
 
-        stream = progressInputStreamFactory.create(stream, connection.getURL(), originalUrl);
+        stream =
+            progressInputStreamFactory.create(stream, connection.getURL(), originalUrl, totalBytes);
 
         // Determine if we need to transparently gunzip. See RFC2616 § 3.5 and § 14.11. Please note
         // that some web servers will send Content-Encoding: gzip even when we didn't request it if

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java

@@ -24,6 +24,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.util.Locale;
+import java.util.OptionalLong;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.WillCloseWhenClosed;
 
@@ -50,9 +51,20 @@ static class Factory {
       this.eventHandler = eventHandler;
     }
 
-    InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL originalUrl) {
+    InputStream create(
+        @WillCloseWhenClosed InputStream delegate,
+        URL url,
+        URL originalUrl,
+        OptionalLong totalBytes) {
       return new ProgressInputStream(
-          locale, clock, eventHandler, PROGRESS_INTERVAL_MS, delegate, url, originalUrl);
+          locale,
+          clock,
+          eventHandler,
+          PROGRESS_INTERVAL_MS,
+          delegate,
+          url,
+          originalUrl,
+          totalBytes);
     }
   }
 
@@ -63,6 +75,7 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
   private final long intervalMs;
   private final URL url;
   private final URL originalUrl;
+  private final OptionalLong totalBytes;
   private final AtomicLong toto = new AtomicLong();
   private final AtomicLong nextEvent;
 
@@ -73,7 +86,8 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
       long intervalMs,
       InputStream delegate,
       URL url,
-      URL originalUrl) {
+      URL originalUrl,
+      OptionalLong totalBytes) {
     Preconditions.checkArgument(intervalMs >= 0);
     this.locale = locale;
     this.clock = clock;
@@ -82,8 +96,9 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
     this.delegate = delegate;
     this.url = url;
     this.originalUrl = originalUrl;
+    this.totalBytes = totalBytes;
     this.nextEvent = new AtomicLong(clock.currentTimeMillis() + intervalMs);
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, 0, false));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, 0, totalBytes, false));
   }
 
   @Override
@@ -112,7 +127,7 @@ public int available() throws IOException {
   @Override
   public void close() throws IOException {
     delegate.close();
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, toto.get(), true));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, toto.get(), totalBytes, true));
   }
 
   private void reportProgress(long bytesRead) {
@@ -124,7 +139,7 @@ private void reportProgress(long bytesRead) {
     if (!url.getHost().equals(originalUrl.getHost())) {
       via = " via " + url.getHost();
     }
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, bytesRead, false));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, bytesRead, totalBytes, false));
     eventHandler.handle(
         Event.progress(
             String.format(locale, "Downloading %s%s: %,d bytes", originalUrl, via, bytesRead)));

src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java

@@ -93,7 +93,7 @@ public void before() throws Exception {
     nRetries = 0;
 
     when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(data));
-    when(progress.create(any(InputStream.class), any(), any(URL.class)))
+    when(progress.create(any(InputStream.class), any(), any(URL.class), any()))
         .thenAnswer(
             new Answer<InputStream>() {
               @Override

src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java

@@ -37,6 +37,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.util.Locale;
+import java.util.OptionalLong;
 import org.junit.After;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -53,7 +54,8 @@ public class ProgressInputStreamTest {
   private final InputStream delegate = mock(InputStream.class);
   private final URL url = makeUrl("http://lol.example");
   private ProgressInputStream stream =
-      new ProgressInputStream(Locale.US, clock, extendedEventHandler, 1, delegate, url, url);
+      new ProgressInputStream(
+          Locale.US, clock, extendedEventHandler, 1, delegate, url, url, OptionalLong.empty());
 
   @After
   public void after() throws Exception {
@@ -126,7 +128,15 @@ public void bufferReadsAfterInterval_emitsProgressOnce() throws Exception {
   @Test
   public void bufferReadsAfterIntervalInGermany_usesPeriodAsSeparator() throws Exception {
     stream =
-        new ProgressInputStream(Locale.GERMANY, clock, extendedEventHandler, 1, delegate, url, url);
+        new ProgressInputStream(
+            Locale.GERMANY,
+            clock,
+            extendedEventHandler,
+            1,
+            delegate,
+            url,
+            url,
+            OptionalLong.empty());
     byte[] buffer = new byte[1024];
     when(delegate.read(any(byte[].class), anyInt(), anyInt())).thenReturn(1024);
     clock.advanceMillis(1);
@@ -145,14 +155,30 @@ public void redirectedToDifferentServer_showsOriginalUrlWithVia() throws Excepti
             1,
             delegate,
             new URL("http://cdn.example/foo"),
-            url);
+            url,
+            OptionalLong.empty());
     when(delegate.read()).thenReturn(42);
     assertThat(stream.read()).isEqualTo(42);
     clock.advanceMillis(1);
     assertThat(stream.read()).isEqualTo(42);
     assertThat(stream.read()).isEqualTo(42);
     verify(delegate, times(3)).read();
-    verify(eventHandler).handle(
-        Event.progress("Downloading http://lol.example via cdn.example: 2 bytes"));
+    verify(eventHandler)
+        .handle(Event.progress("Downloading http://lol.example via cdn.example: 2 bytes"));
+  }
+
+  @Test
+  public void percentualProgress() {
+    DownloadProgressEvent event =
+        new DownloadProgressEvent(
+            url, url, 25 * 1024 * 1024, OptionalLong.of(100 * 1024 * 1024), false);
+    assertThat(event.getProgress()).isEqualTo("25.0 MiB (25.0%)");
+  }
+
+  @Test
+  public void percentualProgress_zeroTotalBytes() {
+    DownloadProgressEvent event =
+        new DownloadProgressEvent(url, url, 25 * 1024 * 1024, OptionalLong.of(0), false);
+    assertThat(event.getProgress()).isEqualTo("25.0 MiB (100.0%)");
   }
 }