[Java] FPS support for local grid using bidi

visual-java/pom.xml

@@ -57,7 +57,7 @@
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
       <artifactId>selenium-java</artifactId>
-      <version>4.10.0</version>
+      <version>4.32.0</version>
       <scope>provided</scope>
     </dependency>
     <dependency>

visual-java/src/main/java/com/saucelabs/visual/VisualApi.java

@@ -14,7 +14,6 @@
 import com.saucelabs.visual.utils.*;
 import dev.failsafe.Failsafe;
 import dev.failsafe.RetryPolicy;
-import java.awt.image.BufferedImage;
 import java.lang.reflect.Field;
 import java.net.URL;
 import java.time.Duration;
@@ -24,6 +23,8 @@
 import java.util.stream.Collectors;
 import org.apache.http.client.config.RequestConfig;
 import org.openqa.selenium.*;
+import org.openqa.selenium.bidi.browsingcontext.BrowsingContext;
+import org.openqa.selenium.bidi.browsingcontext.CaptureScreenshotParameters;
 import org.openqa.selenium.remote.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -617,36 +618,53 @@ public <D> D execute(GraphQLOperation operation, Class<D> responseType) {
   }
 
   private String sauceVisualCheckLocal(String snapshotName, CheckOptions options) {
+    Capabilities caps = driver.getCapabilities();
     Window window = new Window(this.driver);
-    Rectangle viewport = window.getViewport();
-
-    byte[] screenshot;
-
-    // clip image if required
+    Map<String, Object> windowDims =
+        (Map<String, Object>)
+            driver.executeScript(
+                "return { height: window.innerHeight, width: window.innerWidth, dpr: window.devicePixelRatio }");
+    double devicePixelRatio = formatDevicePixelRatio(windowDims.get("dpr"));
+    boolean fullPage = options.getFullPageScreenshotConfig() != null;
     WebElement clipElement = getClipElement(options);
+    Point scrollOffset = new Point(0, 0);
+    Point clipOffset = new Point(0, 0);
+    Rectangle clipRect = null;
+
     if (clipElement != null) {
-      Rectangle clipRect = clipElement.getRect();
+      Rectangle clipElementDims = getClipElementDimensions(fullPage, clipElement);
+
+      if (!fullPage) {
+        scrollOffset = window.getViewport().getPoint();
+      }
+
+      clipRect =
+          new Rectangle(
+              (int) Math.round(clipElementDims.getX() * devicePixelRatio),
+              (int) Math.round(clipElementDims.getY() * devicePixelRatio),
+              (int) Math.round(clipElementDims.getHeight() * devicePixelRatio),
+              (int) Math.round(clipElementDims.getWidth() * devicePixelRatio));
+
+      clipOffset = new Point(clipElementDims.getX(), clipElementDims.getY());
+    }
 
-      // Scroll to the clipped element
-      Rectangle newViewport = window.scrollTo(clipRect.getPoint());
-      screenshot = driver.getScreenshotAs(OutputType.BYTES);
+    byte[] screenshot = getScreenshot(fullPage);
 
-      // Restore the original scroll
-      window.scrollTo(viewport.getPoint());
+    // Handle clipping, if rect is present
+    if (clipRect != null) {
+      Dimension imageDims = ImageHelpers.getImageDimensions(screenshot);
+      // bound the clip rect to the image bounds so there is no outside of bounds exception
+      Optional<Rectangle> offsetRect =
+          CartesianHelpers.intersect(clipRect, new Rectangle(new Point(0, 0), imageDims));
 
-      Optional<Rectangle> cropRect = CartesianHelpers.intersect(clipRect, newViewport);
-      if (!cropRect.isPresent()) {
+      if (offsetRect.isPresent()) {
+        screenshot =
+            ImageHelpers.saveImage(
+                ImageHelpers.cropImage(ImageHelpers.loadImage(screenshot), offsetRect.get()),
+                "png");
+      } else {
         throw new VisualApiException("Clipping would result in an empty image");
       }
-
-      BufferedImage image = ImageHelpers.loadImage(screenshot);
-      BufferedImage cropped =
-          ImageHelpers.cropImage(
-              image, CartesianHelpers.relativeTo(newViewport.getPoint(), cropRect.get()));
-      screenshot = ImageHelpers.saveImage(cropped, "png");
-      viewport = cropRect.get();
-    } else {
-      screenshot = driver.getScreenshotAs(OutputType.BYTES);
     }
 
     // create upload and get urls
@@ -663,19 +681,30 @@ private String sauceVisualCheckLocal(String snapshotName, CheckOptions options)
             Optional.ofNullable(options.getIgnoreElements()).orElse(Collections.emptyList())));
     ignoreRegions.addAll(extractIgnoreSelectors(options));
 
+    // get adjusted screenshot dimensions
+    Dimension imageDims = ImageHelpers.getImageDimensions(screenshot);
+    Rectangle imageDimsDpr =
+        new Rectangle(
+            new Point(0, 0),
+            new Dimension(
+                (int) Math.round(imageDims.getWidth() / devicePixelRatio),
+                (int) Math.round(imageDims.getHeight() / devicePixelRatio)));
+    String deviceName =
+        String.format("Desktop (%sx%s)", windowDims.get("width"), windowDims.get("height"));
+
     // make regions relative to viewport
     List<RegionIn> visibleIgnoreRegions = new ArrayList<>();
     for (RegionIn region : ignoreRegions) {
+      // Update the offset to all ignore regions
+      Point newPoint =
+          CartesianHelpers.relativeTo(clipOffset, new Point(region.getX(), region.getY()));
+      region.setX(newPoint.x - scrollOffset.getX());
+      region.setY(newPoint.y - scrollOffset.getY());
+
       Rectangle regionRect =
           new Rectangle(region.getX(), region.getY(), region.getHeight(), region.getWidth());
-
-      if (CartesianHelpers.intersect(regionRect, viewport).isPresent()) {
-        Point newPoint =
-            CartesianHelpers.relativeTo(
-                viewport.getPoint(), new Point(region.getX(), region.getY()));
-        region.setX(newPoint.x);
-        region.setY(newPoint.y);
-
+      // Only add it if the region is visible inside the new image
+      if (CartesianHelpers.intersect(regionRect, imageDimsDpr).isPresent()) {
         visibleIgnoreRegions.add(region);
       }
     }
@@ -695,22 +724,14 @@ private String sauceVisualCheckLocal(String snapshotName, CheckOptions options)
       }
     }
 
-    Capabilities caps = driver.getCapabilities();
-
-    Map<String, Object> dims =
-        (Map<String, Object>)
-            driver.executeScript(
-                "return { height: window.innerHeight, width: window.innerWidth, dpr: window.devicePixelRatio }");
-    String deviceName = String.format("Desktop (%sx%s)", dims.get("width"), dims.get("height"));
-
     // create snapshot using upload id
     CreateSnapshotMutation snapshotMutation =
         new CreateSnapshotMutation(
             SnapshotIn.builder()
                 .withBrowser(CapabilityUtils.getBrowser(caps))
                 .withBrowserVersion(caps.getBrowserVersion())
                 .withDevice(deviceName)
-                .withDevicePixelRatio(formatDevicePixelRatio(dims.get("dpr")))
+                .withDevicePixelRatio(devicePixelRatio)
                 .withOperatingSystem(CapabilityUtils.getOperatingSystem(caps))
                 .withUploadId(uploadResult.getId())
                 .withBuildId(this.build.getId())
@@ -734,6 +755,51 @@ private String sauceVisualCheckLocal(String snapshotName, CheckOptions options)
     return result.result.getId();
   }
 
+  private byte[] getScreenshot(boolean fullPage) {
+    if (fullPage) {
+      BrowsingContext browsingContext = new BrowsingContext(driver, driver.getWindowHandle());
+      CaptureScreenshotParameters csp = new CaptureScreenshotParameters();
+      csp.origin(CaptureScreenshotParameters.Origin.DOCUMENT);
+      String screenshot = browsingContext.captureScreenshot(csp);
+      return OutputType.BYTES.convertFromBase64Png(screenshot);
+    }
+
+    return driver.getScreenshotAs(OutputType.BYTES);
+  }
+
+  /** Query the browser for the dimensions and offset of the current clip element. */
+  private Rectangle getClipElementDimensions(boolean fullPage, WebElement clipElement) {
+    final String getElementDimensions =
+        "const [clipElement, isFullPage] = arguments;\n"
+            + "      if (!isFullPage) {\n"
+            + "        clipElement.scrollIntoView({\n"
+            + "          behavior: 'instant',\n"
+            + "        });\n"
+            + "      }\n"
+            + "      const rect = clipElement.getBoundingClientRect().toJSON();\n"
+            + "      return isFullPage ? {\n"
+            + "        // FPS should use original coordinates of element on page\n"
+            + "        width: Math.round(rect.width),\n"
+            + "        height: Math.round(rect.height),\n"
+            + "        x: Math.round(rect.x + window.scrollX),\n"
+            + "        y: Math.round(rect.y + window.scrollY),\n"
+            + "      } : {"
+            + "        width: Math.round(rect.width),\n"
+            + "        height: Math.round(rect.height),\n"
+            + "        x: Math.round(rect.x),\n"
+            + "        y: Math.round(rect.y),\n"
+            + "      };";
+
+    Map<String, Long> clipMap =
+        (Map<String, Long>) driver.executeScript(getElementDimensions, clipElement, fullPage);
+
+    return new Rectangle(
+        clipMap.get("x").intValue(),
+        clipMap.get("y").intValue(),
+        clipMap.get("height").intValue(),
+        clipMap.get("width").intValue());
+  }
+
   /** Parse the Selenium parsed value from window.devicePixelRatio into a Double for our API */
   private double formatDevicePixelRatio(Object rawDpr) {
     double dpr = 1.0;

visual-java/src/main/java/com/saucelabs/visual/utils/ImageHelpers.java

@@ -6,6 +6,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import javax.imageio.ImageIO;
+import org.openqa.selenium.Dimension;
 import org.openqa.selenium.Rectangle;
 
 public class ImageHelpers {
@@ -31,4 +32,9 @@ public static byte[] saveImage(BufferedImage image, String imageFormat) {
       throw new VisualApiException("Failed to save image to bytes", e);
     }
   }
+
+  public static Dimension getImageDimensions(byte[] image) {
+    BufferedImage bImage = loadImage(image);
+    return new Dimension(bImage.getWidth(), bImage.getHeight());
+  }
 }

visual-java/src/main/java/com/saucelabs/visual/utils/Window.java

@@ -25,7 +25,7 @@ public Rectangle scrollTo(Point point) {
   }
 
   private final String jsViewportTupleSnippet =
-      "return [window.scrollX, window.scrollY, window.innerWidth, window.innerHeight];";
+      "return [Math.round(window.scrollX), Math.round(window.scrollY), Math.round(window.innerWidth), Math.round(window.innerHeight)];";
 
   private Rectangle convertJsTupleToRectangle(Object result) {
     if (!(result instanceof List<?>)) {

visual-java/src/test/java/com/saucelabs/visual/integration/IntegrationBase.java

@@ -21,6 +21,7 @@
 import org.openqa.selenium.MutableCapabilities;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.firefox.FirefoxDriver;
+import org.openqa.selenium.firefox.FirefoxOptions;
 import org.openqa.selenium.remote.RemoteWebDriver;
 
 abstract class IntegrationBase {
@@ -59,7 +60,10 @@ public static void setUp() throws MalformedURLException {
     Map<String, Object> sauceOptions = new HashMap<>();
     sauceOptions.put("screenResolution", "1600x1200");
     caps.setCapability("sauce:options", sauceOptions);
-    driver = isLocal ? new FirefoxDriver() : new RemoteWebDriver(getDriverUrl(), caps);
+    FirefoxOptions firefoxOptions = new FirefoxOptions();
+    firefoxOptions.setCapability("webSocketUrl", true);
+    driver =
+        isLocal ? new FirefoxDriver(firefoxOptions) : new RemoteWebDriver(getDriverUrl(), caps);
     driver.manage().window().setSize(new Dimension(1400, 1100));
     visual =
         new VisualApi.Builder(driver, username, accessKey)