Incomplete Changes

Author: PulseBeat02

TODO

@@ -1,5 +1,7 @@
 Development Milestones
+  - Fix Interaction Support for Virtual Machine / Browser
+  - Look at PixelStacker (https://github.com/Pangamma/PixelStacker)
+  - Add Images and GIF Support into Plugin
+
   - Build QEMU Statically for All Platforms
-  - Add Interaction Support for Virtual Machine
-  - Add Interaction Support for Browser
-  - Add Images and GIF Support into Plugin
+  - Rewrite all JavaDocs

mcav-bukkit/src/main/java/me/brandonli/mcav/bukkit/utils/InteractUtils.java

@@ -108,6 +108,15 @@ default CompletableFuture<Boolean> startAsync(final VideoPipelineStep videoPipel
    */
   void sendMouseEvent(final int x, final int y, final MouseClick type);
 
+  /**
+   * Sends a key event by simulating the entry of the specified text.
+   *
+   * @param text the text to be sent as a key event. This parameter represents
+   *             the string of characters or input sequence that the method simulates
+   *             being typed.
+   */
+  void sendKeyEvent(final String text);
+
   /**
    * Creates a default instance of {@code ChromeDriverPlayer} with pre-defined arguments to
    * configure the Chrome browser in a headless environment.

mcav-common/src/main/java/me/brandonli/mcav/media/player/driver/BrowserPlayer.java

@@ -166,21 +166,28 @@ public void sendMouseEvent(final int x, final int y, final MouseClick type) {
     action.perform();
   }
 
-  private Actions getAction(final MouseClick type, final Actions move) {
-    switch (type) {
-      case LEFT:
-        return move.click();
-      case RIGHT:
-        return move.contextClick();
-      case DOUBLE:
-        return move.doubleClick();
-      case HOLD:
-        return move.clickAndHold();
-      case RELEASE:
-        return move.release();
-      default:
-        throw new InvalidMouseClickArgument("Invalid mouse click type!");
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void sendKeyEvent(final String text) {
+    if (!this.running.get()) {
+      return;
     }
+    final Actions actions = new Actions(this.driver);
+    final Actions move = actions.sendKeys(text);
+    final Action action = move.build();
+    action.perform();
+  }
+
+  private Actions getAction(final MouseClick type, final Actions move) {
+    return switch (type) {
+      case LEFT -> move.click();
+      case RIGHT -> move.contextClick();
+      case DOUBLE -> move.doubleClick();
+      case HOLD -> move.clickAndHold();
+      case RELEASE -> move.release();
+    };
   }
 
   /**

mcav-common/src/main/java/me/brandonli/mcav/media/player/driver/ChromeDriverPlayer.java

@@ -98,6 +98,22 @@ default CompletableFuture<Boolean> startAsync(
    */
   void moveMouse(final int x, final int y);
 
+  /**
+   * Simulates a mouse click at the specified coordinates within the VNC session.
+   * This method moves the mouse to the given position and performs a press-and-release
+   * action for the specified mouse button type.
+   *
+   * @param x    the x-coordinate to click at, in pixels
+   * @param y    the y-coordinate to click at, in pixels
+   * @param type the mouse button type to click, where 0 typically represents the left button,
+   *             1 represents the middle button, and 2 represents the right button
+   */
+  default void click(final int x, final int y, final int type) {
+    this.moveMouse(x, y);
+    this.updateMouseButton(type, true);
+    this.updateMouseButton(type, false);
+  }
+
   /**
    * Simulates typing the provided text into the VNC session by sending corresponding keyboard input events.
    *

mcav-common/src/main/java/me/brandonli/mcav/media/player/vnc/VNCPlayer.java

@@ -27,6 +27,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 import me.brandonli.mcav.media.image.StaticImage;
+import me.brandonli.mcav.media.player.PlayerException;
 import me.brandonli.mcav.media.player.metadata.VideoMetadata;
 import me.brandonli.mcav.media.player.pipeline.step.VideoPipelineStep;
 import me.brandonli.mcav.media.source.VNCSource;
@@ -49,18 +50,16 @@ public class VNCPlayerImpl implements VNCPlayer {
   private final AtomicBoolean running;
   private final Object lock = new Object();
 
-  private boolean connected;
-  private volatile BufferedImage current;
   private @Nullable CompletableFuture<?> processingFuture;
   private @Nullable VernacularClient vncClient;
 
-  private VideoPipelineStep videoPipeline;
-  private VideoMetadata videoMetadata;
+  private volatile BufferedImage current;
+  private volatile VideoPipelineStep videoPipeline;
+  private volatile VideoMetadata videoMetadata;
 
   VNCPlayerImpl() {
     this.frameProcessorExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
     this.running = new AtomicBoolean(false);
-    this.connected = false;
   }
 
   /**
@@ -69,7 +68,7 @@ public class VNCPlayerImpl implements VNCPlayer {
   @Override
   public boolean start(final VideoPipelineStep videoPipeline, final VNCSource source) {
     synchronized (this.lock) {
-      if (this.connected) {
+      if (this.running.get()) {
         return true;
       }
       this.videoPipeline = videoPipeline;
@@ -79,7 +78,6 @@ public boolean start(final VideoPipelineStep videoPipeline, final VNCSource sour
       this.vncClient = new VernacularClient(config);
       this.vncClient.start(source.getHost(), source.getPort());
 
-      this.connected = true;
       this.running.set(true);
 
       this.processingFuture = CompletableFuture.runAsync(this::processFrames, this.frameProcessorExecutor);
@@ -139,7 +137,7 @@ private void processFrames() {
   @Override
   public boolean pause() {
     synchronized (this.lock) {
-      if (this.connected && this.running.get()) {
+      if (this.running.get()) {
         this.running.set(false);
         return true;
       }
@@ -153,7 +151,7 @@ public boolean pause() {
   @Override
   public boolean resume() {
     synchronized (this.lock) {
-      if (this.connected && !this.running.get()) {
+      if (!this.running.get()) {
         this.running.set(true);
         if (this.processingFuture == null || this.processingFuture.isDone()) {
           this.processingFuture = CompletableFuture.runAsync(this::processFrames, this.frameProcessorExecutor);
@@ -171,23 +169,15 @@ public boolean resume() {
   public boolean release() {
     synchronized (this.lock) {
       this.running.set(false);
-
-      if (this.connected) {
-        if (this.processingFuture != null) {
-          this.processingFuture.cancel(true);
-          this.processingFuture = null;
-        }
-
-        if (this.vncClient != null) {
-          this.vncClient.stop();
-          this.vncClient = null;
-        }
-
-        this.connected = false;
+      if (this.processingFuture != null) {
+        this.processingFuture.cancel(true);
+        this.processingFuture = null;
+      }
+      if (this.vncClient != null) {
+        this.vncClient.stop();
+        this.vncClient = null;
       }
-
       ExecutorUtils.shutdownExecutorGracefully(this.frameProcessorExecutor);
-
       return true;
     }
   }
@@ -197,9 +187,24 @@ public boolean release() {
    */
   @Override
   public void moveMouse(final int x, final int y) {
-    if (this.vncClient != null) {
-      this.vncClient.moveMouse(x, y);
+    final VernacularClient client = this.vncClient;
+    if (client != null) {
+      final int[] translated = this.translateCoordinates(x, y);
+      client.moveMouse(translated[0], translated[1]);
+    }
+  }
+
+  private int[] translateCoordinates(final int x, final int y) {
+    final int originWidth = this.videoMetadata.getVideoWidth();
+    final int originHeight = this.videoMetadata.getVideoHeight();
+    if (this.current == null) {
+      throw new PlayerException("VNC source not started!");
     }
+    final int targetWidth = this.current.getWidth();
+    final int targetHeight = this.current.getHeight();
+    final int newX = (int) (((float) x / originWidth) * targetWidth);
+    final int newY = (int) (((float) y / originHeight) * targetHeight);
+    return new int[] { newX, newY };
   }
 
   /**