Minecraft Mod Action handling

.github/workflows/gradle-build.yml

@@ -25,4 +25,4 @@ jobs:
       - name: Build with Gradle
         run: |
           cd forge
-          pixi run minecraft-build
+          pixi run gradle-build

.gitignore

@@ -199,3 +199,6 @@ run
 # Files from Forge MDK
 forge*changelog.txt
 forge/run-data/
+
+# Cursor
+.cursor/

forge/build.gradle

@@ -57,7 +57,7 @@ minecraft {
     // However, it must be at "META-INF/accesstransformer.cfg" in the final mod jar to be loaded by Forge.
     // This default location is a best practice to automatically put the file in the right place in the final jar.
     // See https://docs.minecraftforge.net/en/latest/advanced/accesstransformers/ for more information.
-    // accessTransformer = file('src/main/resources/META-INF/accesstransformer.cfg')
+    accessTransformer = file('src/main/resources/META-INF/accesstransformer.cfg')
 
     // Default run configurations.
     // These can be tweaked, removed, or duplicated as needed.
@@ -128,6 +128,14 @@ dependencies {
     // then special handling is done to allow a setup of a vanilla dependency without the use of an external repository.
     minecraft "net.minecraftforge:forge:${minecraft_version}-${forge_version}"
 
+    // JUnit 5 for unit testing
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.4'
+    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+
+    // Mockito for mocking in tests
+    testImplementation 'org.mockito:mockito-core:5.14.2'
+    testImplementation 'org.mockito:mockito-junit-jupiter:5.14.2'
+
     // Example mod dependency with JEI
     // The JEI API is declared for compile time use, while the full JEI artifact is used at runtime
     // compileOnly "mezz.jei:jei-${mc_version}-common-api:${jei_version}"

forge/src/main/java/com/mineagent/ClientEventHandler.java

@@ -4,19 +4,20 @@
 import com.mojang.logging.LogUtils;
 import java.nio.ByteBuffer;
 import net.minecraft.client.Minecraft;
-import net.minecraft.world.entity.player.Player;
 import net.minecraftforge.event.TickEvent;
 import net.minecraftforge.event.entity.living.LivingDeathEvent;
 import net.minecraftforge.event.entity.living.LivingHurtEvent;
 import net.minecraftforge.event.server.ServerStartingEvent;
 import net.minecraftforge.event.server.ServerStoppingEvent;
 import net.minecraftforge.eventbus.api.SubscribeEvent;
+import org.lwjgl.glfw.GLFW;
 import org.lwjgl.opengl.GL11;
 import org.slf4j.Logger;
 
+/** Handles client-side game events and coordinates input injection with observations. */
 public class ClientEventHandler {
   private static final Logger LOGGER = LogUtils.getLogger();
-  private static DataBridge dataBridge = DataBridge.getInstance();
+  private static final DataBridge dataBridge = DataBridge.getInstance();
 
   @SubscribeEvent
   public static void onServerStarting(ServerStartingEvent event) {
@@ -28,35 +29,71 @@ public static void onServerStopping(ServerStoppingEvent event) {
     LOGGER.info("MineAgent Mod Server Stopping");
   }
 
+  /** Main game tick handler. Processes raw input and captures observations. */
   @SubscribeEvent
   public static void onClientTick(TickEvent.ClientTickEvent event) {
-    if (event.phase == TickEvent.Phase.END) {
-      captureAndSendFrame();
+    Minecraft mc = Minecraft.getInstance();
+    if (mc.level != null && mc.player != null && event.phase == TickEvent.Phase.END) {
+      // Handle input suppression when client is connected
+      handleInputSuppression(mc);
+
+      // Process any pending raw input
+      final RawInput rawInput = dataBridge.getLatestRawInput();
+      if (rawInput != null) {
+        dataBridge.getInputInjector().inject(rawInput);
+      }
+
+      // IMPORTANT: Maintain button state every tick for continuous actions
+      // This fires press events and sets KeyMapping states for held buttons
+      dataBridge.getInputInjector().maintainButtonState();
+
+      // Capture and send observation
+      final byte[] frame = captureFrame();
+      dataBridge.sendObservation(new Observation(0.0, frame));
+    }
+  }
+
+  /**
+   * Handles input suppression when a Python client is connected. Disables the system cursor to
+   * prevent real mouse input from interfering.
+   */
+  private static void handleInputSuppression(Minecraft mc) {
+    boolean clientConnected = dataBridge.isClientConnected();
+    boolean suppressMouse = Config.SUPPRESS_SYSTEM_MOUSE_INPUT.get();
+    boolean suppressKeyboard = Config.SUPPRESS_SYSTEM_KEYBOARD_INPUT.get();
+
+    if (clientConnected && (suppressMouse || suppressKeyboard)) {
+      long windowHandle = mc.getWindow().getWindow();
+
+      // Suppress mouse by hiding/disabling cursor
+      if (suppressMouse) {
+        GLFW.glfwSetInputMode(windowHandle, GLFW.GLFW_CURSOR, GLFW.GLFW_CURSOR_DISABLED);
+      }
+
+      // Note: Keyboard suppression would require intercepting at a lower level
+      // For now, the agent's input will override via the GLFW handlers
     }
   }
 
   @SubscribeEvent
   public static void onPlayerHurt(LivingHurtEvent event) {
-    if (event.getEntity() instanceof Player) {
-      dataBridge.sendEvent("PLAYER_HURT", String.valueOf(event.getAmount()));
-    }
+    // Future: Calculate reward based on damage
+    // if (event.getEntity() instanceof Player) {
+    //     dataBridge.sendEvent("PLAYER_HURT", String.valueOf(event.getAmount()));
+    // }
   }
 
   @SubscribeEvent
   public static void onPlayerDeath(LivingDeathEvent event) {
-    if (event.getEntity() instanceof Player) {
-      dataBridge.sendEvent("PLAYER_DEATH", "-100.0");
-    }
+    // Future: Calculate negative reward on death
+    // if (event.getEntity() instanceof Player) {
+    //     dataBridge.sendEvent("PLAYER_DEATH", "-100.0");
+    // }
   }
 
-  private static void captureAndSendFrame() {
+  private static byte[] captureFrame() {
     Minecraft mc = Minecraft.getInstance();
-    if (mc.level != null && mc.player != null) {
-      LOGGER.info("Capturing screenshot");
-      byte[] frameData = captureScreenshot(mc.getWindow());
-      LOGGER.info("Sending frame data");
-      dataBridge.sendFrame(frameData);
-    }
+    return captureScreenshot(mc.getWindow());
   }
 
   private static byte[] captureScreenshot(Window window) {

forge/src/main/java/com/mineagent/Config.java

@@ -17,6 +17,8 @@ public class Config {
   public static final ForgeConfigSpec.ConfigValue<Double> JPEG_QUALITY;
   public static final ForgeConfigSpec.ConfigValue<Integer> MAX_FRAME_SIZE;
   public static final ForgeConfigSpec.ConfigValue<Integer> CHANGE_THRESHOLD;
+  public static final ForgeConfigSpec.ConfigValue<Boolean> SUPPRESS_SYSTEM_MOUSE_INPUT;
+  public static final ForgeConfigSpec.ConfigValue<Boolean> SUPPRESS_SYSTEM_KEYBOARD_INPUT;
 
   // Built Configuration Specification
   public static final ForgeConfigSpec SPEC;
@@ -73,6 +75,23 @@ public class Config {
 
     BUILDER.pop();
 
+    // Input Configuration
+    BUILDER.comment("Input Configuration");
+    BUILDER.push("input");
+
+    SUPPRESS_SYSTEM_MOUSE_INPUT =
+        BUILDER
+            .comment(
+                "If true, disables OS cursor when Python client is connected for agent mouse control")
+            .define("suppress_system_mouse_input", true);
+
+    SUPPRESS_SYSTEM_KEYBOARD_INPUT =
+        BUILDER
+            .comment("If true, agent keyboard input takes priority when Python client is connected")
+            .define("suppress_system_keyboard_input", true);
+
+    BUILDER.pop();
+
     // Build the specification after all values are defined
     SPEC = BUILDER.build();
   }

forge/src/main/java/com/mineagent/DataBridge.java

@@ -1,14 +1,32 @@
 package com.mineagent;
 
 import com.mojang.logging.LogUtils;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import org.slf4j.Logger;
 
+/**
+ * Central bridge for data exchange between network handler and game events. Manages the latest raw
+ * input, observations, and connection state.
+ */
 public class DataBridge {
   private static final Logger LOGGER = LogUtils.getLogger();
   private static DataBridge instance;
+
   private NetworkHandler networkHandler;
 
-  public static DataBridge getInstance() {
+  // Raw input handling
+  private final AtomicReference<RawInput> latestRawInput = new AtomicReference<>();
+
+  // Input injection
+  private final InputInjector inputInjector = new InputInjector();
+
+  // Connection state for input suppression
+  private final AtomicBoolean clientConnected = new AtomicBoolean(false);
+
+  private DataBridge() {}
+
+  public static synchronized DataBridge getInstance() {
     if (instance == null) {
       instance = new DataBridge();
       LOGGER.info("DataBridge instance created");
@@ -21,21 +39,44 @@ public void setNetworkHandler(NetworkHandler handler) {
     LOGGER.info("NetworkHandler connected to DataBridge");
   }
 
-  public void sendEvent(String eventType, String data) {
+  /** Sends an observation to connected clients. */
+  public void sendObservation(Observation obs) {
     if (networkHandler != null) {
-      LOGGER.info("Sending event: {} with data: {}", eventType, data);
-      // TODO: Implement this
+      networkHandler.setLatest(obs.frame(), obs.reward());
     } else {
-      LOGGER.warn("Cannot send event - NetworkHandler is null");
+      LOGGER.warn("Cannot send frame - NetworkHandler is null");
     }
   }
 
-  public void sendFrame(byte[] frameData) {
-    if (networkHandler != null) {
-      LOGGER.info("DataBridge sending frame data (size: {} bytes)", frameData.length);
-      networkHandler.setLatest(frameData, 0);
-    } else {
-      LOGGER.warn("Cannot send frame - NetworkHandler is null");
+  /** Sets the latest raw input received from the Python agent. */
+  public void setLatestRawInput(RawInput rawInput) {
+    latestRawInput.set(rawInput);
+  }
+
+  /** Gets and clears the latest raw input. Returns null if no new input is available. */
+  public RawInput getLatestRawInput() {
+    RawInput rawInput = latestRawInput.getAndSet(null);
+    if (rawInput != null) {
+      LOGGER.debug("DataBridge getting latest raw input: {} keys", rawInput.keyCodes().length);
     }
+    return rawInput;
+  }
+
+  /** Gets the input injector for injecting raw input into Minecraft. */
+  public InputInjector getInputInjector() {
+    return inputInjector;
+  }
+
+  /** Sets whether a Python client is connected. Used for input suppression. */
+  public void setClientConnected(boolean connected) {
+    boolean wasConnected = clientConnected.getAndSet(connected);
+    if (wasConnected != connected) {
+      LOGGER.info("Client connection state changed: {}", connected ? "CONNECTED" : "DISCONNECTED");
+    }
+  }
+
+  /** Returns whether a Python client is currently connected. */
+  public boolean isClientConnected() {
+    return clientConnected.get();
   }
 }