keyboard camera prototype

src/main/java/net/montoyo/wd/WebDisplays.java

@@ -38,6 +38,7 @@ import net.minecraftforge.network.PacketDistributor;
 import net.minecraftforge.registries.DeferredRegister;
 import net.minecraftforge.registries.ForgeRegistries;
 import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.client.gui.camera.KeyboardCamera;
 import net.montoyo.wd.config.ClientConfig;
 import net.montoyo.wd.config.CommonConfig;
 import net.montoyo.wd.controls.ScreenControlRegistry;
@@ -114,6 +115,8 @@ public class WebDisplays {
             // proxies are annoying, so from now on, I'mma be just registering stuff in here
             FMLJavaModLoadingContext.get().getModEventBus().addListener(ClientProxy::onKeybindRegistry);
             MinecraftForge.EVENT_BUS.addListener(ClientProxy::onDrawSelection);
+            MinecraftForge.EVENT_BUS.addListener(KeyboardCamera::updateCamera);
+            MinecraftForge.EVENT_BUS.addListener(KeyboardCamera::gameTick);
             ClientConfig.init();
         }
         

src/main/java/net/montoyo/wd/client/ClientProxy.java

@@ -65,6 +65,7 @@ import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.block.BlockScreen;
 import net.montoyo.wd.client.gui.*;
 import net.montoyo.wd.client.gui.loading.GuiLoader;
+import net.montoyo.wd.client.js.WDRouter;
 import net.montoyo.wd.client.renderers.*;
 import net.montoyo.wd.config.ClientConfig;
 import net.montoyo.wd.core.HasAdvancement;
@@ -85,7 +86,10 @@ import net.montoyo.wd.utilities.*;
 import org.cef.CefSettings;
 import org.cef.browser.CefBrowser;
 import org.cef.browser.CefFrame;
+import org.cef.browser.CefMessageRouter;
+import org.cef.callback.CefQueryCallback;
 import org.cef.handler.CefDisplayHandler;
+import org.cef.handler.CefMessageRouterHandlerAdapter;
 import org.cef.misc.CefCursorType;
 import org.cef.network.CefRequest;
 import org.joml.Vector3d;
@@ -286,6 +290,8 @@ public class ClientProxy extends SharedProxy implements CefDisplayHandler/*, IJS
 		MCEF.getClient().addDisplayHandler(this);
 //		mcef.registerJSQueryHandler(this);
 
+		MCEF.getClient().getHandle().addMessageRouter(CefMessageRouter.create(WDRouter.INSTANCE));
+
 		findAdvancementToProgressField();
 	}
 	

src/main/java/net/montoyo/wd/client/gui/GuiKeyboard.java

@@ -4,7 +4,10 @@
 
 package net.montoyo.wd.client.gui;
 
+import com.cinemamod.mcef.MCEFBrowser;
+import com.google.gson.JsonObject;
 import net.minecraft.client.Minecraft;
+import net.minecraft.client.gui.screens.Screen;
 import net.minecraft.core.BlockPos;
 import net.minecraft.network.chat.Component;
 import net.minecraft.resources.ResourceLocation;
@@ -13,10 +16,13 @@ import net.minecraftforge.api.distmarker.OnlyIn;
 import net.minecraftforge.fml.ModList;
 import net.minecraftforge.fml.loading.FMLPaths;
 import net.montoyo.wd.WebDisplays;
+import net.montoyo.wd.client.gui.camera.KeyboardCamera;
 import net.montoyo.wd.client.gui.controls.Button;
 import net.montoyo.wd.client.gui.controls.Control;
 import net.montoyo.wd.client.gui.controls.Label;
 import net.montoyo.wd.client.gui.loading.FillControl;
+import net.montoyo.wd.client.js.WDRouter;
+import net.montoyo.wd.entity.ScreenData;
 import net.montoyo.wd.entity.TileEntityScreen;
 import net.montoyo.wd.net.WDNetworkRegistry;
 import net.montoyo.wd.net.server_bound.C2SMessageScreenCtrl;
@@ -24,16 +30,23 @@ import net.montoyo.wd.utilities.BlockSide;
 import net.montoyo.wd.utilities.Log;
 import net.montoyo.wd.utilities.TypeData;
 import net.montoyo.wd.utilities.Util;
+import org.cef.browser.CefBrowser;
+import org.cef.browser.CefFrame;
+import org.cef.browser.CefMessageRouter;
+import org.cef.callback.CefQueryCallback;
+import org.cef.handler.CefMessageRouterHandlerAdapter;
 import org.lwjgl.glfw.GLFW;
 import org.vivecraft.client_vr.gameplay.VRPlayer;
 import org.vivecraft.client_vr.gameplay.screenhandlers.KeyboardHandler;
 //import org.vivecraft.gameplay.VRPlayer;
 //import org.vivecraft.gameplay.screenhandlers.KeyboardHandler;
 
+import java.awt.*;
 import java.io.*;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 
 @OnlyIn(Dist.CLIENT)
 public class GuiKeyboard extends WDScreen {
@@ -69,7 +82,7 @@ public class GuiKeyboard extends WDScreen {
     }
 
     private static final boolean vivecraftPresent;
-    
+
     static {
         boolean vivePres = false;
         if (ModList.get().isLoaded("vivecraft")) vivePres = true;
@@ -91,7 +104,7 @@ public class GuiKeyboard extends WDScreen {
         }
         vivecraftPresent = vivePres;
     }
-    
+
     @Override
     public void init() {
         super.init();
@@ -137,43 +150,48 @@ public class GuiKeyboard extends WDScreen {
 
         defaultBackground = showWarning;
         syncTicks = 5;
-    
+
         if (vivecraftPresent)
             if (VRPlayer.get() != null)
                 KeyboardHandler.setOverlayShowing(true);
+
+        KeyboardCamera.focus(tes, side);
     }
-    
+
     @Override
     public void onClose() {
         if (vivecraftPresent)
             if (VRPlayer.get() != null)
                 KeyboardHandler.setOverlayShowing(false);
         super.onClose();
+        KeyboardCamera.focus(null, null);
     }
-    
+
     @Override
     public boolean keyPressed(int keyCode, int scanCode, int modifiers) {
-        if(quitOnEscape && keyCode == GLFW.GLFW_KEY_ESCAPE)
+        if(quitOnEscape && keyCode == GLFW.GLFW_KEY_ESCAPE) {
             Minecraft.getInstance().setScreen(null);
+            onClose();
+        }
         addKey(new TypeData(TypeData.Action.PRESS, keyCode, modifiers, scanCode));
         return super.keyPressed(keyCode, scanCode, modifiers);
     }
-    
+
     @Override
     public boolean charTyped(char codePoint, int modifiers) {
         addKey(new TypeData(TypeData.Action.TYPE, codePoint, modifiers, 0));
         return super.charTyped(codePoint, modifiers);
     }
-    
+
     @Override
     public boolean keyReleased(int keyCode, int scanCode, int modifiers) {
         addKey(new TypeData(TypeData.Action.RELEASE, keyCode, modifiers, scanCode));
         return super.keyPressed(keyCode, scanCode, modifiers);
     }
-    
+
     void addKey(TypeData data) {
         tes.type(side, "[" + WebDisplays.GSON.toJson(data) + "]", kbPos);
-        
+
         evStack.add(data);
         if (!evStack.isEmpty() && !syncRequested())
             requestSync();

src/main/java/net/montoyo/wd/client/gui/camera/KeyboardCamera.java

@@ -0,0 +1,252 @@
+package net.montoyo.wd.client.gui.camera;
+
+import com.google.gson.JsonObject;
+import net.minecraft.client.Minecraft;
+import net.minecraft.commands.arguments.EntityAnchorArgument;
+import net.minecraft.util.Mth;
+import net.minecraft.world.entity.Entity;
+import net.minecraft.world.phys.Vec3;
+import net.minecraftforge.client.event.ViewportEvent;
+import net.minecraftforge.event.TickEvent;
+import net.montoyo.wd.client.js.WDRouter;
+import net.montoyo.wd.entity.ScreenData;
+import net.montoyo.wd.entity.TileEntityScreen;
+import net.montoyo.wd.utilities.BlockSide;
+
+import java.util.concurrent.CompletableFuture;
+
+public class KeyboardCamera {
+    private static TileEntityScreen tes;
+    private static BlockSide side;
+
+    private static double oxCrd = -1;
+    private static double xCrd = -1;
+    private static double nxCrd = -1;
+    private static double oyCrd = -1;
+    private static double yCrd = -1;
+    private static double nyCrd = -1;
+
+    protected static void updateCrd(JsonObject obj) {
+        if (obj.getAsJsonPrimitive("exists").getAsBoolean()) {
+            ScreenData scr = tes.getScreen(side);
+            if (scr != null) {
+                nxCrd = obj.getAsJsonPrimitive("x").getAsDouble() + obj.getAsJsonPrimitive("w").getAsDouble() / 2;
+                nyCrd = obj.getAsJsonPrimitive("y").getAsDouble() + obj.getAsJsonPrimitive("h").getAsDouble() / 2;
+
+                nxCrd /= scr.resolution.x;
+                nxCrd *= scr.size.x;
+
+                nyCrd /= scr.resolution.y;
+                nyCrd = 1 - nyCrd;
+                nyCrd *= scr.size.y;
+            }
+        }
+    }
+
+    private static WDRouter.Task<JsonObject> activeTask;
+    private static long futureStart = 0;
+
+    protected static void pollElement() {
+        if (activeTask != null) return;
+
+        TileEntityScreen teTmp = tes;
+        BlockSide sdTmp = side;
+
+        // async nonsense can occur here
+        if (teTmp == null || sdTmp == null) return;
+
+        ScreenData scr = teTmp.getScreen(sdTmp);
+        if (scr != null) {
+            activeTask = WDRouter.INSTANCE.requestJson(
+                    scr.browser, "ActiveElement", """
+                                                        try {
+                                                        let focusedElement = document.activeElement;
+                                                        if (focusedElement == null) {
+                                                            window.cefQuery({
+                                                              request: 'WebDisplays_ActiveElement{exists: false}',
+                                                              onSuccess: function(response) {},
+                                                              onFailure: function(error_code, error_message) {}
+                                                            });
+                                                        } else {
+                                                            let bodyRect = document.body.getBoundingClientRect();
+                                                            let elemRect = focusedElement.getBoundingClientRect();
+                                                            
+                                                            window.cefQuery({
+                                                              request: 'WebDisplays_ActiveElement{exists: true,'+
+                                                                'x: ' + (elemRect.left) + ',' +
+                                                                'y: ' + (elemRect.top) + ',' +
+                                                                'w: ' + ((elemRect.right - elemRect.left)) + ',' +
+                                                                'h: ' + ((elemRect.bottom - elemRect.top)) +
+                                                              '}',
+                                                              onSuccess: function(response) {},
+                                                              onFailure: function(error_code, error_message) {}
+                                                            });
+                                                        }
+                                                        } catch (err) {
+                                                            console.error(err);
+                                                            window.cefQuery({
+                                                              request: 'WebDisplays_ActiveElement{exists: false}',
+                                                              onSuccess: function(response) {},
+                                                              onFailure: function(error_code, error_message) {}
+                                                            });
+                                                        }
+                            """.replace("\n", "")
+            ).thenAccept((o1) -> {
+                System.out.println(o1);
+                updateCrd(o1);
+                activeTask = null;
+            });
+            futureStart = System.currentTimeMillis();
+        }
+    }
+
+    protected static double signedSqrt(double v) {
+        double sv = Math.signum(v);
+        v *= sv;
+        return Math.sqrt(v) * sv;
+    }
+
+    public static void updateCamera(ViewportEvent.ComputeCameraAngles event) {
+        if (futureStart != 0) {
+            if (futureStart < System.currentTimeMillis() - 5000 || tes == null) {
+                WDRouter.Task<?> active = activeTask;
+                if (active != null) {
+                    active.cancel();
+                    activeTask = null;
+                    futureStart = 0;
+                }
+            }
+        }
+
+        if (tes == null) {
+            xCrd = -1;
+            yCrd = -1;
+            return; // nothing to do
+        }
+
+        if (xCrd < 0) return;
+        if (yCrd < 0) return;
+
+        // TODO: implement
+
+        double focalX = tes.getBlockPos().getX() +
+                side.right.x * (xCrd - 1) + side.up.x * yCrd;
+        double focalY = tes.getBlockPos().getY() +
+                side.right.y * (xCrd - 1) + side.up.y * yCrd;
+        double focalZ = tes.getBlockPos().getZ() +
+                side.right.z * (xCrd - 1) + side.up.z * yCrd;
+
+        double pct = (delay - event.getPartialTick()) / 8f;
+        pct *= pct;
+        focalX = Mth.lerp(pct,
+                focalX,
+                tes.getBlockPos().getX() +
+                        side.right.x * (oxCrd - 1) + side.up.x * oyCrd
+        );
+        focalY = Mth.lerp(pct,
+                focalY,
+                tes.getBlockPos().getY() +
+                        side.right.y * (oxCrd - 1) + side.up.y * oyCrd
+        );
+        focalZ = Mth.lerp(pct,
+                focalZ,
+                tes.getBlockPos().getZ() +
+                        side.right.z * (oxCrd - 1) + side.up.z * oyCrd
+        );
+
+//        focalX -= side.forward.x * 0.5f;
+//        focalY -= side.forward.y * 0.5f;
+//        focalZ -= side.forward.z * 0.5f;
+
+        float[] angle = lookAt(
+                event.getCamera().getEntity(),
+                EntityAnchorArgument.Anchor.EYES,
+                new Vec3(focalX, focalY, focalZ)
+        );
+
+        double scl = 20;
+
+        double mx = Minecraft.getInstance().mouseHandler.xpos();
+        mx /= Minecraft.getInstance().getWindow().getWidth();
+        mx *= scl;
+        mx -= (scl / 2);
+        mx = signedSqrt(mx);
+        angle[1] += mx;
+
+        double my = Minecraft.getInstance().mouseHandler.ypos();
+        my /= Minecraft.getInstance().getWindow().getHeight();
+        my *= scl;
+        my -= (scl / 2);
+        my = signedSqrt(my);
+        angle[0] += my;
+
+        float xRot = event.getYaw(); // left right
+        float yRot = event.getPitch(); // up down
+
+        event.setYaw(angle[1]);
+        event.setPitch(angle[0]);
+    }
+
+    public static void focus(TileEntityScreen screen, BlockSide side) {
+        KeyboardCamera.tes = screen;
+        KeyboardCamera.side = side;
+    }
+
+    public static float[] lookAt(Entity entity, EntityAnchorArgument.Anchor pAnchor, Vec3 pTarget) {
+        Vec3 vec3 = pAnchor.apply(entity);
+        double d0 = pTarget.x - vec3.x;
+        double d1 = pTarget.y - vec3.y;
+        double d2 = pTarget.z - vec3.z;
+        double d3 = Math.sqrt(d0 * d0 + d2 * d2);
+        float xr = (Mth.wrapDegrees((float) (-(Mth.atan2(d1, d3) * (double) (180F / (float) Math.PI)))));
+        float yr = (Mth.wrapDegrees((float) (Mth.atan2(d2, d0) * (double) (180F / (float) Math.PI)) - 90.0F));
+        return new float[]{xr, yr};
+    }
+
+    protected static int delay = 8;
+
+    public static void gameTick(TickEvent.ClientTickEvent event) {
+        if (event.phase.equals(TickEvent.Phase.END)) {
+            if (activeTask != null)
+                return;
+
+            if (side == null) {
+                delay = 1;
+                oxCrd = -1;
+                oyCrd = -1;
+                xCrd = -1;
+                yCrd = -1;
+                nxCrd = -1;
+                nyCrd = -1;
+                return;
+            }
+
+            if (nxCrd == -1) {
+                if (activeTask == null) {
+                    delay = 1;
+                    pollElement();
+                    return;
+                }
+            }
+
+            if (oxCrd == xCrd && delay > 2) {
+                delay = 2;
+            }
+
+            delay--;
+            if (delay == 0) {
+                oxCrd = xCrd;
+                xCrd = nxCrd;
+                oyCrd = yCrd;
+                yCrd = nyCrd;
+                pollElement();
+            } else if (delay < 0) {
+                oxCrd = xCrd;
+                xCrd = nxCrd;
+                oyCrd = yCrd;
+                yCrd = nyCrd;
+                delay = 8;
+            }
+        }
+    }
+}

src/main/java/net/montoyo/wd/client/js/WDRouter.java

@@ -0,0 +1,120 @@
+package net.montoyo.wd.client.js;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import net.minecraft.client.Minecraft;
+import net.montoyo.wd.entity.ScreenData;
+import org.cef.browser.CefBrowser;
+import org.cef.browser.CefFrame;
+import org.cef.callback.CefQueryCallback;
+import org.cef.handler.CefMessageRouterHandler;
+import org.cef.handler.CefMessageRouterHandlerAdapter;
+
+import java.util.ArrayList;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+public class WDRouter extends CefMessageRouterHandlerAdapter {
+    public static final WDRouter INSTANCE = new WDRouter();
+
+    private static boolean exists = false;
+
+    public WDRouter() {
+        if (exists) throw new RuntimeException("Can only have one WD message router.");
+        exists = true;
+    }
+
+    class QueryData {
+        CefBrowser browser;
+        String type;
+        BiConsumer<String, CefQueryCallback> consumer;
+
+        public QueryData(CefBrowser browser, String type, BiConsumer<String, CefQueryCallback> consumer) {
+            this.browser = browser;
+            this.type = type;
+            this.consumer = consumer;
+        }
+    }
+
+    ArrayList<QueryData> awaitingQueries = new ArrayList<>();
+
+    @Override
+    public boolean onQuery(CefBrowser browser, CefFrame frame, long queryId, String request, boolean persistent, CefQueryCallback callback) {
+        if (request.startsWith("WebDisplays_")) {
+            request = request.substring("Webdisplays_".length());
+
+            QueryData target = null;
+            for (QueryData awaitingQuery : awaitingQueries) {
+                if (browser != awaitingQuery.browser) continue;
+
+                if (request.startsWith(awaitingQuery.type)) {
+                    String requestData = request.substring(awaitingQuery.type.length());
+                    target = awaitingQuery;
+                    awaitingQuery.consumer.accept(requestData, callback);
+                    break;
+                }
+            }
+
+            if (target != null) {
+                awaitingQueries.remove(target);
+                callback.success("");
+            } else {
+                callback.failure(-1, "Query " + queryId + " with data " + request + " completed, but there was no active request waiting for the result.");
+            }
+
+            return true;
+        }
+        return false;
+    }
+
+    private static final Gson gson = new Gson();
+
+    public class Task<T> {
+        QueryData qd;
+        CompletableFuture<T> wrapped;
+
+        public Task(QueryData qd, CompletableFuture<T> wrapped) {
+            this.qd = qd;
+            this.wrapped = wrapped;
+        }
+
+        public void cancel() {
+            wrapped.cancel(true);
+            awaitingQueries.remove(qd);
+        }
+
+        public Task<T> thenAccept(Consumer<T> consumer) {
+            wrapped.thenAccept(consumer);
+            return this;
+        }
+    }
+
+    public Task<JsonObject> requestJson(CefBrowser screen, String queryType, String script) {
+        JsonObject[] obj = new JsonObject[1];
+
+        QueryData qd = new QueryData(
+                screen, queryType,
+                (data, context) -> {
+                    obj[0] = gson.fromJson(data, JsonObject.class);
+                }
+        );
+        awaitingQueries.add(qd);
+
+        screen.executeJavaScript(script, "", 0);
+
+        return new Task<>(
+                qd,
+                CompletableFuture.supplyAsync(() -> {
+                    while (obj[0] == null) {
+                        try {
+                            Thread.sleep(1);
+                        } catch (Throwable ignored) {
+                        }
+                    }
+                    return obj[0];
+                })
+        );
+    }
+}