audio handling works, but is ~43 audio packets delayed

build.gradle

@@ -80,6 +80,7 @@ repositories{
         url = "https://www.cursemaven.com"
     }
     maven { url 'https://mcef-download.cinemamod.com/repositories/releases/' }
+    flatDir name: 'libs', dirs: 'libs'
 }
 
 dependencies {
@@ -91,7 +92,7 @@ dependencies {
     // here because we need to manually open the VR keyboard
     compileOnly fg.deobf("curse.maven:vivecraft-667903:4794431")
 
-    implementation fg.deobf("com.cinemamod:mcef-forge:2.1.1-1.20.1") {
+    implementation fg.deobf("com.cinemamod:mcef-forge:2.1.5-1.20.1") {
         transitive = false
     }
 }

src/main/java/net/montoyo/wd/client/ClientProxy.java

@@ -61,6 +61,7 @@ import net.minecraftforge.network.NetworkEvent;
 import net.montoyo.wd.SharedProxy;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.block.ScreenBlock;
+import net.montoyo.wd.client.audio.WDAudioHandler;
 import net.montoyo.wd.client.gui.*;
 import net.montoyo.wd.client.gui.loading.GuiLoader;
 import net.montoyo.wd.client.renderers.*;
@@ -286,6 +287,7 @@ public class ClientProxy extends SharedProxy implements ResourceManagerReloadLis
 				}
 		);
 
+		MCEF.getClient().addAudioHandler(WDAudioHandler.INSTANCE);
 		MCEF.getClient().addDisplayHandler(DisplayHandler.INSTANCE);
 		MCEF.getClient().getHandle().addMessageRouter(CefMessageRouter.create(WDRouter.INSTANCE));
 

src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java

@@ -0,0 +1,84 @@
+package net.montoyo.wd.client.audio;
+
+import net.montoyo.wd.WebDisplays;
+import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.entity.ScreenBlockEntity;
+import net.montoyo.wd.utilities.Log;
+import org.cef.browser.CefBrowser;
+import org.cef.handler.CefAudioHandler;
+import org.cef.misc.CefAudioParameters;
+import org.cef.misc.DataPointer;
+
+public class WDAudioHandler implements CefAudioHandler {
+	public static final WDAudioHandler INSTANCE = new WDAudioHandler();
+	
+	@Override
+	public boolean getAudioParameters(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		boolean didParameterize = false;
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.parameterize(cefAudioParameters);
+				didParameterize = true;
+			}
+		}
+		
+		return didParameterize;
+	}
+	
+	@Override
+	public void onAudioStreamStarted(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters, int channels) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		if (cefAudioParameters == null) return;
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.parameterize(channels, cefAudioParameters);
+			}
+		}
+	}
+	
+	//
+	// Called on the audio stream thread when a PCM packet is received for the
+	// stream. |data| is an array representing the raw PCM data as a floating
+	// point type, i.e. 4-byte value(s). |frames| is the number of frames in the
+	// PCM packet. |pts| is the presentation timestamp (in milliseconds since the
+	// Unix Epoch) and represents the time at which the decompressed packet
+	// should be presented to the user. Based on |frames| and the
+	// |channel_layout| value passed to OnAudioStreamStarted you can calculate
+	// the size of the |data| array in bytes.
+	//
+	@Override
+	public void onAudioStreamPacket(CefBrowser cefBrowser, long pointer, int frames, long pts) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.audioStream.setData(pointer);
+			}
+		}
+	}
+	
+	@Override
+	public void onAudioStreamStopped(CefBrowser cefBrowser) {
+	
+	}
+	
+	@Override
+	public void onAudioStreamError(CefBrowser cefBrowser, String s) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				Log.warning("Audio stream errored: " + s);
+			}
+		}
+	}
+}

src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java

@@ -12,105 +12,115 @@ import net.minecraft.util.RandomSource;
 import net.minecraft.util.valueproviders.SampledFloat;
 import net.montoyo.wd.entity.ScreenBlockEntity;
 import net.montoyo.wd.entity.ScreenData;
+import org.cef.misc.CefAudioParameters;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.concurrent.CompletableFuture;
 
 public class WDAudioSource implements SoundInstance {
-    private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
+	private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
-    private static final WeighedSoundEvents events = new WeighedSoundEvents(
+	private static final WeighedSoundEvents events = new WeighedSoundEvents(
-            location, "webdisplays.browser"
+			location, "webdisplays.browser"
-    );
+	);
-    private static final SampledFloat CONST_1 = new SampledFloat() {
+	private static final SampledFloat CONST_1 = new SampledFloat() {
-        @Override
+		@Override
-        public float sample(RandomSource pRandom) {
+		public float sample(RandomSource pRandom) {
-            return 1.0f;
+			return 1.0f;
-        }
+		}
-    };
+	};
-    private final Sound sound = new Sound(
+	private final Sound sound = new Sound(
-            "unused",
+			"unused",
-            CONST_1,
+			CONST_1,
-            CONST_1,
+			CONST_1,
-            1, Sound.Type.SOUND_EVENT,
+			1, Sound.Type.SOUND_EVENT,
-            true, false,
+			true, false,
-            100
+			100
-    );
+	);
-    ScreenBlockEntity blockEntity;
+	ScreenBlockEntity blockEntity;
-    ScreenData data;
+	ScreenData data;
-
+	WDAudioStream audioStream = new WDAudioStream();
-    public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
+	
-        this.blockEntity = blockEntity;
+	public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
-        this.data = data;
+		this.blockEntity = blockEntity;
-    }
+		this.data = data;
-
+	}
-    @Override
+	
-    public ResourceLocation getLocation() {
+	public void parameterize(int channels, CefAudioParameters cefAudioParameters) {
-        return location;
+		audioStream.setFormat(channels, cefAudioParameters);
-    }
+	}
-
+	
-    @Nullable
+	public void parameterize(CefAudioParameters cefAudioParameters) {
-    @Override
+		audioStream.setFormat(cefAudioParameters);
-    public WeighedSoundEvents resolve(SoundManager pManager) {
+	}
-        return events;
+	
-    }
+	@Override
-
+	public ResourceLocation getLocation() {
-    @Override
+		return location;
-    public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
+	}
-        return null;
+	
-    }
+	@Nullable
-
+	@Override
-    @Override
+	public WeighedSoundEvents resolve(SoundManager pManager) {
-    public Sound getSound() {
+		return events;
-        return sound;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
-    public SoundSource getSource() {
+		return CompletableFuture.completedFuture(audioStream);
-        return SoundSource.RECORDS;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public Sound getSound() {
-    public boolean isLooping() {
+		return sound;
-        return true;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public SoundSource getSource() {
-    public boolean isRelative() {
+		return SoundSource.RECORDS;
-        return false;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public boolean isLooping() {
-    public int getDelay() {
+		return true;
-        return 0;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public boolean isRelative() {
-    public float getVolume() {
+		return false;
-        return blockEntity.ytVolume;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public int getDelay() {
-    public float getPitch() {
+		return 0;
-        return 1;
+	}
-    }
+	
-
+	@Override
-    @Override
+	public float getVolume() {
-    public double getX() {
+		return blockEntity.ytVolume;
-        return blockEntity.getBlockPos().getX();
+	}
-    }
+	
-
+	@Override
-    @Override
+	public float getPitch() {
-    public double getY() {
+		return 1;
-        return blockEntity.getBlockPos().getY();
+	}
-    }
+	
-
+	@Override
-    @Override
+	public double getX() {
-    public double getZ() {
+		return blockEntity.getBlockPos().getX();
-        return blockEntity.getBlockPos().getZ();
+	}
-    }
+	
-
+	@Override
-    @Override
+	public double getY() {
-    public Attenuation getAttenuation() {
+		return blockEntity.getBlockPos().getY();
-        return Attenuation.LINEAR;
+	}
-    }
+	
+	@Override
+	public double getZ() {
+		return blockEntity.getBlockPos().getZ();
+	}
+	
+	@Override
+	public Attenuation getAttenuation() {
+		return Attenuation.LINEAR;
+	}
 }

src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java

@@ -0,0 +1,106 @@
+package net.montoyo.wd.client.audio;
+
+import net.minecraft.client.sounds.AudioStream;
+import org.cef.misc.CefAudioParameters;
+import org.cef.misc.CefChannelLayout;
+import org.cef.misc.DataPointer;
+import org.checkerframework.checker.units.qual.A;
+import org.lwjgl.system.MemoryUtil;
+
+import javax.sound.sampled.AudioFormat;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+
+public class WDAudioStream implements AudioStream {
+	AudioFormat currentFormat = new AudioFormat(
+			AudioFormat.Encoding.PCM_SIGNED,
+			44100, 16,
+			1, (4096 / 8) * 2, 44100,
+			false
+	);
+	
+	@Override
+	public AudioFormat getFormat() {
+		return currentFormat;
+	}
+	
+	int fpb;
+	
+	public void setFormat(int channels, CefAudioParameters cefAudioParameters) {
+		currentFormat = new AudioFormat(
+				AudioFormat.Encoding.PCM_SIGNED,
+				cefAudioParameters.sampleRate, 16,
+				1, (16 / 8) * channels, cefAudioParameters.sampleRate,
+				false
+		);
+		fpb = cefAudioParameters.framesPerBuffer;
+	}
+	
+	public void setFormat(CefAudioParameters cefAudioParameters) {
+		CefChannelLayout layout = cefAudioParameters.channelLayout;
+		int channels = 0;
+		if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_MONO)
+			channels = 1;
+		else if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_STEREO)
+			channels = 2;
+		setFormat(channels, cefAudioParameters);
+	}
+	
+	ArrayDeque<float[]> buffers = new ArrayDeque<>();
+	
+	public void setData(long data) {
+//		DataPointer ptr = data.forCapacity(currentFormat.getChannels() << 3);
+//		ptr.setAlignment(3);
+		long baseAddr = data;
+		for (int i = 0; i < 1; i++) {
+			long addr = MemoryUtil.memGetLong(baseAddr + (i << 3));
+			int cap = fpb;
+			float[] flts = new float[cap];
+			for (int i1 = 0; i1 < cap; i1++) {
+				flts[i1] = MemoryUtil.memGetFloat(addr + (i1 << 2));
+			}
+			buffers.add(flts);
+		}
+	}
+	
+	@Override
+	public ByteBuffer read(int pSize) throws IOException {
+		System.out.println(buffers.size());
+		ByteBuffer buffer = ByteBuffer.allocateDirect(pSize);
+		if (!buffers.isEmpty()) {
+			final int MAX_16_BIT = 32767;
+			final int MIN_16_BIT = -32768;
+			
+			loopBufs:
+			while (true) {
+				if (!buffers.isEmpty()) {
+					for (float v : buffers.pop()) {
+						if (buffer.position() >= pSize) break loopBufs;
+//						buffer.putFloat(v);
+						
+						// Scale and clip the float value to the range of a signed 16-bit int
+						float floatSample = v;
+						int intSample = (int) (floatSample * MAX_16_BIT);
+						
+						if (intSample > MAX_16_BIT) {
+							intSample = MAX_16_BIT;
+						} else if (intSample < MIN_16_BIT) {
+							intSample = MIN_16_BIT;
+						}
+						
+						// Convert the int sample to bytes (little-endian format)
+						buffer.put((byte) (intSample & 0xFF));
+						buffer.put((byte) ((intSample >> 8) & 0xFF));
+					}
+				} else break;
+			}
+			buffer.position(0);
+		}
+		return buffer;
+	}
+	
+	@Override
+	public void close() throws IOException {
+	}
+}

src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java

@@ -28,6 +28,7 @@ import net.minecraftforge.network.PacketDistributor;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.block.ScreenBlock;
 import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.client.audio.WDAudioSource;
 import net.montoyo.wd.config.CommonConfig;
 import net.montoyo.wd.controls.builtin.ClickControl;
 import net.montoyo.wd.core.DefaultUpgrade;
@@ -1143,6 +1144,15 @@ public class ScreenBlockEntity extends BlockEntity {
 
         return bhr;
     }
+    
+    public WDAudioSource getSoundSource(CefBrowser cefBrowser) {
+        for (ScreenData screen : screens) {
+            if (screen.browser == cefBrowser) {
+                return screen.audioSource;
+            }
+        }
+        return null;
+    }
 
 //    @Override
 //    public boolean shouldRefresh(Level world, BlockPos pos, @Nonnull BlockState oldState, @Nonnull BlockState newState) {

src/main/java/net/montoyo/wd/entity/ScreenData.java

@@ -3,6 +3,7 @@ package net.montoyo.wd.entity;
 import com.cinemamod.mcef.MCEF;
 import com.cinemamod.mcef.MCEFBrowser;
 import com.cinemamod.mcef.listeners.MCEFCursorChangeListener;
+import net.minecraft.client.Minecraft;
 import net.minecraft.core.BlockPos;
 import net.minecraft.core.Direction;
 import net.minecraft.nbt.CompoundTag;
@@ -13,6 +14,7 @@ import net.minecraft.world.level.Level;
 import net.minecraft.world.level.block.entity.BlockEntity;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.client.audio.WDAudioSource;
 import net.montoyo.wd.config.CommonConfig;
 import net.montoyo.wd.core.ScreenRights;
 import net.montoyo.wd.utilities.*;
@@ -33,6 +35,7 @@ public class ScreenData {
     public Vector2i resolution;
     public Rotation rotation = Rotation.ROT_0;
     public String url;
+    public WDAudioSource audioSource;
     protected VideoType videoType;
     public NameUUIDPair owner;
     public ArrayList<NameUUIDPair> friends;
@@ -206,6 +209,12 @@ public class ScreenData {
 
             doTurnOnAnim = doAnim;
             turnOnTime = System.currentTimeMillis();
+			
+            audioSource = new WDAudioSource(
+                    be,
+                    this
+            );
+            Minecraft.getInstance().getSoundManager().play(audioSource);
         }
     }
 }