janky and wip audio stuff; getting closer to having it work properly

build.gradle

@@ -80,6 +80,7 @@ repositories{
         url = "https://www.cursemaven.com"
     }
     maven { url 'https://mcef-download.cinemamod.com/repositories/releases/' }
+    flatDir name: 'libs', dirs: 'libs'
 }
 
 dependencies {
@@ -91,7 +92,7 @@ dependencies {
     // here because we need to manually open the VR keyboard
     compileOnly fg.deobf("curse.maven:vivecraft-667903:4794431")
 
-    implementation fg.deobf("com.cinemamod:mcef-forge:2.1.1-1.20.1") {
+    implementation fg.deobf("com.cinemamod:mcef-forge:2.1.5-1.20.1") {
         transitive = false
     }
 }

src/main/java/net/montoyo/wd/client/ClientProxy.java

@@ -61,6 +61,7 @@ import net.minecraftforge.network.NetworkEvent;
 import net.montoyo.wd.SharedProxy;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.block.ScreenBlock;
+import net.montoyo.wd.client.audio.WDAudioHandler;
 import net.montoyo.wd.client.gui.*;
 import net.montoyo.wd.client.gui.loading.GuiLoader;
 import net.montoyo.wd.client.renderers.*;
@@ -286,6 +287,7 @@ public class ClientProxy extends SharedProxy implements ResourceManagerReloadLis
 				}
 		);
 
+		MCEF.getClient().addAudioHandler(WDAudioHandler.INSTANCE);
 		MCEF.getClient().addDisplayHandler(DisplayHandler.INSTANCE);
 		MCEF.getClient().getHandle().addMessageRouter(CefMessageRouter.create(WDRouter.INSTANCE));
 

src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java

@@ -0,0 +1,84 @@
+package net.montoyo.wd.client.audio;
+
+import net.montoyo.wd.WebDisplays;
+import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.entity.ScreenBlockEntity;
+import net.montoyo.wd.utilities.Log;
+import org.cef.browser.CefBrowser;
+import org.cef.handler.CefAudioHandler;
+import org.cef.misc.CefAudioParameters;
+import org.cef.misc.DataPointer;
+
+public class WDAudioHandler implements CefAudioHandler {
+	public static final WDAudioHandler INSTANCE = new WDAudioHandler();
+	
+	@Override
+	public boolean getAudioParameters(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		boolean didParameterize = false;
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.parameterize(cefAudioParameters);
+				didParameterize = true;
+			}
+		}
+		
+		return didParameterize;
+	}
+	
+	@Override
+	public void onAudioStreamStarted(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters, int channels) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		if (cefAudioParameters == null) return;
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.parameterize(channels, cefAudioParameters);
+			}
+		}
+	}
+	
+	//
+	// Called on the audio stream thread when a PCM packet is received for the
+	// stream. |data| is an array representing the raw PCM data as a floating
+	// point type, i.e. 4-byte value(s). |frames| is the number of frames in the
+	// PCM packet. |pts| is the presentation timestamp (in milliseconds since the
+	// Unix Epoch) and represents the time at which the decompressed packet
+	// should be presented to the user. Based on |frames| and the
+	// |channel_layout| value passed to OnAudioStreamStarted you can calculate
+	// the size of the |data| array in bytes.
+	//
+	@Override
+	public void onAudioStreamPacket(CefBrowser cefBrowser, DataPointer pointer, int frames, long pts) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				source.audioStream.setData(pointer);
+			}
+		}
+	}
+	
+	@Override
+	public void onAudioStreamStopped(CefBrowser cefBrowser) {
+	
+	}
+	
+	@Override
+	public void onAudioStreamError(CefBrowser cefBrowser, String s) {
+		ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
+		
+		for (ScreenBlockEntity tes : proxy.getScreens()) {
+			WDAudioSource source = tes.getSoundSource(cefBrowser);
+			if (source != null) {
+				Log.warning("Audio stream errored: " + s);
+			}
+		}
+	}
+}

src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java

@@ -12,105 +12,115 @@ import net.minecraft.util.RandomSource;
 import net.minecraft.util.valueproviders.SampledFloat;
 import net.montoyo.wd.entity.ScreenBlockEntity;
 import net.montoyo.wd.entity.ScreenData;
+import org.cef.misc.CefAudioParameters;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.concurrent.CompletableFuture;
 
 public class WDAudioSource implements SoundInstance {
-    private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
-    private static final WeighedSoundEvents events = new WeighedSoundEvents(
-            location, "webdisplays.browser"
-    );
-    private static final SampledFloat CONST_1 = new SampledFloat() {
-        @Override
-        public float sample(RandomSource pRandom) {
-            return 1.0f;
-        }
-    };
-    private final Sound sound = new Sound(
-            "unused",
-            CONST_1,
-            CONST_1,
-            1, Sound.Type.SOUND_EVENT,
-            true, false,
-            100
-    );
-    ScreenBlockEntity blockEntity;
-    ScreenData data;
-
-    public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
-        this.blockEntity = blockEntity;
-        this.data = data;
-    }
-
-    @Override
-    public ResourceLocation getLocation() {
-        return location;
-    }
-
-    @Nullable
-    @Override
-    public WeighedSoundEvents resolve(SoundManager pManager) {
-        return events;
-    }
-
-    @Override
-    public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
-        return null;
-    }
-
-    @Override
-    public Sound getSound() {
-        return sound;
-    }
-
-    @Override
-    public SoundSource getSource() {
-        return SoundSource.RECORDS;
-    }
-
-    @Override
-    public boolean isLooping() {
-        return true;
-    }
-
-    @Override
-    public boolean isRelative() {
-        return false;
-    }
-
-    @Override
-    public int getDelay() {
-        return 0;
-    }
-
-    @Override
-    public float getVolume() {
-        return blockEntity.ytVolume;
-    }
-
-    @Override
-    public float getPitch() {
-        return 1;
-    }
-
-    @Override
-    public double getX() {
-        return blockEntity.getBlockPos().getX();
-    }
-
-    @Override
-    public double getY() {
-        return blockEntity.getBlockPos().getY();
-    }
-
-    @Override
-    public double getZ() {
-        return blockEntity.getBlockPos().getZ();
-    }
-
-    @Override
-    public Attenuation getAttenuation() {
-        return Attenuation.LINEAR;
-    }
+	private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
+	private static final WeighedSoundEvents events = new WeighedSoundEvents(
+			location, "webdisplays.browser"
+	);
+	private static final SampledFloat CONST_1 = new SampledFloat() {
+		@Override
+		public float sample(RandomSource pRandom) {
+			return 1.0f;
+		}
+	};
+	private final Sound sound = new Sound(
+			"unused",
+			CONST_1,
+			CONST_1,
+			1, Sound.Type.SOUND_EVENT,
+			true, false,
+			100
+	);
+	ScreenBlockEntity blockEntity;
+	ScreenData data;
+	WDAudioStream audioStream = new WDAudioStream();
+	
+	public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
+		this.blockEntity = blockEntity;
+		this.data = data;
+	}
+	
+	public void parameterize(int channels, CefAudioParameters cefAudioParameters) {
+		audioStream.setFormat(channels, cefAudioParameters);
+	}
+	
+	public void parameterize(CefAudioParameters cefAudioParameters) {
+		audioStream.setFormat(cefAudioParameters);
+	}
+	
+	@Override
+	public ResourceLocation getLocation() {
+		return location;
+	}
+	
+	@Nullable
+	@Override
+	public WeighedSoundEvents resolve(SoundManager pManager) {
+		return events;
+	}
+	
+	@Override
+	public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
+		return CompletableFuture.completedFuture(audioStream);
+	}
+	
+	@Override
+	public Sound getSound() {
+		return sound;
+	}
+	
+	@Override
+	public SoundSource getSource() {
+		return SoundSource.RECORDS;
+	}
+	
+	@Override
+	public boolean isLooping() {
+		return true;
+	}
+	
+	@Override
+	public boolean isRelative() {
+		return false;
+	}
+	
+	@Override
+	public int getDelay() {
+		return 0;
+	}
+	
+	@Override
+	public float getVolume() {
+		return blockEntity.ytVolume;
+	}
+	
+	@Override
+	public float getPitch() {
+		return 1;
+	}
+	
+	@Override
+	public double getX() {
+		return blockEntity.getBlockPos().getX();
+	}
+	
+	@Override
+	public double getY() {
+		return blockEntity.getBlockPos().getY();
+	}
+	
+	@Override
+	public double getZ() {
+		return blockEntity.getBlockPos().getZ();
+	}
+	
+	@Override
+	public Attenuation getAttenuation() {
+		return Attenuation.LINEAR;
+	}
 }

src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java

@@ -0,0 +1,141 @@
+package net.montoyo.wd.client.audio;
+
+import com.mojang.blaze3d.audio.Channel;
+import net.minecraft.client.sounds.AudioStream;
+import org.cef.misc.CefAudioParameters;
+import org.cef.misc.CefChannelLayout;
+import org.cef.misc.DataPointer;
+import org.checkerframework.checker.units.qual.A;
+import org.lwjgl.system.MemoryUtil;
+
+import javax.sound.sampled.AudioFormat;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+
+public class WDAudioStream implements AudioStream, WDExtendedAudioStream {
+	AudioFormat currentFormat = new AudioFormat(
+			AudioFormat.Encoding.PCM_SIGNED,
+			44100, 16,
+			1, (4096 / 8) * 2, 44100,
+			false
+	);
+	
+	@Override
+	public AudioFormat getFormat() {
+		return currentFormat;
+	}
+	
+	int fpb;
+	
+	public void setFormat(int channels, CefAudioParameters cefAudioParameters) {
+		currentFormat = new AudioFormat(
+				AudioFormat.Encoding.PCM_SIGNED,
+				cefAudioParameters.sampleRate, 16,
+				1, (16 / 8) * channels, cefAudioParameters.sampleRate,
+				false
+		);
+		fpb = cefAudioParameters.framesPerBuffer;
+	}
+	
+	public void setFormat(CefAudioParameters cefAudioParameters) {
+		CefChannelLayout layout = cefAudioParameters.channelLayout;
+		int channels = 0;
+		if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_MONO)
+			channels = 1;
+		else if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_STEREO)
+			channels = 2;
+		setFormat(channels, cefAudioParameters);
+	}
+	
+	ArrayDeque<float[]> buffers = new ArrayDeque<>();
+	
+	public void setData(DataPointer data) {
+		int cap = fpb;
+		DataPointer ptr = data
+				.forCapacity(currentFormat.getChannels() << 3)
+				.withAlignment(3);
+		
+		for (int i = 0; i < 1; i++) {
+			DataPointer subPtr = ptr.getData(i)
+					.withAlignment(2)
+					.forCapacity(cap << 2);
+			
+			float[] flts = new float[cap];
+			for (int i1 = 0; i1 < cap; i1++)
+				flts[i1] = subPtr.getFloat(i1);
+			
+			buffers.add(flts);
+		}
+	}
+	
+	boolean checking = false;
+	
+	@Override
+	public ByteBuffer read(int pSize) throws IOException {
+		System.out.println(buffers.size());
+		int sz = 2048 * buffers.size();
+		if (sz < 2048) sz = 4096;
+		pSize = sz;
+		ByteBuffer buffer = ByteBuffer.allocateDirect(pSize);
+		if (!buffers.isEmpty()) {
+			final int MAX_16_BIT = 32767;
+			final int MIN_16_BIT = -32768;
+			
+			int i0 = 0;
+			loopBufs:
+			while (true) {
+				if (!buffers.isEmpty()) {
+					for (float v : buffers.pop()) {
+						if (buffer.position() >= pSize) break loopBufs;
+						
+						// Scale and clip the float value to the range of a signed 16-bit int
+						float floatSample = v;
+						int intSample = (int) (floatSample * MAX_16_BIT);
+						
+						if (intSample > MAX_16_BIT) {
+							intSample = MAX_16_BIT;
+						} else if (intSample < MIN_16_BIT) {
+							intSample = MIN_16_BIT;
+						}
+						
+						// Convert the int sample to bytes (little-endian format)
+						buffer.put((byte) (intSample & 0xFF));
+						buffer.put((byte) ((intSample >> 8) & 0xFF));
+					}
+					i0++;
+				} else break;
+			}
+			buffer.position(0);
+			System.out.println("Consumed " + i0 + " buffers");
+		}
+		return buffer;
+	}
+	
+	@Override
+	public void close() throws IOException {
+	}
+	
+	Channel channel;
+	
+	@Override
+	public void attach(Channel channel) {
+		WDExtendedAudioStream.super.attach(channel);
+		this.channel = channel;
+	}
+	
+	@Override
+	public int getPumpCount(int defaultAmount) {
+		return defaultAmount;
+	}
+	
+	@Override
+	public int computeSize(AudioFormat format, int streamingBufferSize) {
+		return 2048;
+	}
+	
+	@Override
+	public boolean canStop() {
+		return true;
+	}
+}

src/main/java/net/montoyo/wd/client/audio/WDExtendedAudioStream.java

@@ -0,0 +1,23 @@
+package net.montoyo.wd.client.audio;
+
+import com.mojang.blaze3d.audio.Channel;
+
+import javax.sound.sampled.AudioFormat;
+
+public interface WDExtendedAudioStream {
+	default int getPumpCount(int defaultAmount) {
+		return defaultAmount;
+	}
+	
+	default void attach(Channel channel) {
+		return;
+	}
+	
+	default int computeSize(AudioFormat format, int streamingBufferSize) {
+		return streamingBufferSize;
+	}
+	
+	default boolean canStop() {
+		return true;
+	}
+}

src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java

@@ -28,6 +28,7 @@ import net.minecraftforge.network.PacketDistributor;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.block.ScreenBlock;
 import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.client.audio.WDAudioSource;
 import net.montoyo.wd.config.CommonConfig;
 import net.montoyo.wd.controls.builtin.ClickControl;
 import net.montoyo.wd.core.DefaultUpgrade;
@@ -1143,6 +1144,15 @@ public class ScreenBlockEntity extends BlockEntity {
 
         return bhr;
     }
+    
+    public WDAudioSource getSoundSource(CefBrowser cefBrowser) {
+        for (ScreenData screen : screens) {
+            if (screen.browser == cefBrowser) {
+                return screen.audioSource;
+            }
+        }
+        return null;
+    }
 
 //    @Override
 //    public boolean shouldRefresh(Level world, BlockPos pos, @Nonnull BlockState oldState, @Nonnull BlockState newState) {

src/main/java/net/montoyo/wd/entity/ScreenData.java

@@ -3,6 +3,7 @@ package net.montoyo.wd.entity;
 import com.cinemamod.mcef.MCEF;
 import com.cinemamod.mcef.MCEFBrowser;
 import com.cinemamod.mcef.listeners.MCEFCursorChangeListener;
+import net.minecraft.client.Minecraft;
 import net.minecraft.core.BlockPos;
 import net.minecraft.core.Direction;
 import net.minecraft.nbt.CompoundTag;
@@ -13,6 +14,7 @@ import net.minecraft.world.level.Level;
 import net.minecraft.world.level.block.entity.BlockEntity;
 import net.montoyo.wd.WebDisplays;
 import net.montoyo.wd.client.ClientProxy;
+import net.montoyo.wd.client.audio.WDAudioSource;
 import net.montoyo.wd.config.CommonConfig;
 import net.montoyo.wd.core.ScreenRights;
 import net.montoyo.wd.utilities.*;
@@ -33,6 +35,7 @@ public class ScreenData {
     public Vector2i resolution;
     public Rotation rotation = Rotation.ROT_0;
     public String url;
+    public WDAudioSource audioSource;
     protected VideoType videoType;
     public NameUUIDPair owner;
     public ArrayList<NameUUIDPair> friends;
@@ -206,6 +209,12 @@ public class ScreenData {
 
             doTurnOnAnim = doAnim;
             turnOnTime = System.currentTimeMillis();
+			
+            audioSource = new WDAudioSource(
+                    be,
+                    this
+            );
+            Minecraft.getInstance().getSoundManager().play(audioSource);
         }
     }
 }

src/main/java/net/montoyo/wd/mixins/audio/AudioChannelMixin.java

@@ -0,0 +1,48 @@
+package net.montoyo.wd.mixins.audio;
+
+import com.mojang.blaze3d.audio.Channel;
+import net.minecraft.client.sounds.AudioStream;
+import net.montoyo.wd.client.audio.WDExtendedAudioStream;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
+import org.spongepowered.asm.mixin.injection.At;
+import org.spongepowered.asm.mixin.injection.Inject;
+import org.spongepowered.asm.mixin.injection.ModifyVariable;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
+
+import javax.annotation.Nullable;
+import javax.sound.sampled.AudioFormat;
+
+@Mixin(Channel.class)
+public class AudioChannelMixin {
+	@Shadow
+	@Nullable
+	private AudioStream stream;
+	
+	@Shadow private int streamingBufferSize;
+	
+	@Inject(at = @At("HEAD"), method = "attachBufferStream")
+	public void postAttach(AudioStream pStream, CallbackInfo ci) {
+		((WDExtendedAudioStream) pStream).attach((Channel) (Object) this);
+	}
+	
+	@Inject(at = @At(value = "INVOKE", target = "Lcom/mojang/blaze3d/audio/Channel;pumpBuffers(I)V", shift = At.Shift.BEFORE), method = "attachBufferStream")
+	public void postComputeSize(AudioStream pStream, CallbackInfo ci) {
+		streamingBufferSize = ((WDExtendedAudioStream) pStream).computeSize(
+				pStream.getFormat(), streamingBufferSize
+		);
+	}
+	
+	@ModifyVariable(argsOnly = true, ordinal = 0, at = @At("HEAD"), method = "pumpBuffers")
+	public int prePump(int defaultAmount) {
+		return ((WDExtendedAudioStream) stream).getPumpCount(defaultAmount);
+	}
+	
+	@Inject(at = @At("HEAD"), method = "stopped", cancellable = true)
+	public void preCheckStopped(CallbackInfoReturnable<Boolean> cir) {
+		if (stream != null)
+			if (!((WDExtendedAudioStream) stream).canStop())
+				cir.setReturnValue(false);
+	}
+}

src/main/java/net/montoyo/wd/mixins/audio/AudioStreamMixin.java

@@ -0,0 +1,9 @@
+package net.montoyo.wd.mixins.audio;
+
+import net.minecraft.client.sounds.AudioStream;
+import net.montoyo.wd.client.audio.WDExtendedAudioStream;
+import org.spongepowered.asm.mixin.Mixin;
+
+@Mixin(AudioStream.class)
+public interface AudioStreamMixin extends WDExtendedAudioStream {
+}

src/main/resources/webdisplays.mixins.json

@@ -8,7 +8,9 @@
   ],
   "client": [
     "MouseHandlerMixin",
-    "OverlayMixin"
+    "OverlayMixin",
+    "audio.AudioChannelMixin",
+    "audio.AudioStreamMixin"
   ],
   "injectors": {
     "defaultRequire": 1