diff --git a/build.gradle b/build.gradle index 10bc6d6..2dd5ab8 100644 --- a/build.gradle +++ b/build.gradle @@ -80,6 +80,7 @@ repositories{ url = "https://www.cursemaven.com" } maven { url 'https://mcef-download.cinemamod.com/repositories/releases/' } + flatDir name: 'libs', dirs: 'libs' } dependencies { @@ -91,7 +92,7 @@ dependencies { // here because we need to manually open the VR keyboard compileOnly fg.deobf("curse.maven:vivecraft-667903:4794431") - implementation fg.deobf("com.cinemamod:mcef-forge:2.1.1-1.20.1") { + implementation fg.deobf("com.cinemamod:mcef-forge:2.1.5-1.20.1") { transitive = false } } diff --git a/src/main/java/net/montoyo/wd/client/ClientProxy.java b/src/main/java/net/montoyo/wd/client/ClientProxy.java index e878e20..832c676 100644 --- a/src/main/java/net/montoyo/wd/client/ClientProxy.java +++ b/src/main/java/net/montoyo/wd/client/ClientProxy.java @@ -61,6 +61,7 @@ import net.minecraftforge.network.NetworkEvent; import net.montoyo.wd.SharedProxy; import net.montoyo.wd.WebDisplays; import net.montoyo.wd.block.ScreenBlock; +import net.montoyo.wd.client.audio.WDAudioHandler; import net.montoyo.wd.client.gui.*; import net.montoyo.wd.client.gui.loading.GuiLoader; import net.montoyo.wd.client.renderers.*; @@ -286,6 +287,7 @@ public class ClientProxy extends SharedProxy implements ResourceManagerReloadLis } ); + MCEF.getClient().addAudioHandler(WDAudioHandler.INSTANCE); MCEF.getClient().addDisplayHandler(DisplayHandler.INSTANCE); MCEF.getClient().getHandle().addMessageRouter(CefMessageRouter.create(WDRouter.INSTANCE)); diff --git a/src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java b/src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java new file mode 100644 index 0000000..a2a90ae --- /dev/null +++ b/src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java @@ -0,0 +1,84 @@ +package net.montoyo.wd.client.audio; + +import net.montoyo.wd.WebDisplays; +import net.montoyo.wd.client.ClientProxy; +import net.montoyo.wd.entity.ScreenBlockEntity; +import net.montoyo.wd.utilities.Log; +import org.cef.browser.CefBrowser; +import org.cef.handler.CefAudioHandler; +import org.cef.misc.CefAudioParameters; +import org.cef.misc.DataPointer; + +public class WDAudioHandler implements CefAudioHandler { + public static final WDAudioHandler INSTANCE = new WDAudioHandler(); + + @Override + public boolean getAudioParameters(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters) { + ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY); + + boolean didParameterize = false; + + for (ScreenBlockEntity tes : proxy.getScreens()) { + WDAudioSource source = tes.getSoundSource(cefBrowser); + if (source != null) { + source.parameterize(cefAudioParameters); + didParameterize = true; + } + } + + return didParameterize; + } + + @Override + public void onAudioStreamStarted(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters, int channels) { + ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY); + + if (cefAudioParameters == null) return; + + for (ScreenBlockEntity tes : proxy.getScreens()) { + WDAudioSource source = tes.getSoundSource(cefBrowser); + if (source != null) { + source.parameterize(channels, cefAudioParameters); + } + } + } + + // + // Called on the audio stream thread when a PCM packet is received for the + // stream. |data| is an array representing the raw PCM data as a floating + // point type, i.e. 4-byte value(s). |frames| is the number of frames in the + // PCM packet. |pts| is the presentation timestamp (in milliseconds since the + // Unix Epoch) and represents the time at which the decompressed packet + // should be presented to the user. Based on |frames| and the + // |channel_layout| value passed to OnAudioStreamStarted you can calculate + // the size of the |data| array in bytes. + // + @Override + public void onAudioStreamPacket(CefBrowser cefBrowser, long pointer, int frames, long pts) { + ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY); + + for (ScreenBlockEntity tes : proxy.getScreens()) { + WDAudioSource source = tes.getSoundSource(cefBrowser); + if (source != null) { + source.audioStream.setData(pointer); + } + } + } + + @Override + public void onAudioStreamStopped(CefBrowser cefBrowser) { + + } + + @Override + public void onAudioStreamError(CefBrowser cefBrowser, String s) { + ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY); + + for (ScreenBlockEntity tes : proxy.getScreens()) { + WDAudioSource source = tes.getSoundSource(cefBrowser); + if (source != null) { + Log.warning("Audio stream errored: " + s); + } + } + } +} diff --git a/src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java b/src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java index 62ca600..dc9b35b 100644 --- a/src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java +++ b/src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java @@ -12,105 +12,115 @@ import net.minecraft.util.RandomSource; import net.minecraft.util.valueproviders.SampledFloat; import net.montoyo.wd.entity.ScreenBlockEntity; import net.montoyo.wd.entity.ScreenData; +import org.cef.misc.CefAudioParameters; import org.jetbrains.annotations.Nullable; import java.util.concurrent.CompletableFuture; public class WDAudioSource implements SoundInstance { - private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source"); - private static final WeighedSoundEvents events = new WeighedSoundEvents( - location, "webdisplays.browser" - ); - private static final SampledFloat CONST_1 = new SampledFloat() { - @Override - public float sample(RandomSource pRandom) { - return 1.0f; - } - }; - private final Sound sound = new Sound( - "unused", - CONST_1, - CONST_1, - 1, Sound.Type.SOUND_EVENT, - true, false, - 100 - ); - ScreenBlockEntity blockEntity; - ScreenData data; - - public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) { - this.blockEntity = blockEntity; - this.data = data; - } - - @Override - public ResourceLocation getLocation() { - return location; - } - - @Nullable - @Override - public WeighedSoundEvents resolve(SoundManager pManager) { - return events; - } - - @Override - public CompletableFuture getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) { - return null; - } - - @Override - public Sound getSound() { - return sound; - } - - @Override - public SoundSource getSource() { - return SoundSource.RECORDS; - } - - @Override - public boolean isLooping() { - return true; - } - - @Override - public boolean isRelative() { - return false; - } - - @Override - public int getDelay() { - return 0; - } - - @Override - public float getVolume() { - return blockEntity.ytVolume; - } - - @Override - public float getPitch() { - return 1; - } - - @Override - public double getX() { - return blockEntity.getBlockPos().getX(); - } - - @Override - public double getY() { - return blockEntity.getBlockPos().getY(); - } - - @Override - public double getZ() { - return blockEntity.getBlockPos().getZ(); - } - - @Override - public Attenuation getAttenuation() { - return Attenuation.LINEAR; - } + private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source"); + private static final WeighedSoundEvents events = new WeighedSoundEvents( + location, "webdisplays.browser" + ); + private static final SampledFloat CONST_1 = new SampledFloat() { + @Override + public float sample(RandomSource pRandom) { + return 1.0f; + } + }; + private final Sound sound = new Sound( + "unused", + CONST_1, + CONST_1, + 1, Sound.Type.SOUND_EVENT, + true, false, + 100 + ); + ScreenBlockEntity blockEntity; + ScreenData data; + WDAudioStream audioStream = new WDAudioStream(); + + public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) { + this.blockEntity = blockEntity; + this.data = data; + } + + public void parameterize(int channels, CefAudioParameters cefAudioParameters) { + audioStream.setFormat(channels, cefAudioParameters); + } + + public void parameterize(CefAudioParameters cefAudioParameters) { + audioStream.setFormat(cefAudioParameters); + } + + @Override + public ResourceLocation getLocation() { + return location; + } + + @Nullable + @Override + public WeighedSoundEvents resolve(SoundManager pManager) { + return events; + } + + @Override + public CompletableFuture getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) { + return CompletableFuture.completedFuture(audioStream); + } + + @Override + public Sound getSound() { + return sound; + } + + @Override + public SoundSource getSource() { + return SoundSource.RECORDS; + } + + @Override + public boolean isLooping() { + return true; + } + + @Override + public boolean isRelative() { + return false; + } + + @Override + public int getDelay() { + return 0; + } + + @Override + public float getVolume() { + return blockEntity.ytVolume; + } + + @Override + public float getPitch() { + return 1; + } + + @Override + public double getX() { + return blockEntity.getBlockPos().getX(); + } + + @Override + public double getY() { + return blockEntity.getBlockPos().getY(); + } + + @Override + public double getZ() { + return blockEntity.getBlockPos().getZ(); + } + + @Override + public Attenuation getAttenuation() { + return Attenuation.LINEAR; + } } diff --git a/src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java b/src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java new file mode 100644 index 0000000..e827a53 --- /dev/null +++ b/src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java @@ -0,0 +1,106 @@ +package net.montoyo.wd.client.audio; + +import net.minecraft.client.sounds.AudioStream; +import org.cef.misc.CefAudioParameters; +import org.cef.misc.CefChannelLayout; +import org.cef.misc.DataPointer; +import org.checkerframework.checker.units.qual.A; +import org.lwjgl.system.MemoryUtil; + +import javax.sound.sampled.AudioFormat; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; + +public class WDAudioStream implements AudioStream { + AudioFormat currentFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100, 16, + 1, (4096 / 8) * 2, 44100, + false + ); + + @Override + public AudioFormat getFormat() { + return currentFormat; + } + + int fpb; + + public void setFormat(int channels, CefAudioParameters cefAudioParameters) { + currentFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + cefAudioParameters.sampleRate, 16, + 1, (16 / 8) * channels, cefAudioParameters.sampleRate, + false + ); + fpb = cefAudioParameters.framesPerBuffer; + } + + public void setFormat(CefAudioParameters cefAudioParameters) { + CefChannelLayout layout = cefAudioParameters.channelLayout; + int channels = 0; + if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_MONO) + channels = 1; + else if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_STEREO) + channels = 2; + setFormat(channels, cefAudioParameters); + } + + ArrayDeque buffers = new ArrayDeque<>(); + + public void setData(long data) { +// DataPointer ptr = data.forCapacity(currentFormat.getChannels() << 3); +// ptr.setAlignment(3); + long baseAddr = data; + for (int i = 0; i < 1; i++) { + long addr = MemoryUtil.memGetLong(baseAddr + (i << 3)); + int cap = fpb; + float[] flts = new float[cap]; + for (int i1 = 0; i1 < cap; i1++) { + flts[i1] = MemoryUtil.memGetFloat(addr + (i1 << 2)); + } + buffers.add(flts); + } + } + + @Override + public ByteBuffer read(int pSize) throws IOException { + System.out.println(buffers.size()); + ByteBuffer buffer = ByteBuffer.allocateDirect(pSize); + if (!buffers.isEmpty()) { + final int MAX_16_BIT = 32767; + final int MIN_16_BIT = -32768; + + loopBufs: + while (true) { + if (!buffers.isEmpty()) { + for (float v : buffers.pop()) { + if (buffer.position() >= pSize) break loopBufs; +// buffer.putFloat(v); + + // Scale and clip the float value to the range of a signed 16-bit int + float floatSample = v; + int intSample = (int) (floatSample * MAX_16_BIT); + + if (intSample > MAX_16_BIT) { + intSample = MAX_16_BIT; + } else if (intSample < MIN_16_BIT) { + intSample = MIN_16_BIT; + } + + // Convert the int sample to bytes (little-endian format) + buffer.put((byte) (intSample & 0xFF)); + buffer.put((byte) ((intSample >> 8) & 0xFF)); + } + } else break; + } + buffer.position(0); + } + return buffer; + } + + @Override + public void close() throws IOException { + } +} diff --git a/src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java b/src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java index 0e5ef47..d22a468 100644 --- a/src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java +++ b/src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java @@ -28,6 +28,7 @@ import net.minecraftforge.network.PacketDistributor; import net.montoyo.wd.WebDisplays; import net.montoyo.wd.block.ScreenBlock; import net.montoyo.wd.client.ClientProxy; +import net.montoyo.wd.client.audio.WDAudioSource; import net.montoyo.wd.config.CommonConfig; import net.montoyo.wd.controls.builtin.ClickControl; import net.montoyo.wd.core.DefaultUpgrade; @@ -1143,6 +1144,15 @@ public class ScreenBlockEntity extends BlockEntity { return bhr; } + + public WDAudioSource getSoundSource(CefBrowser cefBrowser) { + for (ScreenData screen : screens) { + if (screen.browser == cefBrowser) { + return screen.audioSource; + } + } + return null; + } // @Override // public boolean shouldRefresh(Level world, BlockPos pos, @Nonnull BlockState oldState, @Nonnull BlockState newState) { diff --git a/src/main/java/net/montoyo/wd/entity/ScreenData.java b/src/main/java/net/montoyo/wd/entity/ScreenData.java index 3e79256..b8afa72 100644 --- a/src/main/java/net/montoyo/wd/entity/ScreenData.java +++ b/src/main/java/net/montoyo/wd/entity/ScreenData.java @@ -3,6 +3,7 @@ package net.montoyo.wd.entity; import com.cinemamod.mcef.MCEF; import com.cinemamod.mcef.MCEFBrowser; import com.cinemamod.mcef.listeners.MCEFCursorChangeListener; +import net.minecraft.client.Minecraft; import net.minecraft.core.BlockPos; import net.minecraft.core.Direction; import net.minecraft.nbt.CompoundTag; @@ -13,6 +14,7 @@ import net.minecraft.world.level.Level; import net.minecraft.world.level.block.entity.BlockEntity; import net.montoyo.wd.WebDisplays; import net.montoyo.wd.client.ClientProxy; +import net.montoyo.wd.client.audio.WDAudioSource; import net.montoyo.wd.config.CommonConfig; import net.montoyo.wd.core.ScreenRights; import net.montoyo.wd.utilities.*; @@ -33,6 +35,7 @@ public class ScreenData { public Vector2i resolution; public Rotation rotation = Rotation.ROT_0; public String url; + public WDAudioSource audioSource; protected VideoType videoType; public NameUUIDPair owner; public ArrayList friends; @@ -206,6 +209,12 @@ public class ScreenData { doTurnOnAnim = doAnim; turnOnTime = System.currentTimeMillis(); + + audioSource = new WDAudioSource( + be, + this + ); + Minecraft.getInstance().getSoundManager().play(audioSource); } } }