Leisuretimedock/webdisplays
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

audio handling works, but is ~43 audio packets delayed

Browse Source
This commit is contained in:
GiantLuigi4 2024-10-20 00:31:00 -04:00
parent ecb0dcee56
commit 3b7c4a5bf8
7 changed files with 319 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.gradle
View File

@ -80,6 +80,7 @@ repositories{
url = "https://www.cursemaven.com"
}
maven { url 'https://mcef-download.cinemamod.com/repositories/releases/' }
flatDir name: 'libs', dirs: 'libs'
}
dependencies {
@ -91,7 +92,7 @@ dependencies {
// here because we need to manually open the VR keyboard
compileOnly fg.deobf("curse.maven:vivecraft-667903:4794431")
implementation fg.deobf("com.cinemamod:mcef-forge:2.1.1-1.20.1") {
implementation fg.deobf("com.cinemamod:mcef-forge:2.1.5-1.20.1") {
transitive = false
}
}

2
src/main/java/net/montoyo/wd/client/ClientProxy.java
View File

@ -61,6 +61,7 @@ import net.minecraftforge.network.NetworkEvent;
import net.montoyo.wd.SharedProxy;
import net.montoyo.wd.WebDisplays;
import net.montoyo.wd.block.ScreenBlock;
import net.montoyo.wd.client.audio.WDAudioHandler;
import net.montoyo.wd.client.gui.*;
import net.montoyo.wd.client.gui.loading.GuiLoader;
import net.montoyo.wd.client.renderers.*;
@ -286,6 +287,7 @@ public class ClientProxy extends SharedProxy implements ResourceManagerReloadLis
}
);
MCEF.getClient().addAudioHandler(WDAudioHandler.INSTANCE);
MCEF.getClient().addDisplayHandler(DisplayHandler.INSTANCE);
MCEF.getClient().getHandle().addMessageRouter(CefMessageRouter.create(WDRouter.INSTANCE));

84
src/main/java/net/montoyo/wd/client/audio/WDAudioHandler.java Normal file
View File

@ -0,0 +1,84 @@
package net.montoyo.wd.client.audio;
import net.montoyo.wd.WebDisplays;
import net.montoyo.wd.client.ClientProxy;
import net.montoyo.wd.entity.ScreenBlockEntity;
import net.montoyo.wd.utilities.Log;
import org.cef.browser.CefBrowser;
import org.cef.handler.CefAudioHandler;
import org.cef.misc.CefAudioParameters;
import org.cef.misc.DataPointer;
public class WDAudioHandler implements CefAudioHandler {
public static final WDAudioHandler INSTANCE = new WDAudioHandler();
@Override
public boolean getAudioParameters(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters) {
ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
boolean didParameterize = false;
for (ScreenBlockEntity tes : proxy.getScreens()) {
WDAudioSource source = tes.getSoundSource(cefBrowser);
if (source != null) {
source.parameterize(cefAudioParameters);
didParameterize = true;
}
}
return didParameterize;
}
@Override
public void onAudioStreamStarted(CefBrowser cefBrowser, CefAudioParameters cefAudioParameters, int channels) {
ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
if (cefAudioParameters == null) return;
for (ScreenBlockEntity tes : proxy.getScreens()) {
WDAudioSource source = tes.getSoundSource(cefBrowser);
if (source != null) {
source.parameterize(channels, cefAudioParameters);
}
}
}
//
// Called on the audio stream thread when a PCM packet is received for the
// stream. |data| is an array representing the raw PCM data as a floating
// point type, i.e. 4-byte value(s). |frames| is the number of frames in the
// PCM packet. |pts| is the presentation timestamp (in milliseconds since the
// Unix Epoch) and represents the time at which the decompressed packet
// should be presented to the user. Based on |frames| and the
// |channel_layout| value passed to OnAudioStreamStarted you can calculate
// the size of the |data| array in bytes.
//
@Override
public void onAudioStreamPacket(CefBrowser cefBrowser, long pointer, int frames, long pts) {
ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
for (ScreenBlockEntity tes : proxy.getScreens()) {
WDAudioSource source = tes.getSoundSource(cefBrowser);
if (source != null) {
source.audioStream.setData(pointer);
}
}
}
@Override
public void onAudioStreamStopped(CefBrowser cefBrowser) {
}
@Override
public void onAudioStreamError(CefBrowser cefBrowser, String s) {
ClientProxy proxy = ((ClientProxy) WebDisplays.PROXY);
for (ScreenBlockEntity tes : proxy.getScreens()) {
WDAudioSource source = tes.getSoundSource(cefBrowser);
if (source != null) {
Log.warning("Audio stream errored: " + s);
}
}
}
}

202
src/main/java/net/montoyo/wd/client/audio/WDAudioSource.java
View File

@ -12,105 +12,115 @@ import net.minecraft.util.RandomSource;
import net.minecraft.util.valueproviders.SampledFloat;
import net.montoyo.wd.entity.ScreenBlockEntity;
import net.montoyo.wd.entity.ScreenData;
import org.cef.misc.CefAudioParameters;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.CompletableFuture;
public class WDAudioSource implements SoundInstance {
private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
private static final WeighedSoundEvents events = new WeighedSoundEvents(
location, "webdisplays.browser"
);
private static final SampledFloat CONST_1 = new SampledFloat() {
@Override
public float sample(RandomSource pRandom) {
return 1.0f;
}
};
private final Sound sound = new Sound(
"unused",
CONST_1,
CONST_1,
1, Sound.Type.SOUND_EVENT,
true, false,
100
);
ScreenBlockEntity blockEntity;
ScreenData data;
public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
this.blockEntity = blockEntity;
this.data = data;
}
@Override
public ResourceLocation getLocation() {
return location;
}
@Nullable
@Override
public WeighedSoundEvents resolve(SoundManager pManager) {
return events;
}
@Override
public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
return null;
}
@Override
public Sound getSound() {
return sound;
}
@Override
public SoundSource getSource() {
return SoundSource.RECORDS;
}
@Override
public boolean isLooping() {
return true;
}
@Override
public boolean isRelative() {
return false;
}
@Override
public int getDelay() {
return 0;
}
@Override
public float getVolume() {
return blockEntity.ytVolume;
}
@Override
public float getPitch() {
return 1;
}
@Override
public double getX() {
return blockEntity.getBlockPos().getX();
}
@Override
public double getY() {
return blockEntity.getBlockPos().getY();
}
@Override
public double getZ() {
return blockEntity.getBlockPos().getZ();
}
@Override
public Attenuation getAttenuation() {
return Attenuation.LINEAR;
}
private static final ResourceLocation location = new ResourceLocation("webdisplays:audio_source");
private static final WeighedSoundEvents events = new WeighedSoundEvents(
location, "webdisplays.browser"
);
private static final SampledFloat CONST_1 = new SampledFloat() {
@Override
public float sample(RandomSource pRandom) {
return 1.0f;
}
};
private final Sound sound = new Sound(
"unused",
CONST_1,
CONST_1,
1, Sound.Type.SOUND_EVENT,
true, false,
100
);
ScreenBlockEntity blockEntity;
ScreenData data;
WDAudioStream audioStream = new WDAudioStream();
public WDAudioSource(ScreenBlockEntity blockEntity, ScreenData data) {
this.blockEntity = blockEntity;
this.data = data;
}
public void parameterize(int channels, CefAudioParameters cefAudioParameters) {
audioStream.setFormat(channels, cefAudioParameters);
}
public void parameterize(CefAudioParameters cefAudioParameters) {
audioStream.setFormat(cefAudioParameters);
}
@Override
public ResourceLocation getLocation() {
return location;
}
@Nullable
@Override
public WeighedSoundEvents resolve(SoundManager pManager) {
return events;
}
@Override
public CompletableFuture<AudioStream> getStream(SoundBufferLibrary soundBuffers, Sound sound, boolean looping) {
return CompletableFuture.completedFuture(audioStream);
}
@Override
public Sound getSound() {
return sound;
}
@Override
public SoundSource getSource() {
return SoundSource.RECORDS;
}
@Override
public boolean isLooping() {
return true;
}
@Override
public boolean isRelative() {
return false;
}
@Override
public int getDelay() {
return 0;
}
@Override
public float getVolume() {
return blockEntity.ytVolume;
}
@Override
public float getPitch() {
return 1;
}
@Override
public double getX() {
return blockEntity.getBlockPos().getX();
}
@Override
public double getY() {
return blockEntity.getBlockPos().getY();
}
@Override
public double getZ() {
return blockEntity.getBlockPos().getZ();
}
@Override
public Attenuation getAttenuation() {
return Attenuation.LINEAR;
}
}

106
src/main/java/net/montoyo/wd/client/audio/WDAudioStream.java Normal file
View File

@ -0,0 +1,106 @@
package net.montoyo.wd.client.audio;
import net.minecraft.client.sounds.AudioStream;
import org.cef.misc.CefAudioParameters;
import org.cef.misc.CefChannelLayout;
import org.cef.misc.DataPointer;
import org.checkerframework.checker.units.qual.A;
import org.lwjgl.system.MemoryUtil;
import javax.sound.sampled.AudioFormat;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayDeque;
public class WDAudioStream implements AudioStream {
AudioFormat currentFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
44100, 16,
1, (4096 / 8) * 2, 44100,
false
);
@Override
public AudioFormat getFormat() {
return currentFormat;
}
int fpb;
public void setFormat(int channels, CefAudioParameters cefAudioParameters) {
currentFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
cefAudioParameters.sampleRate, 16,
1, (16 / 8) * channels, cefAudioParameters.sampleRate,
false
);
fpb = cefAudioParameters.framesPerBuffer;
}
public void setFormat(CefAudioParameters cefAudioParameters) {
CefChannelLayout layout = cefAudioParameters.channelLayout;
int channels = 0;
if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_MONO)
channels = 1;
else if (layout == CefChannelLayout.CEF_CHANNEL_LAYOUT_STEREO)
channels = 2;
setFormat(channels, cefAudioParameters);
}
ArrayDeque<float[]> buffers = new ArrayDeque<>();
public void setData(long data) {
// DataPointer ptr = data.forCapacity(currentFormat.getChannels() << 3);
// ptr.setAlignment(3);
long baseAddr = data;
for (int i = 0; i < 1; i++) {
long addr = MemoryUtil.memGetLong(baseAddr + (i << 3));
int cap = fpb;
float[] flts = new float[cap];
for (int i1 = 0; i1 < cap; i1++) {
flts[i1] = MemoryUtil.memGetFloat(addr + (i1 << 2));
}
buffers.add(flts);
}
}
@Override
public ByteBuffer read(int pSize) throws IOException {
System.out.println(buffers.size());
ByteBuffer buffer = ByteBuffer.allocateDirect(pSize);
if (!buffers.isEmpty()) {
final int MAX_16_BIT = 32767;
final int MIN_16_BIT = -32768;
loopBufs:
while (true) {
if (!buffers.isEmpty()) {
for (float v : buffers.pop()) {
if (buffer.position() >= pSize) break loopBufs;
// buffer.putFloat(v);
// Scale and clip the float value to the range of a signed 16-bit int
float floatSample = v;
int intSample = (int) (floatSample * MAX_16_BIT);
if (intSample > MAX_16_BIT) {
intSample = MAX_16_BIT;
} else if (intSample < MIN_16_BIT) {
intSample = MIN_16_BIT;
}
// Convert the int sample to bytes (little-endian format)
buffer.put((byte) (intSample & 0xFF));
buffer.put((byte) ((intSample >> 8) & 0xFF));
}
} else break;
}
buffer.position(0);
}
return buffer;
}
@Override
public void close() throws IOException {
}
}

10
src/main/java/net/montoyo/wd/entity/ScreenBlockEntity.java
View File

@ -28,6 +28,7 @@ import net.minecraftforge.network.PacketDistributor;
import net.montoyo.wd.WebDisplays;
import net.montoyo.wd.block.ScreenBlock;
import net.montoyo.wd.client.ClientProxy;
import net.montoyo.wd.client.audio.WDAudioSource;
import net.montoyo.wd.config.CommonConfig;
import net.montoyo.wd.controls.builtin.ClickControl;
import net.montoyo.wd.core.DefaultUpgrade;
@ -1143,6 +1144,15 @@ public class ScreenBlockEntity extends BlockEntity {
return bhr;
}
public WDAudioSource getSoundSource(CefBrowser cefBrowser) {
for (ScreenData screen : screens) {
if (screen.browser == cefBrowser) {
return screen.audioSource;
}
}
return null;
}
// @Override
// public boolean shouldRefresh(Level world, BlockPos pos, @Nonnull BlockState oldState, @Nonnull BlockState newState) {

9
src/main/java/net/montoyo/wd/entity/ScreenData.java
View File

@ -3,6 +3,7 @@ package net.montoyo.wd.entity;
import com.cinemamod.mcef.MCEF;
import com.cinemamod.mcef.MCEFBrowser;
import com.cinemamod.mcef.listeners.MCEFCursorChangeListener;
import net.minecraft.client.Minecraft;
import net.minecraft.core.BlockPos;
import net.minecraft.core.Direction;
import net.minecraft.nbt.CompoundTag;
@ -13,6 +14,7 @@ import net.minecraft.world.level.Level;
import net.minecraft.world.level.block.entity.BlockEntity;
import net.montoyo.wd.WebDisplays;
import net.montoyo.wd.client.ClientProxy;
import net.montoyo.wd.client.audio.WDAudioSource;
import net.montoyo.wd.config.CommonConfig;
import net.montoyo.wd.core.ScreenRights;
import net.montoyo.wd.utilities.*;
@ -33,6 +35,7 @@ public class ScreenData {
public Vector2i resolution;
public Rotation rotation = Rotation.ROT_0;
public String url;
public WDAudioSource audioSource;
protected VideoType videoType;
public NameUUIDPair owner;
public ArrayList<NameUUIDPair> friends;
@ -206,6 +209,12 @@ public class ScreenData {
doTurnOnAnim = doAnim;
turnOnTime = System.currentTimeMillis();
audioSource = new WDAudioSource(
be,
this
);
Minecraft.getInstance().getSoundManager().play(audioSource);
}
}
}