diff --git a/.idea/libraries/jetbrains_kotlinx_coroutines_core.xml b/.idea/libraries/jetbrains_kotlinx_coroutines_core.xml new file mode 100644 index 000000000..76054adeb --- /dev/null +++ b/.idea/libraries/jetbrains_kotlinx_coroutines_core.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/net/torvald/terrarum/App.java b/src/net/torvald/terrarum/App.java index c354b73a9..b94cde6fb 100644 --- a/src/net/torvald/terrarum/App.java +++ b/src/net/torvald/terrarum/App.java @@ -920,6 +920,8 @@ public class App implements ApplicationListener { audioManagerThread.interrupt(); } + AudioMixer.INSTANCE.dispose(); + if (currentScreen != null) { currentScreen.hide(); currentScreen.dispose(); diff --git a/src/net/torvald/terrarum/audio/AudioMixer.kt b/src/net/torvald/terrarum/audio/AudioMixer.kt index f3e5f8aa2..2abab216b 100644 --- a/src/net/torvald/terrarum/audio/AudioMixer.kt +++ b/src/net/torvald/terrarum/audio/AudioMixer.kt @@ -2,15 +2,17 @@ package net.torvald.terrarum.audio import com.badlogic.gdx.Gdx import com.badlogic.gdx.backends.lwjgl3.audio.Lwjgl3Audio +import com.badlogic.gdx.utils.Disposable import net.torvald.terrarum.App import net.torvald.terrarum.modulebasegame.MusicContainer +import net.torvald.terrarum.tryDispose /** * Any audio reference fed into this manager will get lost; you must manually store and dispose of them on your own. * * Created by minjaesong on 2023-11-07. */ -object AudioMixer { +object AudioMixer: Disposable { const val DEFAULT_FADEOUT_LEN = 2.4 /** Returns a master volume */ @@ -28,8 +30,8 @@ object AudioMixer { private val tracks = Array(10) { TerrarumAudioMixerTracks() } - private val masterTrack = TerrarumAudioMixerTracks().also { master -> - tracks.forEach { master.sidechainInputs.add(it to 1.0) } + private val masterTrack = TerrarumAudioMixerTracks(true).also { master -> + tracks.forEach { master.addSidechainInput(it, 1.0) } } private val musicTrack: TerrarumAudioMixerTracks @@ -122,4 +124,9 @@ object AudioMixer { } } + + override fun dispose() { + tracks.forEach { it.tryDispose() } + masterTrack.tryDispose() + } } \ No newline at end of file diff --git a/src/net/torvald/terrarum/audio/AudioProcessBuf.kt b/src/net/torvald/terrarum/audio/AudioProcessBuf.kt new file mode 100644 index 000000000..0a46dd024 --- /dev/null +++ b/src/net/torvald/terrarum/audio/AudioProcessBuf.kt @@ -0,0 +1,36 @@ +package net.torvald.terrarum.audio + +import net.torvald.terrarum.serialise.toUint + +/** + * Audio is assumed to be 16 bits + * + * Created by minjaesong on 2023-11-17. + */ +class AudioProcessBuf(val size: Int) { + + var buf0 = ByteArray(size); private set + var buf1 = ByteArray(size); private set + + var fbuf0 = FloatArray(size / 2); private set + var fbuf1 = FloatArray(size / 2); private set + + fun shift(): ByteArray { + buf0 = buf1 + buf1 = ByteArray(size) + + fbuf0 = fbuf1 + fbuf1 = FloatArray(size / 2) { + val i16 = (buf1[4*it].toUint() or buf1[4*it+1].toUint().shl(8)).toShort() + i16 / 32767f + } + + return buf1 + } + + fun getL0() = FloatArray(size / 4) { fbuf0[2*it] } + fun getR0() = FloatArray(size / 4) { fbuf0[2*it+1] } + fun getL1() = FloatArray(size / 4) { fbuf1[2*it] } + fun getR1() = FloatArray(size / 4) { fbuf1[2*it+1] } + +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/audio/OpenALBufferedAudioDevice.kt b/src/net/torvald/terrarum/audio/OpenALBufferedAudioDevice.kt new file mode 100644 index 000000000..2076ea92c --- /dev/null +++ b/src/net/torvald/terrarum/audio/OpenALBufferedAudioDevice.kt @@ -0,0 +1,255 @@ +package net.torvald.terrarum.audio + +import com.badlogic.gdx.audio.AudioDevice +import com.badlogic.gdx.backends.lwjgl3.audio.OpenALLwjgl3Audio +import com.badlogic.gdx.math.MathUtils +import com.badlogic.gdx.utils.GdxRuntimeException +import org.lwjgl.BufferUtils +import org.lwjgl.openal.AL10 +import org.lwjgl.openal.AL11 +import java.nio.Buffer +import java.nio.ByteBuffer +import java.nio.IntBuffer + +/** + * Created by minjaesong on 2023-01-01. + */ +/******************************************************************************* + * Copyright 2011 See AUTHORS file. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** @author Nathan Sweet + */ +class OpenALBufferedAudioDevice( + private val audio: OpenALLwjgl3Audio, + val rate: Int, + isMono: Boolean, + val bufferSize: Int, + val bufferCount: Int, + private val fillBufferCallback: () -> Unit +) : AudioDevice { + private val channels: Int + private var buffers: IntBuffer? = null + private var sourceID = -1 + private val format: Int + private var isPlaying = false + private var volume = 1f + private var renderedSeconds = 0f + private val secondsPerBuffer: Float + private var bytes: ByteArray? = null + private var bytesLength = 2 + private val tempBuffer: ByteBuffer + + /** + * Invoked whenever a buffer is emptied after writing samples + * + * Preferably you write 2-3 buffers worth of samples at the beginning of the playback + */ + + init { + channels = if (isMono) 1 else 2 + format = if (channels > 1) AL10.AL_FORMAT_STEREO16 else AL10.AL_FORMAT_MONO16 + secondsPerBuffer = bufferSize.toFloat() / bytesPerSample / channels / rate + tempBuffer = BufferUtils.createByteBuffer(bufferSize) + } + + override fun writeSamples(samples: ShortArray, offset: Int, numSamples: Int) { + if (bytes == null || bytes!!.size < numSamples * 2) bytes = ByteArray(numSamples * 2) + val end = Math.min(offset + numSamples, samples.size) + var i = offset + var ii = 0 + while (i < end) { + val sample = samples[i] + bytes!![ii++] = (sample.toInt() and 0xFF).toByte() + bytes!![ii++] = (sample.toInt() shr 8 and 0xFF).toByte() + i++ + } + bytesLength = ii + writeSamples(bytes!!, 0, numSamples * 2) + } + + override fun writeSamples(samples: FloatArray, offset: Int, numSamples: Int) { + if (bytes == null || bytes!!.size < numSamples * 2) bytes = ByteArray(numSamples * 2) + val end = Math.min(offset + numSamples, samples.size) + var i = offset + var ii = 0 + while (i < end) { + var floatSample = samples[i] + floatSample = MathUtils.clamp(floatSample, -1f, 1f) + val intSample = (floatSample * 32767).toInt() + bytes!![ii++] = (intSample and 0xFF).toByte() + bytes!![ii++] = (intSample shr 8 and 0xFF).toByte() + i++ + } + bytesLength = ii + writeSamples(bytes!!, 0, numSamples * 2) + } + + private fun audioObtainSource(isMusic: Boolean): Int { + val obtainSourceMethod = OpenALLwjgl3Audio::class.java.getDeclaredMethod("obtainSource", java.lang.Boolean.TYPE) + obtainSourceMethod.isAccessible = true + return obtainSourceMethod.invoke(audio, isMusic) as Int + } + private fun audioFreeSource(sourceID: Int) { + val freeSourceMethod = OpenALLwjgl3Audio::class.java.getDeclaredMethod("freeSource", java.lang.Integer.TYPE) + freeSourceMethod.isAccessible = true + freeSourceMethod.invoke(audio, sourceID) + } + + private val alErrors = hashMapOf( + AL10.AL_INVALID_NAME to "AL_INVALID_NAME", + AL10.AL_INVALID_ENUM to "AL_INVALID_ENUM", + AL10.AL_INVALID_VALUE to "AL_INVALID_VALUE", + AL10.AL_INVALID_OPERATION to "AL_INVALID_OPERATION", + AL10.AL_OUT_OF_MEMORY to "AL_OUT_OF_MEMORY" + ) + + fun writeSamples(data: ByteArray, offset: Int, length: Int) { + var offset = offset + var length = length + require(length >= 0) { "length cannot be < 0." } + if (sourceID == -1) { + sourceID = audioObtainSource(true) + if (sourceID == -1) return + if (buffers == null) { + buffers = BufferUtils.createIntBuffer(bufferCount) + AL10.alGetError() + AL10.alGenBuffers(buffers) + AL10.alGetError().let { + if (it != AL10.AL_NO_ERROR) throw GdxRuntimeException("Unabe to allocate audio buffers: ${alErrors[it]}") + } + } + AL10.alSourcei(sourceID, AL10.AL_LOOPING, AL10.AL_FALSE) + AL10.alSourcef(sourceID, AL10.AL_GAIN, volume) + // Fill initial buffers. + var queuedBuffers = 0 + for (i in 0 until bufferCount) { + val bufferID = buffers!![i] + val written = Math.min(bufferSize, length) + (tempBuffer as Buffer).clear() + (tempBuffer.put(data, offset, written) as Buffer).flip() + AL10.alBufferData(bufferID, format, tempBuffer, rate) + AL10.alSourceQueueBuffers(sourceID, bufferID) + length -= written + offset += written + queuedBuffers++ + } + // Queue rest of buffers, empty. + (tempBuffer as Buffer).clear().flip() + for (i in queuedBuffers until bufferCount) { + val bufferID = buffers!![i] + AL10.alBufferData(bufferID, format, tempBuffer, rate) + AL10.alSourceQueueBuffers(sourceID, bufferID) + } + AL10.alSourcePlay(sourceID) + isPlaying = true + } + while (length > 0) { + val written = fillBuffer(data, offset, length) + length -= written + offset += written + } + } + + /** Blocks until some of the data could be buffered. */ + private fun fillBuffer(data: ByteArray, offset: Int, length: Int): Int { + val written = Math.min(bufferSize, length) + outer@ while (true) { + var buffers = AL10.alGetSourcei(sourceID, AL10.AL_BUFFERS_PROCESSED) + while (buffers-- > 0) { + val bufferID = AL10.alSourceUnqueueBuffers(sourceID) + if (bufferID == AL10.AL_INVALID_VALUE) break + renderedSeconds += secondsPerBuffer + (tempBuffer as Buffer).clear() + (tempBuffer.put(data, offset, written) as Buffer).flip() + AL10.alBufferData(bufferID, format, tempBuffer, rate) + AL10.alSourceQueueBuffers(sourceID, bufferID) + break@outer + } + // Wait for buffer to be free. + try { + Thread.sleep((1000 * secondsPerBuffer).toLong()) + fillBufferCallback() + } + catch (ignored: InterruptedException) { + } + } + + // A buffer underflow will cause the source to stop. + if (!isPlaying || AL10.alGetSourcei(sourceID, AL10.AL_SOURCE_STATE) != AL10.AL_PLAYING) { + AL10.alSourcePlay(sourceID) + isPlaying = true + } + return written + } + + fun stop() { + if (sourceID == -1) return + audioFreeSource(sourceID) + sourceID = -1 + renderedSeconds = 0f + isPlaying = false + } + + fun isPlaying(): Boolean { + return if (sourceID == -1) false else isPlaying + } + + override fun setVolume(volume: Float) { + this.volume = volume + if (sourceID != -1) AL10.alSourcef(sourceID, AL10.AL_GAIN, volume) + } + + var position: Float + get() = if (sourceID == -1) 0f else renderedSeconds + AL10.alGetSourcef(sourceID, AL11.AL_SEC_OFFSET) + set(position) { + renderedSeconds = position + } + + fun getChannels(): Int { + return if (format == AL10.AL_FORMAT_STEREO16) 2 else 1 + } + + override fun dispose() { + if (buffers == null) return + if (sourceID != -1) { + audioFreeSource(sourceID) + sourceID = -1 + } + AL10.alDeleteBuffers(buffers) + buffers = null + } + + override fun isMono(): Boolean { + return channels == 1 + } + + override fun getLatency(): Int { + return (secondsPerBuffer * bufferCount * 1000).toInt() + } + + override fun pause() { + // A buffer underflow will cause the source to stop. + } + + override fun resume() { + // Automatically resumes when samples are written + } + + companion object { + private const val bytesPerSample = 2 + private val ui8toI16Hi = ByteArray(256) { (128 + it).toByte() } + + } +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt b/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt index 74f7dc5a0..ee19b6884 100644 --- a/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt +++ b/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt @@ -1,12 +1,41 @@ package net.torvald.terrarum.audio +import com.jme3.math.FastMath + interface TerrarumAudioFilters { - fun thru(inbufL: FloatArray, inbufR: FloatArray, outbufL: FloatArray, outbufR: FloatArray) + fun thru(inbuf0: List, inbuf1: List, outbuf0: List, outbuf1: List) } object NullFilter: TerrarumAudioFilters { - override fun thru(inbufL: FloatArray, inbufR: FloatArray, outbufL: FloatArray, outbufR: FloatArray) { - System.arraycopy(inbufL, 0, outbufL, 0, inbufL.size) - System.arraycopy(inbufR, 0, outbufR, 0, inbufL.size) + override fun thru(inbuf0: List, inbuf1: List, outbuf0: List, outbuf1: List) { + outbuf1.forEachIndexed { index, outTrack -> + System.arraycopy(inbuf1[index], 0, outTrack, 0, outTrack.size) + } } } + + +class Lowpass(cutoff: Int, rate: Int): TerrarumAudioFilters { + + val alpha: Float + init { + val RC: Float = 1f / (cutoff.toFloat() * FastMath.TWO_PI) + val dt: Float = 1f / rate + alpha = dt / (RC + dt) + } + + override fun thru(inbuf0: List, inbuf1: List, outbuf0: List, outbuf1: List) { + for (ch in outbuf1.indices) { + val out = outbuf1[ch] + val inn = inbuf1[ch] +// System.arraycopy(inn, 0, out, 0, out.size) + + out[0] = outbuf0[ch].last() + + for (i in 1 until outbuf1[ch].size) { + out[i] = out[i-1] + (alpha * (inn[i] - out[i-1])) + } + } + } + +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt b/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt index 88b09a6fb..cfa507063 100644 --- a/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt +++ b/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt @@ -1,14 +1,25 @@ package net.torvald.terrarum.audio +import com.badlogic.gdx.Gdx +import com.badlogic.gdx.backends.lwjgl3.audio.OpenALLwjgl3Audio +import com.badlogic.gdx.utils.Disposable +import kotlinx.coroutines.* +import net.torvald.reflection.forceInvoke import net.torvald.terrarum.getHashStr import net.torvald.terrarum.modulebasegame.MusicContainer -import net.torvald.terrarum.utils.PasswordBase32 +import kotlin.coroutines.Continuation +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine import kotlin.math.log10 import kotlin.math.pow typealias TrackVolume = Double -class TerrarumAudioMixerTracks { +class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { + + companion object { + const val SAMPLING_RATE = 48000 + } val hash = getHashStr() @@ -28,8 +39,50 @@ class TerrarumAudioMixerTracks { get() = fullscaleToDecibels(volume) set(value) { volume = decibelsToFullscale(value) } - val filters = arrayListOf() - val sidechainInputs = arrayListOf>() + val filters = Array(4) { NullFilter } + + private val sidechainInputs = Array?>(16) { null } + internal fun getSidechains(): List = sidechainInputs.map { it?.first } + fun addSidechainInput(input: TerrarumAudioMixerTracks, inputVolume: TrackVolume) { + if (input.isMaster) + throw IllegalArgumentException("Cannot add master track as a sidechain") + + if (sidechainInputs.map { it?.first }.any { it?.hash == input.hash }) + throw IllegalArgumentException("The track '${input.hash}' already exists") + + if (getSidechains().any { mySidechain -> + val theirSidechains = mySidechain?.getSidechains() + theirSidechains?.any { theirSidechain -> theirSidechain?.hash == this.hash } == true + }) + throw IllegalArgumentException("The track '${input.hash}' contains current track (${this.hash}) as its sidechain") + + val emptySpot = sidechainInputs.indexOf(null) + if (emptySpot != -1) { + sidechainInputs[emptySpot] = (input to inputVolume) + } + else { + throw IllegalStateException("Sidechain is full (${sidechainInputs.size})!") + } + } + + + // in bytes + private val deviceBufferSize = Gdx.audio.javaClass.getDeclaredField("deviceBufferSize").let { + it.isAccessible = true + it.get(Gdx.audio) as Int + } + private val deviceBufferCount = Gdx.audio.javaClass.getDeclaredField("deviceBufferCount").let { + it.isAccessible = true + it.get(Gdx.audio) as Int + } + private val adev = OpenALBufferedAudioDevice( + Gdx.audio as OpenALLwjgl3Audio, + SAMPLING_RATE, + false, + deviceBufferSize, + deviceBufferCount + ) {} + /** * assign nextTrack to currentTrack, then assign nextNext to nextTrack. @@ -40,15 +93,77 @@ class TerrarumAudioMixerTracks { nextTrack = nextNext } + + private var streamPlaying = false fun play() { - currentTrack?.gdxMusic?.play() + streamPlaying = true +// currentTrack?.gdxMusic?.play() } val isPlaying: Boolean? get() = currentTrack?.gdxMusic?.isPlaying + override fun dispose() { + adev.dispose() + } override fun equals(other: Any?) = this.hash == (other as TerrarumAudioMixerTracks).hash + + + // 1st ring of the hell: the THREADING HELL // + + private val processJob: Job + private var processContinuation: Continuation? = null + + + + private val streamBuf = AudioProcessBuf(deviceBufferSize) + private val sideChainBufs = Array(sidechainInputs.size) { AudioProcessBuf(deviceBufferSize) } + private val outBufL0 = FloatArray(deviceBufferSize / 4) + private val outBufR0 = FloatArray(deviceBufferSize / 4) + private val outBufL1 = FloatArray(deviceBufferSize / 4) + private val outBufR1 = FloatArray(deviceBufferSize / 4) + + init { + processJob = GlobalScope.launch { // calling 'launch' literally launches the coroutine right awya + // fetch deviceBufferSize amount of sample from the disk + if (streamPlaying) { + currentTrack?.gdxMusic?.forceInvoke("read", arrayOf(streamBuf.shift())) + } + + // also fetch samples from sidechainInputs + // TODO + + // combine all the inputs + // TODO this code just uses streamBuf + + val samplesL0 = streamBuf.getL0() + val samplesR0 = streamBuf.getR0() + val samplesL1 = streamBuf.getL1() + val samplesR1 = streamBuf.getR1() + + // run the input through the stack of filters + // TODO skipped lol + + // final writeout + System.arraycopy(samplesL0, 0, outBufL0, 0, outBufL0.size) + System.arraycopy(samplesR0, 0, outBufR0, 0, outBufR0.size) + System.arraycopy(samplesL1, 0, outBufL1, 0, outBufL1.size) + System.arraycopy(samplesR1, 0, outBufR1, 0, outBufR1.size) + + // by this time, the output buffer is filled with processed results, pause the execution + if (!isMaster) { + suspendCoroutine { + processContinuation = it + } + } + else { + getSidechains().forEach { + it?.processContinuation?.resume(Unit) + } + } + } + } } fun fullscaleToDecibels(fs: Double) = 10.0 * log10(fs)