From eb5483ae375531a641bbff2b39547daa4dd11235 Mon Sep 17 00:00:00 2001 From: minjaesong Date: Fri, 17 Nov 2023 19:58:15 +0900 Subject: [PATCH] somewhat working audio pipeline --- src/net/torvald/terrarum/audio/AudioMixer.kt | 8 +- .../torvald/terrarum/audio/AudioProcessBuf.kt | 12 +- .../terrarum/audio/MixerTrackProcessor.kt | 156 ++++++++++++++++++ ...AudioFilters.kt => TerrarumAudioFilter.kt} | 7 +- ...erTracks.kt => TerrarumAudioMixerTrack.kt} | 83 ++-------- 5 files changed, 186 insertions(+), 80 deletions(-) create mode 100644 src/net/torvald/terrarum/audio/MixerTrackProcessor.kt rename src/net/torvald/terrarum/audio/{TerrarumAudioFilters.kt => TerrarumAudioFilter.kt} (84%) rename src/net/torvald/terrarum/audio/{TerrarumAudioMixerTracks.kt => TerrarumAudioMixerTrack.kt} (51%) diff --git a/src/net/torvald/terrarum/audio/AudioMixer.kt b/src/net/torvald/terrarum/audio/AudioMixer.kt index 2abab216b..1c28050a6 100644 --- a/src/net/torvald/terrarum/audio/AudioMixer.kt +++ b/src/net/torvald/terrarum/audio/AudioMixer.kt @@ -28,15 +28,15 @@ object AudioMixer: Disposable { get() = (App.getConfigDouble("sfxvolume") * App.getConfigDouble("mastervolume")) - private val tracks = Array(10) { TerrarumAudioMixerTracks() } + private val tracks = Array(10) { TerrarumAudioMixerTrack("Audio Track #${it+1}") } - private val masterTrack = TerrarumAudioMixerTracks(true).also { master -> + private val masterTrack = TerrarumAudioMixerTrack("Master", true).also { master -> tracks.forEach { master.addSidechainInput(it, 1.0) } } - private val musicTrack: TerrarumAudioMixerTracks + private val musicTrack: TerrarumAudioMixerTrack get() = tracks[0] - private val ambientTrack: TerrarumAudioMixerTracks + private val ambientTrack: TerrarumAudioMixerTrack get() = tracks[1] private var fadeAkku = 0.0 diff --git a/src/net/torvald/terrarum/audio/AudioProcessBuf.kt b/src/net/torvald/terrarum/audio/AudioProcessBuf.kt index 0a46dd024..00a5f042a 100644 --- a/src/net/torvald/terrarum/audio/AudioProcessBuf.kt +++ b/src/net/torvald/terrarum/audio/AudioProcessBuf.kt @@ -15,17 +15,23 @@ class AudioProcessBuf(val size: Int) { var fbuf0 = FloatArray(size / 2); private set var fbuf1 = FloatArray(size / 2); private set - fun shift(): ByteArray { + private fun shift(): ByteArray { buf0 = buf1 buf1 = ByteArray(size) + return buf1 + } + private fun updateFloats() { fbuf0 = fbuf1 fbuf1 = FloatArray(size / 2) { - val i16 = (buf1[4*it].toUint() or buf1[4*it+1].toUint().shl(8)).toShort() + val i16 = (buf1[2*it].toUint() or buf1[2*it+1].toUint().shl(8)).toShort() i16 / 32767f } + } - return buf1 + fun fetchBytes(action: (ByteArray) -> Unit) { + action(shift()) + updateFloats() } fun getL0() = FloatArray(size / 4) { fbuf0[2*it] } diff --git a/src/net/torvald/terrarum/audio/MixerTrackProcessor.kt b/src/net/torvald/terrarum/audio/MixerTrackProcessor.kt new file mode 100644 index 000000000..49932b62b --- /dev/null +++ b/src/net/torvald/terrarum/audio/MixerTrackProcessor.kt @@ -0,0 +1,156 @@ +package net.torvald.terrarum.audio + +import net.torvald.reflection.forceInvoke +import net.torvald.terrarum.audio.TerrarumAudioMixerTrack.Companion.SAMPLING_RATE + +/** + * Created by minjaesong on 2023-11-17. + */ +class MixerTrackProcessor(val bufferSize: Int, val track: TerrarumAudioMixerTrack): Runnable { + @Volatile + private var running = true + + @Volatile + private var paused = false + private val pauseLock = java.lang.Object() + + + + + internal val streamBuf = AudioProcessBuf(bufferSize) + internal val sideChainBufs = Array(track.sidechainInputs.size) { AudioProcessBuf(bufferSize) } +// internal val outBufL0 = FloatArray(bufferSize / 4) +// internal val outBufR0 = FloatArray(bufferSize / 4) +// internal val outBufL1 = FloatArray(bufferSize / 4) +// internal val outBufR1 = FloatArray(bufferSize / 4) + + + private val testFilter = Lowpass(240, SAMPLING_RATE) + + private var fout0 = listOf(FloatArray(bufferSize / 4), FloatArray(bufferSize / 4)) + private var fout1 = listOf(FloatArray(bufferSize / 4), FloatArray(bufferSize / 4)) + + + override fun run() { + w@ while (running) { + synchronized(pauseLock) { + if (!running) { // may have changed while waiting to + // synchronize on pauseLock +// break@w + } + if (paused) { + try { + pauseLock.wait() // will cause this Thread to block until + // another thread calls pauseLock.notifyAll() + // Note that calling wait() will + // relinquish the synchronized lock that this + // thread holds on pauseLock so another thread + // can acquire the lock to call notifyAll() + // (link with explanation below this code) + } + catch (ex: InterruptedException) { +// break@w + } + if (!running) { // running might have changed since we paused +// break@w + } + } + } + // Your code here + + +// println("AudioMixerTrack ${track.name} (${track.hash}) streamPlaying=${track.streamPlaying}") + + // fetch deviceBufferSize amount of sample from the disk + if (!track.isMaster && track.streamPlaying) { + streamBuf.fetchBytes { + track.currentTrack?.gdxMusic?.forceInvoke("read", arrayOf(it)) +// for (i in it.indices) { it[i] = (Math.random() * 255).toInt().toByte() } + } + } + + // also fetch samples from sidechainInputs + // TODO + + // combine all the inputs + // TODO this code just uses streamBuf + + + var samplesL0: FloatArray + var samplesR0: FloatArray + var samplesL1: FloatArray + var samplesR1: FloatArray + + if (track.isMaster) { + val streamBuf = track.sidechainInputs[0]!!.first.processor.streamBuf + samplesL0 = streamBuf.getL0() + samplesR0 = streamBuf.getR0() + samplesL1 = streamBuf.getL1() + samplesR1 = streamBuf.getR1() + } + else { + samplesL0 = streamBuf.getL0() + samplesR0 = streamBuf.getR0() + samplesL1 = streamBuf.getL1() + samplesR1 = streamBuf.getR1() + } + + + // run the input through the stack of filters + val fin0 = listOf(samplesL0, samplesR0) + val fin1 = listOf(samplesL1, samplesR1) + fout0 = fout1 + fout1 = listOf(FloatArray(bufferSize / 4), FloatArray(bufferSize / 4)) + val filter = if (track.isMaster) testFilter else NullFilter + filter.thru(fin0, fin1, fout0, fout1) + + + // final writeout +// System.arraycopy(samplesL0, 0, outBufL0, 0, outBufL0.size) +// System.arraycopy(samplesR0, 0, outBufR0, 0, outBufR0.size) +// System.arraycopy(samplesL1, 0, outBufL1, 0, outBufL1.size) +// System.arraycopy(samplesR1, 0, outBufR1, 0, outBufR1.size) + + // by this time, the output buffer is filled with processed results, pause the execution + if (!track.isMaster) { + this.pause() + } + else { + track.adev!!.setVolume(AudioMixer.masterVolume.toFloat()) + val samples = interleave(fout1[0], fout1[1]) + track.adev!!.writeSamples(samples, 0, samples.size) + Thread.sleep(1) + + track.getSidechains().forEach { + it?.processor?.resume() + } + } + + + } + } + + private fun interleave(f1: FloatArray, f2: FloatArray) = FloatArray(f1.size + f2.size) { + if (it % 2 == 0) f1[it / 2] else f2[it / 2] + } + + fun stop() { + running = false + // you might also want to interrupt() the Thread that is + // running this Runnable, too, or perhaps call: + resume() + // to unblock + } + + fun pause() { + // you may want to throw an IllegalStateException if !running + paused = true + } + + fun resume() { + synchronized(pauseLock) { + paused = false + pauseLock.notifyAll() // Unblocks thread + } + } +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt b/src/net/torvald/terrarum/audio/TerrarumAudioFilter.kt similarity index 84% rename from src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt rename to src/net/torvald/terrarum/audio/TerrarumAudioFilter.kt index ee19b6884..b970665b3 100644 --- a/src/net/torvald/terrarum/audio/TerrarumAudioFilters.kt +++ b/src/net/torvald/terrarum/audio/TerrarumAudioFilter.kt @@ -2,11 +2,11 @@ package net.torvald.terrarum.audio import com.jme3.math.FastMath -interface TerrarumAudioFilters { +interface TerrarumAudioFilter { fun thru(inbuf0: List, inbuf1: List, outbuf0: List, outbuf1: List) } -object NullFilter: TerrarumAudioFilters { +object NullFilter: TerrarumAudioFilter { override fun thru(inbuf0: List, inbuf1: List, outbuf0: List, outbuf1: List) { outbuf1.forEachIndexed { index, outTrack -> System.arraycopy(inbuf1[index], 0, outTrack, 0, outTrack.size) @@ -15,7 +15,7 @@ object NullFilter: TerrarumAudioFilters { } -class Lowpass(cutoff: Int, rate: Int): TerrarumAudioFilters { +class Lowpass(cutoff: Int, rate: Int): TerrarumAudioFilter { val alpha: Float init { @@ -28,7 +28,6 @@ class Lowpass(cutoff: Int, rate: Int): TerrarumAudioFilters { for (ch in outbuf1.indices) { val out = outbuf1[ch] val inn = inbuf1[ch] -// System.arraycopy(inn, 0, out, 0, out.size) out[0] = outbuf0[ch].last() diff --git a/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt b/src/net/torvald/terrarum/audio/TerrarumAudioMixerTrack.kt similarity index 51% rename from src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt rename to src/net/torvald/terrarum/audio/TerrarumAudioMixerTrack.kt index 95608ce11..b9e9f7722 100644 --- a/src/net/torvald/terrarum/audio/TerrarumAudioMixerTracks.kt +++ b/src/net/torvald/terrarum/audio/TerrarumAudioMixerTrack.kt @@ -3,19 +3,15 @@ package net.torvald.terrarum.audio import com.badlogic.gdx.Gdx import com.badlogic.gdx.backends.lwjgl3.audio.OpenALLwjgl3Audio import com.badlogic.gdx.utils.Disposable -import kotlinx.coroutines.* import net.torvald.reflection.forceInvoke import net.torvald.terrarum.getHashStr import net.torvald.terrarum.modulebasegame.MusicContainer -import kotlin.coroutines.Continuation -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine import kotlin.math.log10 import kotlin.math.pow typealias TrackVolume = Double -class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { +class TerrarumAudioMixerTrack(val name: String, val isMaster: Boolean = false): Disposable { companion object { const val SAMPLING_RATE = 48000 @@ -41,9 +37,9 @@ class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { val filters = Array(4) { NullFilter } - private val sidechainInputs = Array?>(16) { null } - internal fun getSidechains(): List = sidechainInputs.map { it?.first } - fun addSidechainInput(input: TerrarumAudioMixerTracks, inputVolume: TrackVolume) { + internal val sidechainInputs = Array?>(16) { null } + internal fun getSidechains(): List = sidechainInputs.map { it?.first } + fun addSidechainInput(input: TerrarumAudioMixerTrack, inputVolume: TrackVolume) { if (input.isMaster) throw IllegalArgumentException("Cannot add master track as a sidechain") @@ -67,15 +63,15 @@ class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { // in bytes - private val deviceBufferSize = Gdx.audio.javaClass.getDeclaredField("deviceBufferSize").let { + internal val deviceBufferSize = Gdx.audio.javaClass.getDeclaredField("deviceBufferSize").let { it.isAccessible = true it.get(Gdx.audio) as Int } - private val deviceBufferCount = Gdx.audio.javaClass.getDeclaredField("deviceBufferCount").let { + internal val deviceBufferCount = Gdx.audio.javaClass.getDeclaredField("deviceBufferCount").let { it.isAccessible = true it.get(Gdx.audio) as Int } - private val adev: OpenALBufferedAudioDevice? = + internal val adev: OpenALBufferedAudioDevice? = if (isMaster) { OpenALBufferedAudioDevice( Gdx.audio as OpenALLwjgl3Audio, @@ -98,7 +94,7 @@ class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { } - private var streamPlaying = false + internal var streamPlaying = false fun play() { streamPlaying = true // currentTrack?.gdxMusic?.play() @@ -108,70 +104,19 @@ class TerrarumAudioMixerTracks(val isMaster: Boolean = false): Disposable { get() = currentTrack?.gdxMusic?.isPlaying override fun dispose() { + processor.stop() + processorThread.join() adev?.dispose() } - override fun equals(other: Any?) = this.hash == (other as TerrarumAudioMixerTracks).hash + override fun equals(other: Any?) = this.hash == (other as TerrarumAudioMixerTrack).hash // 1st ring of the hell: the THREADING HELL // - private val processJob: Job - private var processContinuation: Continuation? = null - - - - private val streamBuf = AudioProcessBuf(deviceBufferSize) - private val sideChainBufs = Array(sidechainInputs.size) { AudioProcessBuf(deviceBufferSize) } - private val outBufL0 = FloatArray(deviceBufferSize / 4) - private val outBufR0 = FloatArray(deviceBufferSize / 4) - private val outBufL1 = FloatArray(deviceBufferSize / 4) - private val outBufR1 = FloatArray(deviceBufferSize / 4) - - init { - processJob = GlobalScope.launch { // calling 'launch' literally launches the coroutine right awya - // fetch deviceBufferSize amount of sample from the disk - if (streamPlaying) { - currentTrack?.gdxMusic?.forceInvoke("read", arrayOf(streamBuf.shift())) - } - - // also fetch samples from sidechainInputs - // TODO - - // combine all the inputs - // TODO this code just uses streamBuf - - val samplesL0 = streamBuf.getL0() - val samplesR0 = streamBuf.getR0() - val samplesL1 = streamBuf.getL1() - val samplesR1 = streamBuf.getR1() - - // run the input through the stack of filters - // TODO skipped lol - - // final writeout - System.arraycopy(samplesL0, 0, outBufL0, 0, outBufL0.size) - System.arraycopy(samplesR0, 0, outBufR0, 0, outBufR0.size) - System.arraycopy(samplesL1, 0, outBufL1, 0, outBufL1.size) - System.arraycopy(samplesR1, 0, outBufR1, 0, outBufR1.size) - - // by this time, the output buffer is filled with processed results, pause the execution - if (!isMaster) { - suspendCoroutine { - processContinuation = it - } - } - else { - adev!!.writeSamples(interleave(samplesL1, samplesR1), 0, samplesL1.size * 2) - withContext(Dispatchers.IO) { - Thread.sleep(12) - } - - getSidechains().forEach { - it?.processContinuation?.resume(Unit) - } - } - } + internal var processor = MixerTrackProcessor(32768, this) + private val processorThread = Thread(processor).also { + it.start() } private fun interleave(f1: FloatArray, f2: FloatArray) = FloatArray(f1.size + f2.size) {