input audio resampling

src/net/torvald/terrarum/audio/AudioProcessBuf.kt

@@ -1,64 +1,127 @@
 package net.torvald.terrarum.audio
 
+import com.jme3.math.FastMath
+import net.torvald.terrarum.audio.TerrarumAudioMixerTrack.Companion.BUFFER_SIZE
+import net.torvald.terrarum.audio.TerrarumAudioMixerTrack.Companion.SAMPLING_RATE
+import net.torvald.terrarum.ceilToInt
+import net.torvald.terrarum.floorToInt
 import net.torvald.terrarum.serialise.toUint
+import org.dyn4j.Epsilon
+import kotlin.math.PI
+import kotlin.math.absoluteValue
+import kotlin.math.sin
 
 /**
- * Audio is assumed to be 16 bits
+ * Audio is assumed to be 2 channels, 16 bits
  *
  * Created by minjaesong on 2023-11-17.
  */
-class AudioProcessBuf(val size: Int) {
+class AudioProcessBuf(inputSamplingRate: Int, val audioReadFun: (ByteArray) -> Int?, val onAudioFinished: () -> Unit) {
 
-    var buf0 = ByteArray(size); private set
+    private val doResample = inputSamplingRate != SAMPLING_RATE
-    var buf1 = ByteArray(size); private set
 
-    var fbuf0 = FloatArray(size / 2); private set
+    companion object {
-    var fbuf1 = FloatArray(size / 2); private set
+        private val epsilon: Double = Epsilon.E
 
-    private fun shift(): ByteArray {
+        private val TAPS = 4 // 2*a tap lanczos intp. Lower = greater artefacts
-        buf0 = buf1
+
-        buf1 = ByteArray(size)
+        fun L(x: Double): Double = if (x.absoluteValue < epsilon)
-        return buf1
+            1.0
+        else if (-TAPS <= x && x < TAPS)
+            (TAPS * sin(PI * x) * sin(PI * x / TAPS)) / (PI * PI * x * x)
+        else
+            0.0
+
+        private val BS = BUFFER_SIZE / 4
     }
 
-    private fun updateFloats() {
+    private val gcd = FastMath.getGCD(inputSamplingRate, SAMPLING_RATE) // 300 for 44100, 48000
-        fbuf0 = fbuf1
+
-        fbuf1 = FloatArray(size / 2) {
+    private val samplesIn = inputSamplingRate / gcd // 147 for 44100
-            val i16 = (buf1[2*it].toUint() or buf1[2*it+1].toUint().shl(8)).toShort()
+    private val samplesOut = SAMPLING_RATE / gcd // 160 for 48000
-            i16 / 32767f
+
+    private val internalBufferSize = samplesOut * ((BS.toFloat()) / samplesOut).ceilToInt() // (512 / 160) -> 640 for 44100, 48000
+
+
+    private fun resampleBlock(inn: FloatArray, out: FloatArray) {
+        fun getInn(i: Int) = if (i in inn.indices) inn[i] else 0f
+
+        for (sampleIdx in out.indices) {
+            val x = (inn.size.toDouble() / out.size) * sampleIdx
+            var sx = 0.0
+            for (i in x.floorToInt() - TAPS + 1..x.floorToInt() + TAPS) {
+                sx += getInn(i) * L(x - i)
+            }
+            out[sampleIdx] = sx.toFloat()
         }
     }
 
-    fun fetchBytes(action: (ByteArray) -> Unit) {
+    var validSamplesInBuf = 0
-        action(shift())
+
-        updateFloats()
+    val foutL = FloatArray(internalBufferSize) // 640 for (44100, 48000), 512 for (48000, 48000) with BUFFER_SIZE = 512 * 4
+    val foutR = FloatArray(internalBufferSize) // 640 for (44100, 48000), 512 for (48000, 48000) with BUFFER_SIZE = 512 * 4
+
+    fun fetchBytes() {
+        val readCount = ((internalBufferSize - validSamplesInBuf) / samplesOut) * samplesIn // in samples (441 or 588 for 44100, 48000)
+        val writeCount = ((internalBufferSize - validSamplesInBuf) / samplesOut) * samplesOut // in samples (480 or 640 for 44100, 48000)
+        val readBuf = ByteArray(readCount * 4)
+        val finL = FloatArray(readCount)
+        val finR = FloatArray(readCount)
+        val foutL = FloatArray(writeCount)
+        val foutR = FloatArray(writeCount)
+
+        fun getFromReadBuf(i: Int, bytesRead: Int) = if (i < bytesRead) readBuf[i].toUint() else 0
+
+        try {
+            val bytesRead = audioReadFun(readBuf)
+
+            if (bytesRead == null || bytesRead <= 0) onAudioFinished()
+            else {
+                for(c in 0 until readCount) {
+                    val sl = (getFromReadBuf(4 * c + 0, bytesRead) or getFromReadBuf(4 * c + 1, bytesRead).shl(8)).toShort()
+                    val sr = (getFromReadBuf(4 * c + 2, bytesRead) or getFromReadBuf(4 * c + 3, bytesRead).shl(8)).toShort()
+
+                    val fl = sl / 32767f
+                    val fr = sr / 32767f
+
+                    finL[c] = fl
+                    finR[c] = fr
+                }
+            }
+        }
+        catch (e: Throwable) {
+            e.printStackTrace()
+        }
+        finally {
+            if (doResample) {
+                // perform resampling
+                resampleBlock(finL, foutL)
+                resampleBlock(finR, foutR)
+
+                // fill in the output buffers
+                System.arraycopy(foutL, 0, this.foutL, validSamplesInBuf, writeCount)
+                System.arraycopy(foutR, 0, this.foutR, validSamplesInBuf, writeCount)
+            }
+            else {
+                // fill in the output buffers
+                System.arraycopy(finL, 0, this.foutL, validSamplesInBuf, writeCount)
+                System.arraycopy(finR, 0, this.foutR, validSamplesInBuf, writeCount)
+            }
+
+            validSamplesInBuf += writeCount
+        }
     }
 
-    // reusing a buffer causes tons of blips in the sound? how??
+    fun getLR(volume: Double): Pair<FloatArray, FloatArray> {
-    /*private val L0buf = FloatArray(size / 4)
+        // copy into the out
-    private val R0buf = FloatArray(size / 4)
+        val outL = FloatArray(BS) { (foutL[it] * volume).toFloat() }
-    private val L1buf = FloatArray(size / 4)
+        val outR = FloatArray(BS) { (foutR[it] * volume).toFloat() }
-    private val R1buf = FloatArray(size / 4)
+        // shift bytes in the fout
+        System.arraycopy(foutL, BS, foutL, 0, validSamplesInBuf - BS)
+        System.arraycopy(foutR, BS, foutR, 0, validSamplesInBuf - BS)
+        // decrement necessary variables
+        validSamplesInBuf -= BS
 
-    fun getL0(volume: Double): FloatArray {
+        return outL to outR
-        for (i in L0buf.indices) { L0buf[i] = (volume * fbuf0[2*i]).toFloat() }
-        return L0buf
     }
-    fun getR0(volume: Double): FloatArray {
-        for (i in R0buf.indices) { R0buf[i] = (volume * fbuf0[2*i+1]).toFloat() }
-        return R0buf
-    }
-    fun getL1(volume: Double): FloatArray {
-        for (i in L1buf.indices) { L1buf[i] = (volume * fbuf1[2*i]).toFloat() }
-        return L1buf
-    }
-    fun getR1(volume: Double): FloatArray {
-        for (i in R1buf.indices) { R1buf[i] = (volume * fbuf1[2*i+1]).toFloat() }
-        return R1buf
-    }*/
-    fun getL0(volume: Double) = FloatArray(size / 4) { (volume * fbuf0[2*it]).toFloat() }
-    fun getR0(volume: Double) = FloatArray(size / 4) { (volume * fbuf0[2*it+1]).toFloat() }
-    fun getL1(volume: Double) = FloatArray(size / 4) { (volume * fbuf1[2*it]).toFloat() }
-    fun getR1(volume: Double) = FloatArray(size / 4) { (volume * fbuf1[2*it+1]).toFloat() }
-
 }

src/net/torvald/terrarum/audio/MixerTrackProcessor.kt

@@ -27,8 +27,7 @@ class MixerTrackProcessor(val bufferSize: Int, val rate: Int, val track: Terraru
     private val emptyBuf = FloatArray(bufferSize / 4)
 
 
-    internal val streamBuf = AudioProcessBuf(bufferSize)
+    internal var streamBuf: AudioProcessBuf? = null
-    internal val sideChainBufs = Array(track.sidechainInputs.size) { AudioProcessBuf(bufferSize) }
 
     private var fout1 = listOf(emptyBuf, emptyBuf)
 
@@ -43,6 +42,16 @@ class MixerTrackProcessor(val bufferSize: Int, val rate: Int, val track: Terraru
     private fun printdbg(msg: Any) {
         if (true) App.printdbg("AudioAdapter ${track.name}", msg)
     }
+
+    private fun allocateStreamBuf(track: TerrarumAudioMixerTrack) {
+        streamBuf = AudioProcessBuf(track.currentTrack!!.samplingRate, {
+            track.currentTrack?.gdxMusic?.forceInvoke<Int>("read", arrayOf(it))
+        }, {
+            track.stop()
+            this.streamBuf = null
+        })
+    }
+
     override fun run() {
 //        while (running) { // uncomment to multithread
             /*synchronized(pauseLock) { // uncomment to multithread
@@ -91,20 +100,8 @@ class MixerTrackProcessor(val bufferSize: Int, val rate: Int, val track: Terraru
 
             // fetch deviceBufferSize amount of sample from the disk
             if (track.trackType != TrackType.MASTER && track.trackType != TrackType.BUS && track.streamPlaying) {
-                streamBuf.fetchBytes {
+                if (streamBuf == null && track.currentTrack != null) allocateStreamBuf(track)
-                    try {
+                streamBuf!!.fetchBytes()
-                        val bytesRead = track.currentTrack?.gdxMusic?.forceInvoke<Int>("read", arrayOf(it))
-                        if (bytesRead == null || bytesRead <= 0) { // some class (namely Mp3) may return 0 instead of negative value
-//                        printdbg("Finished reading audio stream")
-                            track.stop()
-                        }
-                    }
-                    catch (e: Throwable) {
-                        e.printStackTrace()
-                        it.fill(0)
-                    }
-
-                }
             }
 
             var samplesL1: FloatArray
@@ -129,15 +126,17 @@ class MixerTrackProcessor(val bufferSize: Int, val rate: Int, val track: Terraru
             }
             // source channel: skip processing if there's no active input
 //            else if (track.getSidechains().any { it != null && !it.isBus && !it.isMaster && !it.streamPlaying } && !track.streamPlaying) {
-            else if (!track.streamPlaying) {
+            else if (!track.streamPlaying || streamBuf == null) {
                 samplesL1 = emptyBuf
                 samplesR1 = emptyBuf
 
                 bufEmpty = true
             }
             else {
-                samplesL1 = streamBuf.getL1(track.volume)
+                streamBuf!!.getLR(track.volume).let {
-                samplesR1 = streamBuf.getR1(track.volume)
+                    samplesL1 = it.first
+                    samplesR1 = it.second
+                }
             }
 
             if (!bufEmpty) {

src/net/torvald/terrarum/modulebasegame/TerrarumMusicGovernor.kt

@@ -19,6 +19,8 @@ data class MusicContainer(
         gdxMusic.setOnCompletionListener(songFinishedHook)
     }
 
+    val samplingRate: Int = 44100 // TODO
+
     override fun toString() = if (name.isEmpty()) file.nameWithoutExtension else name
 }