movie can now have audio; adjustable pcm queue size

assets/disk0/tvdos/bin/encodemov.js

@@ -6,6 +6,7 @@ let FPS = 24
 let WIDTH = 560
 let HEIGHT = 448
 let PATHFUN = (i) => `/welkom/${(''+i).padStart(4,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given
+let AUDIOTRACK = 'welkom/welkom.pcm'
 // to export video to its frames:
 //     ffmpeg -i file.mp4 file/%05d.bmp
 // the input frames must be resized (and cropped) beforehand, using ImageMagick is recommended, like so:
@@ -38,10 +39,10 @@ function appendToOutfilePtr(ptr, len) {
     outfile.pappend(ptr, len)
 }
 
-let packetType = [
+const packetType = [
     4, (IPFMODE - 1)
 ]
-let syncPacket = [255, 255]
+const syncPacket = [255, 255]
 
 // write header to the file
 let headerBytes = [
@@ -57,41 +58,87 @@ let headerBytes = [
 let ipfFun = (IPFMODE == 1) ? graphics.encodeIpf1 : (IPFMODE == 2) ? graphics.encodeIpf2 : 0
 if (!ipfFun) throw Error("Unknown IPF mode "+IPFMODE)
 
+
+
+const AUDIO_SAMPLE_SIZE = 2 * ((30000 / FPS) + 1)|0 // times 2 because stereo
+let audioBytesRead = 0
+const audioFile = (AUDIOTRACK) ? files.open(_G.shell.resolvePathInput(AUDIOTRACK).full) : undefined
+let audioRemaining = (audioFile) ? audioFile.size : 0
+const audioPacket = [1, 16]
+
+
 outfile.bwrite(headerBytes)
 
-for (let f = 1; f <= TOTAL_FRAMES; f++) {
+for (let f = 1; ; f++) {
 
     // insert sync packet
     if (f > 1) appendToOutfile(syncPacket)
 
-
+    // insert video frame
-    let fname = PATHFUN(f)
+    if (f <= TOTAL_FRAMES) {
-    let framefile = files.open(_G.shell.resolvePathInput(fname).full)
+        let fname = PATHFUN(f)
-    let fileLen = framefile.size
+        let framefile = files.open(_G.shell.resolvePathInput(fname).full)
-    framefile.pread(infile, fileLen)
+        let fileLen = framefile.size
+        framefile.pread(infile, fileLen)
 
 
-    let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea)
+        let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea)
 
-    print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
+        print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
 
-//    graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1)
+    //    graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1)
-    ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f)
+        ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f)
 
-    let gzlen = gzip.compFromTo(ipfarea, FBUF_SIZE, gzippedImage)
+        let gzlen = gzip.compFromTo(ipfarea, FBUF_SIZE, gzippedImage)
 
-    let frameSize = [
+        let frameSize = [
-        (gzlen >>> 0) & 255,
+            (gzlen >>> 0) & 255,
-        (gzlen >>> 8) & 255,
+            (gzlen >>> 8) & 255,
-        (gzlen >>> 16) & 255,
+            (gzlen >>> 16) & 255,
-        (gzlen >>> 24) & 255
+            (gzlen >>> 24) & 255
-    ]
+        ]
 
-    appendToOutfile(packetType)
+        appendToOutfile(packetType)
-    appendToOutfile(frameSize)
+        appendToOutfile(frameSize)
-    appendToOutfilePtr(gzippedImage, gzlen)
+        appendToOutfilePtr(gzippedImage, gzlen)
 
-    print(` ${gzlen} bytes\n`)
+        print(` ${gzlen} bytes\n`)
+    }
+    // insert audio track, if any
+    if (audioRemaining > 0) {
+
+        // first frame gets two audio packets
+        for (let repeat = 0; repeat < ((f == 1) ? 2 : 1); repeat++) {
+
+    //        print(`Frame ${f}/${TOTAL_FRAMES} (ADPCM) ->`)
+            print(`Frame ${f}/${TOTAL_FRAMES} (PCMu8) ->`)
+
+            const actualBytesToRead = Math.min(
+                (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2,
+                audioRemaining
+            )
+            audioFile.pread(infile, actualBytesToRead, audioBytesRead)
+
+            let pcmSize = [
+                (actualBytesToRead >>> 0) & 255,
+                (actualBytesToRead >>> 8) & 255,
+                (actualBytesToRead >>> 16) & 255,
+                (actualBytesToRead >>> 24) & 255
+            ]
+
+            appendToOutfile(audioPacket)
+            appendToOutfile(pcmSize)
+            appendToOutfilePtr(infile, actualBytesToRead)
+
+            print(` ${actualBytesToRead} bytes\n`)
+
+            audioBytesRead += actualBytesToRead
+            audioRemaining -= actualBytesToRead
+        }
+    }
+
+    // if there is no video and audio remaining, exit the loop
+    if (f > TOTAL_FRAMES && audioRemaining <= 0) break
 }
 
 sys.free(infile)

assets/disk0/tvdos/bin/playmov.js

@@ -159,8 +159,13 @@ let startTime = sys.nanoTime()
 
 let framesRead = 0
 
+audio.resetParams(0)
+audio.purgeQueue(0)
+audio.setPcmMode(0)
+audio.setMasterVolume(0, 255)
+
 renderLoop:
-while (framesRendered < frameCount && readCount < FILE_LENGTH) {
+while (readCount < FILE_LENGTH) {
     let t1 = sys.nanoTime()
 
     if (akku >= frameTime) {
@@ -209,10 +214,23 @@ while (framesRendered < frameCount && readCount < FILE_LENGTH) {
                 }
                 // audio packets
                 else if (4096 <= packetType && packetType <= 6133) {
-                    TODO(`Audio Packet with type ${packetType} at offset ${readCount - 2}`)
+                    if (4097 == packetType) {
+                        let readLength = readInt()
+                        let samples = readBytes(readLength)
+
+                        audio.putPcmDataByPtr(samples, readLength, 0)
+                        audio.setSampleUploadLength(0, readLength)
+                        audio.startSampleUpload(0)
+                        audio.play(0)
+
+                        sys.free(samples)
+                    }
+                    else {
+                        throw Error(`Audio Packet with type ${packetType} at offset ${readCount - 2}`)
+                    }
                 }
                 else {
-                    TODO(`Unknown Packet with type ${packetType} at offset ${readCount - 2}`)
+                    println(`Unknown Packet with type ${packetType} at offset ${readCount - 2}`)
                 }
             }
         }
@@ -229,6 +247,7 @@ while (framesRendered < frameCount && readCount < FILE_LENGTH) {
 let endTime = sys.nanoTime()
 
 sys.free(ipfbuf)
+audio.stop(0)
 
 let timeTook = (endTime - startTime) / 1000000000.0
 

terranmon.txt

@@ -588,7 +588,7 @@ Length Param
 
 Play Head Flags
     Byte 1
-        - 0b mrqp 0000
+        - 0b mrqp ssss
           m: mode (0 for Tracker, 1 for PCM)
           r: reset parameters; always 0 when read
             resetting will:
@@ -598,6 +598,24 @@ Play Head Flags
           q: purge queues (likely do nothing if not PCM); always 0 when read
           p: play (0 if not -- mute all output)
 
+          ssss: PCM Mode set PCM Queue Size
+            0 - 4 samples
+            1 - 6 samples
+            2 - 8 samples
+            3 - 12 samples
+            4 - 16 samples
+            5 - 24 samples
+            6 - 32 samples
+            7 - 48 samples
+            8 - 64 samples
+            9 - 96 samples
+           10 - 128 samples
+           11 - 192 samples
+           12 - 256 samples
+           13 - 384 samples
+           14 - 512 samples
+           15 - 768 samples
+
           NOTE: changing from PCM mode to Tracker mode or vice versa will also reset the parameters as described above
     Byte 2
         - PCM Mode: Write non-zero value to start uploading; always 0 when read

tsvm_core/src/net/torvald/tsvm/AudioJSR223Delegate.kt

@@ -8,7 +8,7 @@ import net.torvald.tsvm.peripheral.AudioAdapter
 class AudioJSR223Delegate(private val vm: VM) {
 
     private fun getFirstSnd(): AudioAdapter? = vm.findPeribyType(VM.PERITYPE_SOUND)?.peripheral as? AudioAdapter
-    private fun getPlayhead(playhead: Int) = getFirstSnd()?.playheads?.get(playhead)!!
+    private fun getPlayhead(playhead: Int) = getFirstSnd()?.playheads?.get(playhead)
     
     fun setPcmMode(playhead: Int) { getPlayhead(playhead)?.isPcmMode = true }
     fun isPcmMode(playhead: Int) = getPlayhead(playhead)?.isPcmMode == true
@@ -56,6 +56,10 @@ class AudioJSR223Delegate(private val vm: VM) {
     }
     fun getPcmData(index: Int) = getFirstSnd()?.pcmBin?.get(index.toLong())
 
+    fun setPcmQueueSizeIndex(playhead: Int, index: Int) { getPlayhead(playhead)?.pcmQueueSizeIndex = index }
+    fun getPcmQueueSizeIndex(playhead: Int, index: Int) { getPlayhead(playhead)?.pcmQueueSizeIndex }
+    fun getPcmQueueSize(playhead: Int, index: Int) { getPlayhead(playhead)?.getPcmQueueSize() }
+
     fun resetParams(playhead: Int) {
         getPlayhead(playhead)?.resetParams()
     }

tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt

@@ -12,6 +12,9 @@ import net.torvald.tsvm.VM
 private fun Boolean.toInt() = if (this) 1 else 0
 
 private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable {
+    private fun printdbg(msg: Any) {
+        if (AudioAdapter.DBGPRN) println("[AudioAdapter] $msg")
+    }
     @Volatile private var exit = false
     override fun run() {
         while (!Thread.interrupted()) {
@@ -21,7 +24,7 @@ private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable {
 
                 if (playhead.isPlaying && writeQueue.notEmpty()) {
 
-//                    printdbg("Taking samples from queue (queue size: ${writeQueue.size})")
+                    printdbg("Taking samples from queue (queue size: ${writeQueue.size})")
 
                     val samples = writeQueue.removeFirst()
                     playhead.position = writeQueue.size
@@ -51,13 +54,16 @@ private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable {
 }
 
 private class WriteQueueingRunnable(val playhead: AudioAdapter.Playhead, val pcmBin: UnsafePtr) : Runnable {
+    private fun printdbg(msg: Any) {
+        if (AudioAdapter.DBGPRN) println("[AudioAdapter] $msg")
+    }
     @Volatile private var exit = false
     override fun run() {
         while (!Thread.interrupted()) {
 
             playhead.let {
-                if (it.pcmQueue.size < 4 && it.pcmUpload && it.pcmUploadLength > 0) {
+                if (it.pcmQueue.size < it.getPcmQueueSize() && it.pcmUpload && it.pcmUploadLength > 0) {
-//                    printdbg("Downloading samples ${it.pcmUploadLength}")
+                    printdbg("Downloading samples ${it.pcmUploadLength}")
 
                     val samples = ByteArray(it.pcmUploadLength)
                     UnsafeHelper.memcpyRaw(null, pcmBin.ptr, samples, UnsafeHelper.getArrayOffset(samples), it.pcmUploadLength.toLong())
@@ -83,12 +89,12 @@ private class WriteQueueingRunnable(val playhead: AudioAdapter.Playhead, val pcm
  */
 class AudioAdapter(val vm: VM) : PeriBase {
 
-    private val DBGPRN = true
     private fun printdbg(msg: Any) {
         if (DBGPRN) println("[AudioAdapter] $msg")
     }
 
     companion object {
+        internal val DBGPRN = true
         const val SAMPLING_RATE = 30000
     }
 
@@ -299,7 +305,8 @@ class AudioAdapter(val vm: VM) : PeriBase {
         var pcmUpload: Boolean = false,
 
         var pcmQueue: Queue<ByteArray> = Queue<ByteArray>(),
-        val audioDevice: OpenALBufferedAudioDevice
+        var pcmQueueSizeIndex: Int = 0,
+        val audioDevice: OpenALBufferedAudioDevice,
     ) {
         fun read(index: Int): Byte = when (index) {
             0 -> position.toByte()
@@ -308,7 +315,7 @@ class AudioAdapter(val vm: VM) : PeriBase {
             3 -> pcmUploadLength.ushr(8).toByte()
             4 -> masterVolume.toByte()
             5 -> masterPan.toByte()
-            6 -> (isPcmMode.toInt().shl(7) or isPlaying.toInt().shl(4)).toByte()
+            6 -> (isPcmMode.toInt().shl(7) or isPlaying.toInt().shl(4) or pcmQueueSizeIndex.and(15)).toByte()
             7 -> 0
             8 -> (bpm - 24).toByte()
             9 -> tickRate.toByte()
@@ -331,6 +338,7 @@ class AudioAdapter(val vm: VM) : PeriBase {
                     val oldPcmMode = isPcmMode
                     isPcmMode = (it and 0b10000000) != 0
                     isPlaying = (it and 0b00010000) != 0
+                    pcmQueueSizeIndex = (it and 0b00001111)
                     
                     if (it and 0b01000000 != 0 || oldPcmMode != isPcmMode) resetParams()
                     if (it and 0b00100000 != 0) purgeQueue()
@@ -353,6 +361,7 @@ class AudioAdapter(val vm: VM) : PeriBase {
             position = 0
             pcmUploadLength = 0
             isPlaying = false
+            pcmQueueSizeIndex = 0
         }
 
         fun purgeQueue() {
@@ -363,9 +372,15 @@ class AudioAdapter(val vm: VM) : PeriBase {
             }
         }
         
+        fun getPcmQueueSize() = QUEUE_SIZE[pcmQueueSizeIndex]
+
         fun dispose() {
             audioDevice.dispose()
         }
+        
+        companion object {
+            val QUEUE_SIZE = intArrayOf(4,6,8,12,16,24,32,48,64,96,128,256,384,512,768)
+        }
     }
 
     internal data class TaudPlayData(