From f27caded9bc026e091f0228b97f3c70dbe95f07d Mon Sep 17 00:00:00 2001 From: minjaesong Date: Tue, 3 Jan 2023 17:37:52 +0900 Subject: [PATCH] movie can now have audio; adjustable pcm queue size --- assets/disk0/tvdos/bin/encodemov.js | 93 ++++++++++++++----- assets/disk0/tvdos/bin/playmov.js | 25 ++++- terranmon.txt | 20 +++- .../net/torvald/tsvm/AudioJSR223Delegate.kt | 6 +- .../torvald/tsvm/peripheral/AudioAdapter.kt | 31 +++++-- 5 files changed, 139 insertions(+), 36 deletions(-) diff --git a/assets/disk0/tvdos/bin/encodemov.js b/assets/disk0/tvdos/bin/encodemov.js index 4f12200..1bab813 100644 --- a/assets/disk0/tvdos/bin/encodemov.js +++ b/assets/disk0/tvdos/bin/encodemov.js @@ -6,6 +6,7 @@ let FPS = 24 let WIDTH = 560 let HEIGHT = 448 let PATHFUN = (i) => `/welkom/${(''+i).padStart(4,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given +let AUDIOTRACK = 'welkom/welkom.pcm' // to export video to its frames: // ffmpeg -i file.mp4 file/%05d.bmp // the input frames must be resized (and cropped) beforehand, using ImageMagick is recommended, like so: @@ -38,10 +39,10 @@ function appendToOutfilePtr(ptr, len) { outfile.pappend(ptr, len) } -let packetType = [ +const packetType = [ 4, (IPFMODE - 1) ] -let syncPacket = [255, 255] +const syncPacket = [255, 255] // write header to the file let headerBytes = [ @@ -57,41 +58,87 @@ let headerBytes = [ let ipfFun = (IPFMODE == 1) ? graphics.encodeIpf1 : (IPFMODE == 2) ? graphics.encodeIpf2 : 0 if (!ipfFun) throw Error("Unknown IPF mode "+IPFMODE) + + +const AUDIO_SAMPLE_SIZE = 2 * ((30000 / FPS) + 1)|0 // times 2 because stereo +let audioBytesRead = 0 +const audioFile = (AUDIOTRACK) ? files.open(_G.shell.resolvePathInput(AUDIOTRACK).full) : undefined +let audioRemaining = (audioFile) ? audioFile.size : 0 +const audioPacket = [1, 16] + + outfile.bwrite(headerBytes) -for (let f = 1; f <= TOTAL_FRAMES; f++) { +for (let f = 1; ; f++) { // insert sync packet if (f > 1) appendToOutfile(syncPacket) - - let fname = PATHFUN(f) - let framefile = files.open(_G.shell.resolvePathInput(fname).full) - let fileLen = framefile.size - framefile.pread(infile, fileLen) + // insert video frame + if (f <= TOTAL_FRAMES) { + let fname = PATHFUN(f) + let framefile = files.open(_G.shell.resolvePathInput(fname).full) + let fileLen = framefile.size + framefile.pread(infile, fileLen) - let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea) + let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea) - print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`) + print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`) -// graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1) - ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f) + // graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1) + ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f) - let gzlen = gzip.compFromTo(ipfarea, FBUF_SIZE, gzippedImage) + let gzlen = gzip.compFromTo(ipfarea, FBUF_SIZE, gzippedImage) - let frameSize = [ - (gzlen >>> 0) & 255, - (gzlen >>> 8) & 255, - (gzlen >>> 16) & 255, - (gzlen >>> 24) & 255 - ] + let frameSize = [ + (gzlen >>> 0) & 255, + (gzlen >>> 8) & 255, + (gzlen >>> 16) & 255, + (gzlen >>> 24) & 255 + ] - appendToOutfile(packetType) - appendToOutfile(frameSize) - appendToOutfilePtr(gzippedImage, gzlen) + appendToOutfile(packetType) + appendToOutfile(frameSize) + appendToOutfilePtr(gzippedImage, gzlen) - print(` ${gzlen} bytes\n`) + print(` ${gzlen} bytes\n`) + } + // insert audio track, if any + if (audioRemaining > 0) { + + // first frame gets two audio packets + for (let repeat = 0; repeat < ((f == 1) ? 2 : 1); repeat++) { + + // print(`Frame ${f}/${TOTAL_FRAMES} (ADPCM) ->`) + print(`Frame ${f}/${TOTAL_FRAMES} (PCMu8) ->`) + + const actualBytesToRead = Math.min( + (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2, + audioRemaining + ) + audioFile.pread(infile, actualBytesToRead, audioBytesRead) + + let pcmSize = [ + (actualBytesToRead >>> 0) & 255, + (actualBytesToRead >>> 8) & 255, + (actualBytesToRead >>> 16) & 255, + (actualBytesToRead >>> 24) & 255 + ] + + appendToOutfile(audioPacket) + appendToOutfile(pcmSize) + appendToOutfilePtr(infile, actualBytesToRead) + + print(` ${actualBytesToRead} bytes\n`) + + audioBytesRead += actualBytesToRead + audioRemaining -= actualBytesToRead + } + } + + // if there is no video and audio remaining, exit the loop + if (f > TOTAL_FRAMES && audioRemaining <= 0) break } sys.free(infile) diff --git a/assets/disk0/tvdos/bin/playmov.js b/assets/disk0/tvdos/bin/playmov.js index 920e119..2c9e6cd 100644 --- a/assets/disk0/tvdos/bin/playmov.js +++ b/assets/disk0/tvdos/bin/playmov.js @@ -159,8 +159,13 @@ let startTime = sys.nanoTime() let framesRead = 0 +audio.resetParams(0) +audio.purgeQueue(0) +audio.setPcmMode(0) +audio.setMasterVolume(0, 255) + renderLoop: -while (framesRendered < frameCount && readCount < FILE_LENGTH) { +while (readCount < FILE_LENGTH) { let t1 = sys.nanoTime() if (akku >= frameTime) { @@ -209,10 +214,23 @@ while (framesRendered < frameCount && readCount < FILE_LENGTH) { } // audio packets else if (4096 <= packetType && packetType <= 6133) { - TODO(`Audio Packet with type ${packetType} at offset ${readCount - 2}`) + if (4097 == packetType) { + let readLength = readInt() + let samples = readBytes(readLength) + + audio.putPcmDataByPtr(samples, readLength, 0) + audio.setSampleUploadLength(0, readLength) + audio.startSampleUpload(0) + audio.play(0) + + sys.free(samples) + } + else { + throw Error(`Audio Packet with type ${packetType} at offset ${readCount - 2}`) + } } else { - TODO(`Unknown Packet with type ${packetType} at offset ${readCount - 2}`) + println(`Unknown Packet with type ${packetType} at offset ${readCount - 2}`) } } } @@ -229,6 +247,7 @@ while (framesRendered < frameCount && readCount < FILE_LENGTH) { let endTime = sys.nanoTime() sys.free(ipfbuf) +audio.stop(0) let timeTook = (endTime - startTime) / 1000000000.0 diff --git a/terranmon.txt b/terranmon.txt index b4543a0..abdbd94 100644 --- a/terranmon.txt +++ b/terranmon.txt @@ -588,7 +588,7 @@ Length Param Play Head Flags Byte 1 - - 0b mrqp 0000 + - 0b mrqp ssss m: mode (0 for Tracker, 1 for PCM) r: reset parameters; always 0 when read resetting will: @@ -598,6 +598,24 @@ Play Head Flags q: purge queues (likely do nothing if not PCM); always 0 when read p: play (0 if not -- mute all output) + ssss: PCM Mode set PCM Queue Size + 0 - 4 samples + 1 - 6 samples + 2 - 8 samples + 3 - 12 samples + 4 - 16 samples + 5 - 24 samples + 6 - 32 samples + 7 - 48 samples + 8 - 64 samples + 9 - 96 samples + 10 - 128 samples + 11 - 192 samples + 12 - 256 samples + 13 - 384 samples + 14 - 512 samples + 15 - 768 samples + NOTE: changing from PCM mode to Tracker mode or vice versa will also reset the parameters as described above Byte 2 - PCM Mode: Write non-zero value to start uploading; always 0 when read diff --git a/tsvm_core/src/net/torvald/tsvm/AudioJSR223Delegate.kt b/tsvm_core/src/net/torvald/tsvm/AudioJSR223Delegate.kt index 6094381..646aa86 100644 --- a/tsvm_core/src/net/torvald/tsvm/AudioJSR223Delegate.kt +++ b/tsvm_core/src/net/torvald/tsvm/AudioJSR223Delegate.kt @@ -8,7 +8,7 @@ import net.torvald.tsvm.peripheral.AudioAdapter class AudioJSR223Delegate(private val vm: VM) { private fun getFirstSnd(): AudioAdapter? = vm.findPeribyType(VM.PERITYPE_SOUND)?.peripheral as? AudioAdapter - private fun getPlayhead(playhead: Int) = getFirstSnd()?.playheads?.get(playhead)!! + private fun getPlayhead(playhead: Int) = getFirstSnd()?.playheads?.get(playhead) fun setPcmMode(playhead: Int) { getPlayhead(playhead)?.isPcmMode = true } fun isPcmMode(playhead: Int) = getPlayhead(playhead)?.isPcmMode == true @@ -56,6 +56,10 @@ class AudioJSR223Delegate(private val vm: VM) { } fun getPcmData(index: Int) = getFirstSnd()?.pcmBin?.get(index.toLong()) + fun setPcmQueueSizeIndex(playhead: Int, index: Int) { getPlayhead(playhead)?.pcmQueueSizeIndex = index } + fun getPcmQueueSizeIndex(playhead: Int, index: Int) { getPlayhead(playhead)?.pcmQueueSizeIndex } + fun getPcmQueueSize(playhead: Int, index: Int) { getPlayhead(playhead)?.getPcmQueueSize() } + fun resetParams(playhead: Int) { getPlayhead(playhead)?.resetParams() } diff --git a/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt b/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt index 85901e6..972edd5 100644 --- a/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt +++ b/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt @@ -12,6 +12,9 @@ import net.torvald.tsvm.VM private fun Boolean.toInt() = if (this) 1 else 0 private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable { + private fun printdbg(msg: Any) { + if (AudioAdapter.DBGPRN) println("[AudioAdapter] $msg") + } @Volatile private var exit = false override fun run() { while (!Thread.interrupted()) { @@ -21,7 +24,7 @@ private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable { if (playhead.isPlaying && writeQueue.notEmpty()) { -// printdbg("Taking samples from queue (queue size: ${writeQueue.size})") + printdbg("Taking samples from queue (queue size: ${writeQueue.size})") val samples = writeQueue.removeFirst() playhead.position = writeQueue.size @@ -51,13 +54,16 @@ private class RenderRunnable(val playhead: AudioAdapter.Playhead) : Runnable { } private class WriteQueueingRunnable(val playhead: AudioAdapter.Playhead, val pcmBin: UnsafePtr) : Runnable { + private fun printdbg(msg: Any) { + if (AudioAdapter.DBGPRN) println("[AudioAdapter] $msg") + } @Volatile private var exit = false override fun run() { while (!Thread.interrupted()) { playhead.let { - if (it.pcmQueue.size < 4 && it.pcmUpload && it.pcmUploadLength > 0) { -// printdbg("Downloading samples ${it.pcmUploadLength}") + if (it.pcmQueue.size < it.getPcmQueueSize() && it.pcmUpload && it.pcmUploadLength > 0) { + printdbg("Downloading samples ${it.pcmUploadLength}") val samples = ByteArray(it.pcmUploadLength) UnsafeHelper.memcpyRaw(null, pcmBin.ptr, samples, UnsafeHelper.getArrayOffset(samples), it.pcmUploadLength.toLong()) @@ -83,12 +89,12 @@ private class WriteQueueingRunnable(val playhead: AudioAdapter.Playhead, val pcm */ class AudioAdapter(val vm: VM) : PeriBase { - private val DBGPRN = true private fun printdbg(msg: Any) { if (DBGPRN) println("[AudioAdapter] $msg") } - + companion object { + internal val DBGPRN = true const val SAMPLING_RATE = 30000 } @@ -299,7 +305,8 @@ class AudioAdapter(val vm: VM) : PeriBase { var pcmUpload: Boolean = false, var pcmQueue: Queue = Queue(), - val audioDevice: OpenALBufferedAudioDevice + var pcmQueueSizeIndex: Int = 0, + val audioDevice: OpenALBufferedAudioDevice, ) { fun read(index: Int): Byte = when (index) { 0 -> position.toByte() @@ -308,7 +315,7 @@ class AudioAdapter(val vm: VM) : PeriBase { 3 -> pcmUploadLength.ushr(8).toByte() 4 -> masterVolume.toByte() 5 -> masterPan.toByte() - 6 -> (isPcmMode.toInt().shl(7) or isPlaying.toInt().shl(4)).toByte() + 6 -> (isPcmMode.toInt().shl(7) or isPlaying.toInt().shl(4) or pcmQueueSizeIndex.and(15)).toByte() 7 -> 0 8 -> (bpm - 24).toByte() 9 -> tickRate.toByte() @@ -331,7 +338,8 @@ class AudioAdapter(val vm: VM) : PeriBase { val oldPcmMode = isPcmMode isPcmMode = (it and 0b10000000) != 0 isPlaying = (it and 0b00010000) != 0 - + pcmQueueSizeIndex = (it and 0b00001111) + if (it and 0b01000000 != 0 || oldPcmMode != isPcmMode) resetParams() if (it and 0b00100000 != 0) purgeQueue() } } @@ -353,6 +361,7 @@ class AudioAdapter(val vm: VM) : PeriBase { position = 0 pcmUploadLength = 0 isPlaying = false + pcmQueueSizeIndex = 0 } fun purgeQueue() { @@ -362,10 +371,16 @@ class AudioAdapter(val vm: VM) : PeriBase { pcmUploadLength = 0 } } + + fun getPcmQueueSize() = QUEUE_SIZE[pcmQueueSizeIndex] fun dispose() { audioDevice.dispose() } + + companion object { + val QUEUE_SIZE = intArrayOf(4,6,8,12,16,24,32,48,64,96,128,256,384,512,768) + } } internal data class TaudPlayData(