From c1031545ec65069124748e9baa9b91733fa608f8 Mon Sep 17 00:00:00 2001 From: minjaesong Date: Mon, 23 Jan 2023 03:20:35 +0900 Subject: [PATCH] video and audio stuffs; playmov now uses hardware queue --- assets/disk0/tvdos/bin/encodemov.js | 18 +++++----- assets/disk0/tvdos/bin/playmov.js | 36 ++----------------- assets/disk0/tvdos/bin/playpcm.js | 2 +- assets/disk0/tvdos/bin/playwav.js | 11 +++--- terranmon.txt | 3 +- .../torvald/tsvm/peripheral/AudioAdapter.kt | 4 +-- .../src/net/torvald/tsvm/AudioMenu.kt | 2 +- 7 files changed, 23 insertions(+), 53 deletions(-) diff --git a/assets/disk0/tvdos/bin/encodemov.js b/assets/disk0/tvdos/bin/encodemov.js index 2ed4a21..c0ebd8c 100644 --- a/assets/disk0/tvdos/bin/encodemov.js +++ b/assets/disk0/tvdos/bin/encodemov.js @@ -1,12 +1,12 @@ // some manual configurations // let IPFMODE = 2 // 1 or 2 -let TOTAL_FRAMES = 1318 +let TOTAL_FRAMES = 3813 let FPS = 30 let WIDTH = 560 let HEIGHT = 448 -let PATHFUN = (i) => `/namu2/${(''+i).padStart(5,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given -let AUDIOTRACK = 'namu.mp2' +let PATHFUN = (i) => `/ddol2/${(''+i).padStart(5,'0')}.bmp` // how can be the image file found, if a frame number (starts from 1) were given +let AUDIOTRACK = 'ddol.mp2' let AUDIOFORMAT = 'MP2fr' // PCMu8 or MP2fr // to export video to its frames: // ffmpeg -i file.mp4 file/%05d.bmp @@ -16,6 +16,7 @@ let AUDIOFORMAT = 'MP2fr' // PCMu8 or MP2fr // end of manual configuration let MP2_RATE_INDEX; let MP2_PACKETSIZE; +const DECODE_TIME_FACTOR = 1.000 let outfilename = exec_args[1] if (!outfilename) { @@ -52,7 +53,7 @@ const videoPacketType = [4, (IPFMODE - 1)] const syncPacket = [255, 255] const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo const AUDIO_BLOCK_SIZE = ("MP2fr" == AUDIOFORMAT) ? 0x240 : 0 -const AUDIO_QUEUE_SIZE = ("MP2fr" == AUDIOFORMAT) ? Math.ceil(AUDIO_SAMPLE_SIZE / 2304) + 1 : 0 +const AUDIO_QUEUE_SIZE = ("MP2fr" == AUDIOFORMAT) ? Math.ceil(AUDIO_SAMPLE_SIZE / (2304 * DECODE_TIME_FACTOR)) + 1 : 0 // write header to the file let headerBytes = [ 0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56, // magic @@ -83,8 +84,8 @@ function getRepeatCount(fnum) { return (fnum == 1) ? 2 : 1 } else if ("MP2fr" == AUDIOFORMAT) { - let r = Math.ceil((AUDIO_SAMPLE_SIZE - audioSamplesWrote) / AUDIO_SAMPLE_SIZE) + ((fnum == 1) ? 1 : 0) - return (fnum > TOTAL_FRAMES) ? Math.ceil(audioRemaining / MP2_PACKETSIZE) : r + let r = Math.ceil((AUDIO_SAMPLE_SIZE - audioSamplesWrote) / AUDIO_SAMPLE_SIZE) * ((fnum == 1) ? 2 : 1) + return (fnum == 2) ? 1 : (fnum > TOTAL_FRAMES) ? Math.ceil(audioRemaining / MP2_PACKETSIZE) : r } } @@ -117,7 +118,8 @@ for (let f = 1; ; f++) { if (audioRemaining > 0) { // first frame gets two audio packets - for (let repeat = 0; repeat < getRepeatCount(f); repeat++) { + let rrrr = getRepeatCount(f) // must be called only once + for (let q = 0; q < rrrr; q++) { print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`) serial.print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`) @@ -140,7 +142,7 @@ for (let f = 1; ; f++) { actualBytesToRead = Math.min(MP2_PACKETSIZE, audioRemaining) audioFile.pread(infile, actualBytesToRead, audioBytesRead) - audioSamplesWrote += 2304 + if (f > 1) audioSamplesWrote += 2304 / DECODE_TIME_FACTOR // a little hack to ensure first 2 or so frames get more MP2 frames than they should } else throw Error("Unknown audio format: " + AUDIOFORMAT) diff --git a/assets/disk0/tvdos/bin/playmov.js b/assets/disk0/tvdos/bin/playmov.js index 1575024..e8ba0ff 100644 --- a/assets/disk0/tvdos/bin/playmov.js +++ b/assets/disk0/tvdos/bin/playmov.js @@ -47,7 +47,7 @@ const FRAME_TIME = 1.0 / fps const FRAME_COUNT = seqread.readInt() % 16777216 const globalType = seqread.readShort() const audioQueueInfo = seqread.readShort() -let AUDIO_QUEUE_LENGTH = (audioQueueInfo >> 12) + 1 +const AUDIO_QUEUE_LENGTH = (audioQueueInfo >> 12) + 1 const AUDIO_QUEUE_BYTES = (audioQueueInfo & 0xFFF) << 2 sys.free(seqread.readBytes(10)) // skip 12 bytes let audioQueuePos = 0 @@ -71,12 +71,6 @@ graphics.setGraphicsMode(4) let startTime = sys.nanoTime() let framesRead = 0 let audioFired = false -let audioQueue = (AUDIO_QUEUE_LENGTH < 1) ? undefined : new Int32Array(AUDIO_QUEUE_LENGTH) -if (AUDIO_QUEUE_BYTES > 0 && AUDIO_QUEUE_LENGTH > 1) { - for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) { - audioQueue[i] = sys.malloc(AUDIO_QUEUE_BYTES) - } -} audio.resetParams(0) audio.purgeQueue(0) @@ -231,11 +225,6 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { // MP2 if (packetType >>> 8 == 17) { - if (audioQueue[audioQueuePos] === undefined) { -// throw Error(`Audio queue overflow: attempt to write to index ${audioQueuePos}; queue size: ${audioQueue.length}; frame: ${framesRead}`) - AUDIO_QUEUE_LENGTH += 1 - audioQueue.push(sys.malloc(AUDIO_QUEUE_BYTES)) - } if (!mp2Initialised) { mp2Initialised = true audio.mp2Init() @@ -243,7 +232,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { seqread.readBytes(readLength, SND_BASE_ADDR - 2368) audio.mp2Decode() - sys.memcpy(SND_BASE_ADDR - 64, audioQueue[audioQueuePos++], 2304) + audio.mp2UploadDecoded(0) } // RAW PCM packets (decode on the fly) else if (packetType == 0x1000 || packetType == 0x1001) { @@ -262,23 +251,6 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { } - // manage audio playback - if (audioFired && audioQueue) { - if (audio.getPosition(0) < 1 && audioQueuePos > 0) { - // push audio sample - audio.putPcmDataByPtr(audioQueue[0], AUDIO_QUEUE_BYTES, 0) - audio.setSampleUploadLength(0, AUDIO_QUEUE_BYTES) - audio.startSampleUpload(0) - - // unshift the queue - const tmp = audioQueue[0] - for (let i = 1; i < AUDIO_QUEUE_LENGTH; i++) audioQueue[i - 1] = audioQueue[i] - audioQueue[AUDIO_QUEUE_LENGTH - 1] = tmp - - audioQueuePos -= 1 - sys.spin() - } - } } } else { @@ -314,9 +286,7 @@ finally { sys.free(ipfbuf) if (AUDIO_QUEUE_BYTES > 0 && AUDIO_QUEUE_LENGTH > 1) { - for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) { - sys.free(audioQueue[i]) - } + } //audio.stop(0) diff --git a/assets/disk0/tvdos/bin/playpcm.js b/assets/disk0/tvdos/bin/playpcm.js index 3bdabfc..73fe682 100644 --- a/assets/disk0/tvdos/bin/playpcm.js +++ b/assets/disk0/tvdos/bin/playpcm.js @@ -45,7 +45,7 @@ seqread.prepare(filename) let BLOCK_SIZE = 4096 let INFILE_BLOCK_SIZE = BLOCK_SIZE -const QUEUE_MAX = 4 // according to the spec +const QUEUE_MAX = 8 // according to the spec let nChannels = 2 let samplingRate = pcm.HW_SAMPLING_RATE; diff --git a/assets/disk0/tvdos/bin/playwav.js b/assets/disk0/tvdos/bin/playwav.js index 6cc59bc..b934c94 100644 --- a/assets/disk0/tvdos/bin/playwav.js +++ b/assets/disk0/tvdos/bin/playwav.js @@ -53,7 +53,7 @@ if (seqread.readFourCC() != "WAVE") { let BLOCK_SIZE = 0 let INFILE_BLOCK_SIZE = 0 -const QUEUE_MAX = 4 // according to the spec +const QUEUE_MAX = 8 // according to the spec let pcmType; let nChannels; @@ -217,14 +217,14 @@ while (!stopPlay && seqread.getReadCount() < FILE_SIZE - 8) { } } + printPlayBar(startOffset) let queueSize = audio.getPosition(0) if (queueSize <= 1) { - printPlayBar(startOffset) // upload four samples for lag-safely - for (let repeat = QUEUE_MAX - queueSize; repeat > 0; repeat--) { + for (let repeat = 0; repeat < QUEUE_MAX; repeat++) { let remainingBytes = FILE_SIZE - 8 - seqread.getReadCount() readLength = (remainingBytes < INFILE_BLOCK_SIZE) ? remainingBytes : INFILE_BLOCK_SIZE @@ -244,10 +244,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_SIZE - 8) { audio.setSampleUploadLength(0, decodedSampleLength) audio.startSampleUpload(0) - - if (repeat > 1) sys.sleep(10) - - printPlayBar(startOffset) + sys.spin() } audio.play(0) diff --git a/terranmon.txt b/terranmon.txt index 4fdf665..7a11fa4 100644 --- a/terranmon.txt +++ b/terranmon.txt @@ -643,6 +643,7 @@ Play Head Flags resetting will: set position to 0, set length param to 0, + set queue capacity to 8 samples, unset play bit q: purge queues (likely do nothing if not PCM); always 0 when read p: play (0 if not -- mute all output) @@ -650,7 +651,7 @@ Play Head Flags ssss: PCM Mode set PCM Queue Size 0 - 4 samples 1 - 6 samples - 2 - 8 samples + 2 - 8 samples (the default size) 3 - 12 samples 4 - 16 samples 5 - 24 samples diff --git a/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt b/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt index 3b75680..d1d17d5 100644 --- a/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt +++ b/tsvm_core/src/net/torvald/tsvm/peripheral/AudioAdapter.kt @@ -412,7 +412,7 @@ class AudioAdapter(val vm: VM) : PeriBase(VM.PERITYPE_SOUND) { position = 0 pcmUploadLength = 0 isPlaying = false - pcmQueueSizeIndex = 0 + pcmQueueSizeIndex = 2 } fun purgeQueue() { @@ -431,7 +431,7 @@ class AudioAdapter(val vm: VM) : PeriBase(VM.PERITYPE_SOUND) { } companion object { - val QUEUE_SIZE = intArrayOf(4,6,8,12,16,24,32,48,64,96,128,256,384,512,768) + val QUEUE_SIZE = intArrayOf(4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768) } } diff --git a/tsvm_executable/src/net/torvald/tsvm/AudioMenu.kt b/tsvm_executable/src/net/torvald/tsvm/AudioMenu.kt index 9c61e3f..97cf662 100644 --- a/tsvm_executable/src/net/torvald/tsvm/AudioMenu.kt +++ b/tsvm_executable/src/net/torvald/tsvm/AudioMenu.kt @@ -81,7 +81,7 @@ class AudioMenu(parent: VMEmuExecutable, x: Int, y: Int, w: Int, h: Int) : EmuMe // Queue sparkline batch.color = COL_SOUNDSCOPE_BACK batch.fillRect(x + 5*FONT.W + 2, y + 2*FONT.H, FONT.W * 7, FONT.H) - val qgrsize = ahead.getPcmQueueCapacity().let { ahead.position.coerceAtMost(it) / it.toDouble() }.times(FONT.W * 7).roundToInt() + val qgrsize = ahead.getPcmQueueCapacity().let { ahead.position / it.toDouble() }.times(FONT.W * 7).roundToInt() batch.color = COL_HIGHLIGHT2 batch.fillRect(x + 5*FONT.W + 2, y + 2*FONT.H + 1, qgrsize, FONT.H - 2)