From 6d195431397e69b0953457ebe89aebbbb063dabe Mon Sep 17 00:00:00 2001 From: minjaesong Date: Wed, 18 Jan 2023 18:05:34 +0900 Subject: [PATCH] mov: mp2 audio encoding and playing --- assets/disk0/tvdos/bin/encodemov.js | 82 ++++++++++++---- assets/disk0/tvdos/bin/playmov.js | 141 +++++++++++++++++++++------- assets/disk0/tvdos/bin/playmp2.js | 10 +- terranmon.txt | 17 +++- 4 files changed, 188 insertions(+), 62 deletions(-) diff --git a/assets/disk0/tvdos/bin/encodemov.js b/assets/disk0/tvdos/bin/encodemov.js index 6eb894f..cedc97c 100644 --- a/assets/disk0/tvdos/bin/encodemov.js +++ b/assets/disk0/tvdos/bin/encodemov.js @@ -6,7 +6,9 @@ let FPS = 30 let WIDTH = 560 let HEIGHT = 448 let PATHFUN = (i) => `/namu2/${(''+i).padStart(5,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given -let AUDIOTRACK = 'namu.u8' +let AUDIOTRACK = 'namu.mp2' +let AUDIOFORMAT = 'MP2fr' // PCMu8 or MP2fr +let MP2_PACKETSIZE; // to export video to its frames: // ffmpeg -i file.mp4 file/%05d.bmp // the input frames must be resized (and cropped) beforehand, using ImageMagick is recommended, like so: @@ -39,11 +41,17 @@ function appendToOutfilePtr(ptr, len) { outfile.pappend(ptr, len) } -const packetType = [ - 4, (IPFMODE - 1) -] -const syncPacket = [255, 255] +function audioFormatToAudioPacketType() { + return ("PCMu8" == AUDIOFORMAT) ? [1, 16] + : ("MP2fr" == AUDIOFORMAT) ? [1, 17] + : [255, 16] +} +const videoPacketType = [4, (IPFMODE - 1)] +const syncPacket = [255, 255] +const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo +const AUDIO_BLOCK_SIZE = ("MP2fr" == AUDIOFORMAT) ? 0x240 : 0 +const AUDIO_QUEUE_SIZE = ("MP2fr" == AUDIOFORMAT) ? Math.ceil(AUDIO_SAMPLE_SIZE / 2304) + 1 : 0 // write header to the file let headerBytes = [ 0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56, // magic @@ -52,7 +60,8 @@ let headerBytes = [ FPS & 255, (FPS >> 8) & 255, // FPS TOTAL_FRAMES & 255, (TOTAL_FRAMES >> 8) & 255, (TOTAL_FRAMES >> 16) & 255, (TOTAL_FRAMES >> 24) & 255, // frame count 0xFF, 0x00, // new standard deprecates global type - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved + AUDIO_BLOCK_SIZE & 255, (AUDIO_BLOCK_SIZE >>> 8) | (AUDIO_QUEUE_SIZE << 4), + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved ] let ipfFun = (IPFMODE == 1) ? graphics.encodeIpf1 : (IPFMODE == 2) ? graphics.encodeIpf2 : 0 @@ -60,15 +69,25 @@ if (!ipfFun) throw Error("Unknown IPF mode "+IPFMODE) -const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo let audioBytesRead = 0 const audioFile = (AUDIOTRACK) ? files.open(_G.shell.resolvePathInput(AUDIOTRACK).full) : undefined let audioRemaining = (audioFile) ? audioFile.size : 0 -const audioPacket = [1, 16] +const audioPacketType = audioFormatToAudioPacketType() outfile.bwrite(headerBytes) +function getRepeatCount(fnum) { + if ("PCMu8" == AUDIOFORMAT) { + return (fnum == 1) ? 2 : 1 + } + else if ("MP2fr" == AUDIOFORMAT) { + let r = Math.ceil((AUDIO_SAMPLE_SIZE*2 - audioSamplesWrote) / AUDIO_SAMPLE_SIZE) + ((fnum == 1) ? 1 : 0) + return (fnum > TOTAL_FRAMES) ? Math.ceil(audioRemaining / MP2_PACKETSIZE) : r + } +} + +let audioSamplesWrote = 0 for (let f = 1; ; f++) { // insert sync packet @@ -78,29 +97,48 @@ for (let f = 1; ; f++) { if (audioRemaining > 0) { // first frame gets two audio packets - for (let repeat = 0; repeat < ((f == 1) ? 2 : 1); repeat++) { + for (let repeat = 0; repeat < getRepeatCount(f); repeat++) { - // print(`Frame ${f}/${TOTAL_FRAMES} (ADPCM) ->`) - print(`Frame ${f}/${TOTAL_FRAMES} (PCMu8) ->`) + print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`) + serial.print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`) - const actualBytesToRead = Math.min( - (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2, - audioRemaining - ) - audioFile.pread(infile, actualBytesToRead, audioBytesRead) + // read a chunk/mpeg-frame + let actualBytesToRead; + if ("PCMu8" == AUDIOFORMAT) { + actualBytesToRead = Math.min( + (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2, + audioRemaining + ) + audioFile.pread(infile, actualBytesToRead, audioBytesRead) + } + else if ("MP2fr" == AUDIOFORMAT) { + if (!MP2_PACKETSIZE) { + audioFile.pread(infile, 3, 0) + MP2_PACKETSIZE = audio.mp2GetInitialFrameSize([sys.peek(infile),sys.peek(infile+1),sys.peek(infile+2)]) + } - let pcmSize = [ + actualBytesToRead = Math.min(MP2_PACKETSIZE, audioRemaining) + audioFile.pread(infile, actualBytesToRead, audioBytesRead) + audioSamplesWrote += 2304 + } + else throw Error("Unknown audio format: " + AUDIOFORMAT) + + // writeout + let audioSize = [ (actualBytesToRead >>> 0) & 255, (actualBytesToRead >>> 8) & 255, (actualBytesToRead >>> 16) & 255, (actualBytesToRead >>> 24) & 255 ] - appendToOutfile(audioPacket) - appendToOutfile(pcmSize) + appendToOutfile(audioPacketType) + appendToOutfile(audioSize) appendToOutfilePtr(infile, actualBytesToRead) + + print(` ${actualBytesToRead} bytes\n`) + serial.print(` ${actualBytesToRead} bytes\n`) audioBytesRead += actualBytesToRead audioRemaining -= actualBytesToRead @@ -117,6 +155,7 @@ for (let f = 1; ; f++) { let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea) print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`) + serial.print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`) // graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1) ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f) @@ -130,11 +169,14 @@ for (let f = 1; ; f++) { (gzlen >>> 24) & 255 ] - appendToOutfile(packetType) + appendToOutfile(videoPacketType) appendToOutfile(frameSize) appendToOutfilePtr(gzippedImage, gzlen) print(` ${gzlen} bytes\n`) + serial.print(` ${gzlen} bytes\n`) + + audioSamplesWrote -= AUDIO_SAMPLE_SIZE } // if there is no video and audio remaining, exit the loop diff --git a/assets/disk0/tvdos/bin/playmov.js b/assets/disk0/tvdos/bin/playmov.js index 1788128..513fe97 100644 --- a/assets/disk0/tvdos/bin/playmov.js +++ b/assets/disk0/tvdos/bin/playmov.js @@ -5,7 +5,7 @@ const HEIGHT = 448 const FBUF_SIZE = WIDTH * HEIGHT const AUTO_BGCOLOUR_CHANGE = true const MAGIC = [0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56] - +const pcm = require("pcm") const fullFilePath = _G.shell.resolvePathInput(exec_args[1]) const FILE_LENGTH = files.open(fullFilePath.full).size @@ -41,14 +41,18 @@ let fps = seqread.readShort(); if (fps == 0) fps = 9999 //fps = 9999 -let frameTime = 1.0 / fps -let frameCount = seqread.readInt() % 16777216 -let globalType = seqread.readShort() -sys.free(seqread.readBytes(12)) // skip 12 bytes -let akku = frameTime +const FRAME_TIME = 1.0 / fps +const FRAME_COUNT = seqread.readInt() % 16777216 +const globalType = seqread.readShort() +const audioQueueInfo = seqread.readShort() +let AUDIO_QUEUE_LENGTH = (audioQueueInfo >> 12) + 1 +const AUDIO_QUEUE_BYTES = (audioQueueInfo & 0xFFF) << 2 +sys.free(seqread.readBytes(10)) // skip 12 bytes +let audioQueuePos = 0 +let akku = FRAME_TIME let framesRendered = 0 //serial.println(seqread.getReadCount()) // must say 18 -//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${frameCount}`) +//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${FRAME_COUNT}`) /*if (type != 4 && type != 5 && type != 260 && type != 261) { printerrln("Not an iPF mov") @@ -58,6 +62,10 @@ if (globalType != 255) { printerrln(`Unsupported MOV type (${globalType})`) return 1 } +// MP2 stuffs +let mp2context; +let samplePtrL; +let samplePtrR; let ipfbuf = sys.malloc(FBUF_SIZE) @@ -66,12 +74,27 @@ graphics.setGraphicsMode(4) let startTime = sys.nanoTime() let framesRead = 0 let audioFired = false +let audioQueue = (AUDIO_QUEUE_LENGTH < 1) ? undefined : new Int32Array(AUDIO_QUEUE_LENGTH) +if (AUDIO_QUEUE_BYTES > 0 && AUDIO_QUEUE_LENGTH > 1) { + for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) { + audioQueue[i] = sys.malloc(AUDIO_QUEUE_BYTES) + } +} audio.resetParams(0) audio.purgeQueue(0) audio.setPcmMode(0) audio.setMasterVolume(0, 255) +function s16StTou8St(inPtrL, inPtrR, outPtr, length) { + for (let k = 0; k < length; k+=2) { + let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8)) + let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8)) + sys.poke(outPtr + k, pcm.s16Tou8(sample1)) + sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2)) + } +} + function getRGBfromScr(x, y) { let offset = y * WIDTH + x let rg = sys.peek(-1048577 - offset) @@ -89,14 +112,16 @@ if (interactive) { let notifHideTimer = 0 const NOTIF_SHOWUPTIME = 3000000000 let [cy, cx] = con.getyx() +let errorlevel = 0 +try { let t1 = sys.nanoTime() renderLoop: while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { - if (akku >= frameTime) { + if (akku >= FRAME_TIME) { let frameUnit = 0 // 0: no decode, 1: normal playback, 2+: skip (n-1) frames - while (!stopPlay && akku >= frameTime) { + while (!stopPlay && akku >= FRAME_TIME) { if (interactive) { sys.poke(-40, 1) if (sys.peek(-41) == 67) { @@ -104,7 +129,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { } } - akku -= frameTime + akku -= FRAME_TIME frameUnit += 1 } @@ -144,7 +169,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { let decodefun = (packetType > 255) ? graphics.decodeIpf2 : graphics.decodeIpf1 let payloadLen = seqread.readInt() - if (framesRead >= frameCount) { + if (framesRead >= FRAME_COUNT) { break renderLoop } @@ -156,6 +181,11 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { gzip.decompFromTo(gzippedPtr, payloadLen, ipfbuf) // should return FBUF_SIZE decodefun(ipfbuf, -1048577, -1310721, width, height, (packetType & 255) == 5) + // defer audio playback until a first frame is sent + if (!audioFired) { + audio.play(0) + audioFired = true + } // calculate bgcolour from the edges of the screen if (AUTO_BGCOLOUR_CHANGE) { @@ -187,13 +217,6 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { graphics.setBackground(Math.round(bgr * 255), Math.round(bgg * 255), Math.round(bgb * 255)) } - - - // defer audio playback until a first frame is sent - if (!audioFired) { - audio.play(0) - audioFired = true - } } sys.free(gzippedPtr) @@ -203,18 +226,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { } } // audio packets - else if (4096 <= packetType && packetType <= 6133) { - if (4097 == packetType) { - let readLength = seqread.readInt() - let samples = seqread.readBytes(readLength) + else if (4096 <= packetType && packetType <= 6143) { + let readLength = seqread.readInt() + if (readLength == 0) throw Error("Readlength is zero") - if (readLength == 0) throw Error("Readlength is zero") + // MP2 + if (packetType == 0x1100 || packetType == 0x1101) { + if (audioQueue[audioQueuePos] === undefined) { +// throw Error(`Audio queue overflow: attempt to write to index ${audioQueuePos}; queue size: ${audioQueue.length}; frame: ${framesRead}`) + AUDIO_QUEUE_LENGTH += 1 + audioQueue.push(sys.malloc(AUDIO_QUEUE_BYTES)) + } + if (mp2context === undefined) mp2context = audio.mp2Init() + if (samplePtrL === undefined) samplePtrL = sys.malloc(2304) // 16b samples + if (samplePtrR === undefined) samplePtrR = sys.malloc(2304) // 16b samples - audio.putPcmDataByPtr(samples, readLength, 0) + let frame = seqread.readBytes(readLength) + let [frameSize, samples] = audio.mp2DecodeFrame(mp2context, frame, true, samplePtrL, samplePtrR) + s16StTou8St(samplePtrL, samplePtrR, audioQueue[audioQueuePos++], samples) + sys.free(frame) + } + // RAW PCM packets (decode on the fly) + else if (packetType == 0x1000 || packetType == 0x1001) { + let frame = seqread.readBytes(readLength) + audio.putPcmDataByPtr(frame, readLength, 0) audio.setSampleUploadLength(0, readLength) audio.startSampleUpload(0) - - sys.free(samples) + sys.free(frame) } else { throw Error(`Audio Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`) @@ -223,10 +261,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { else { println(`Unknown Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`) } + + + // manage audio playback + if (audioFired && audioQueue) { + if (audio.getPosition(0) < 1 && audioQueuePos > 0) { + // push audio sample + audio.putPcmDataByPtr(audioQueue[0], AUDIO_QUEUE_BYTES, 0) + audio.setSampleUploadLength(0, AUDIO_QUEUE_BYTES) + audio.startSampleUpload(0) + + // unshift the queue + const tmp = audioQueue[0] + for (let i = 1; i < AUDIO_QUEUE_LENGTH; i++) audioQueue[i - 1] = audioQueue[i] + audioQueue[AUDIO_QUEUE_LENGTH - 1] = tmp + + audioQueuePos -= 1 + sys.spin() + } + } } } else { + + serial.println(`frameunit ${frameUnit}`) + framesRendered += 1 + } } @@ -237,18 +298,34 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) { if (interactive) { notifHideTimer += (t2 - t1) - if (notifHideTimer > (NOTIF_SHOWUPTIME + frameTime)) { + if (notifHideTimer > (NOTIF_SHOWUPTIME + FRAME_TIME)) { con.clear() } } t1 = t2 } -let endTime = sys.nanoTime() +} +catch (e) { + printerrln(e) + errorlevel = 1 +} +finally { + let endTime = sys.nanoTime() -sys.free(ipfbuf) -//audio.stop(0) + sys.free(ipfbuf) + if (audioQueue) { + for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) { + sys.free(audioQueue[i]) + } + } + if (samplePtrL !== undefined) sys.free(samplePtrL) + if (samplePtrR !== undefined) sys.free(samplePtrR) + //audio.stop(0) -let timeTook = (endTime - startTime) / 1000000000.0 + let timeTook = (endTime - startTime) / 1000000000.0 -//println(`Actual FPS: ${framesRendered / timeTook}`) \ No newline at end of file + //println(`Actual FPS: ${framesRendered / timeTook}`) +} + +return errorlevel \ No newline at end of file diff --git a/assets/disk0/tvdos/bin/playmp2.js b/assets/disk0/tvdos/bin/playmp2.js index c7c3514..a23ba38 100644 --- a/assets/disk0/tvdos/bin/playmp2.js +++ b/assets/disk0/tvdos/bin/playmp2.js @@ -82,12 +82,10 @@ let decodedLength = 0 function decodeAndResample(inPtrL, inPtrR, outPtr, inputLen) { // TODO resample for (let k = 0; k < inputLen; k+=2) { - let sample = [ - pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8)), - pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8)) - ] - sys.poke(outPtr + k, pcm.s16Tou8(sample[0])) - sys.poke(outPtr + k + 1, pcm.s16Tou8(sample[1])) + let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8)) + let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8)) + sys.poke(outPtr + k, pcm.s16Tou8(sample1)) + sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2)) } } function decodeEvent(frameSize, len) { diff --git a/terranmon.txt b/terranmon.txt index 461ccbb..2681c2e 100644 --- a/terranmon.txt +++ b/terranmon.txt @@ -379,9 +379,18 @@ METADATA - uint16 FPS (0: play as fast as can) uint32 NUMBER OF FRAMES uint16 GLOBAL PACKET TYPE (will be deprecated; please use 255,0) - byte[12] RESERVED + uint16 AUDIO QUEUE INFO + when read as little endian: + 0b nnnn bbbb bbbb bbbb + [byte 21] [byte 20] + n: size of the queue (number of entries). Allocate at least 1 more entry than the number specified! + b: size of each entry in bytes DIVIDED BY FOUR (all zero = 16384; always 0x240 for MP2 because MP2-VBR is not supported) - Packet Types: + n=0 indicates the video audio must be decoded on-the-fly instead of being queued, or has no audio packets + byte[10] RESERVED + + +Packet Types -