mov: mp2 audio encoding and playing

assets/disk0/tvdos/bin/encodemov.js

@@ -6,7 +6,9 @@ let FPS = 30
 let WIDTH = 560
 let HEIGHT = 448
 let PATHFUN = (i) => `/namu2/${(''+i).padStart(5,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given
-let AUDIOTRACK = 'namu.u8'
+let AUDIOTRACK = 'namu.mp2'
+let AUDIOFORMAT = 'MP2fr' // PCMu8 or MP2fr
+let MP2_PACKETSIZE;
 // to export video to its frames:
 //     ffmpeg -i file.mp4 file/%05d.bmp
 // the input frames must be resized (and cropped) beforehand, using ImageMagick is recommended, like so:
@@ -39,11 +41,17 @@ function appendToOutfilePtr(ptr, len) {
     outfile.pappend(ptr, len)
 }
 
-const packetType = [
+function audioFormatToAudioPacketType() {
-    4, (IPFMODE - 1)
+    return ("PCMu8" == AUDIOFORMAT) ? [1, 16]
-]
+    : ("MP2fr" == AUDIOFORMAT) ? [1, 17]
-const syncPacket = [255, 255]
+    : [255, 16]
+}
 
+const videoPacketType = [4, (IPFMODE - 1)]
+const syncPacket = [255, 255]
+const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo
+const AUDIO_BLOCK_SIZE = ("MP2fr" == AUDIOFORMAT) ? 0x240 : 0
+const AUDIO_QUEUE_SIZE = ("MP2fr" == AUDIOFORMAT) ? Math.ceil(AUDIO_SAMPLE_SIZE / 2304) + 1 : 0
 // write header to the file
 let headerBytes = [
     0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56, // magic
@@ -52,7 +60,8 @@ let headerBytes = [
     FPS & 255, (FPS >> 8) & 255, // FPS
     TOTAL_FRAMES & 255, (TOTAL_FRAMES >> 8) & 255, (TOTAL_FRAMES >> 16) & 255, (TOTAL_FRAMES >> 24) & 255, // frame count
     0xFF, 0x00, // new standard deprecates global type
-    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved
+    AUDIO_BLOCK_SIZE & 255, (AUDIO_BLOCK_SIZE >>> 8) | (AUDIO_QUEUE_SIZE << 4),
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved
 ]
 
 let ipfFun = (IPFMODE == 1) ? graphics.encodeIpf1 : (IPFMODE == 2) ? graphics.encodeIpf2 : 0
@@ -60,15 +69,25 @@ if (!ipfFun) throw Error("Unknown IPF mode "+IPFMODE)
 
 
 
-const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo
 let audioBytesRead = 0
 const audioFile = (AUDIOTRACK) ? files.open(_G.shell.resolvePathInput(AUDIOTRACK).full) : undefined
 let audioRemaining = (audioFile) ? audioFile.size : 0
-const audioPacket = [1, 16]
+const audioPacketType = audioFormatToAudioPacketType()
 
 
 outfile.bwrite(headerBytes)
 
+function getRepeatCount(fnum) {
+    if ("PCMu8" == AUDIOFORMAT) {
+        return (fnum == 1) ? 2 : 1
+    }
+    else if ("MP2fr" == AUDIOFORMAT) {
+        let r = Math.ceil((AUDIO_SAMPLE_SIZE*2 - audioSamplesWrote) / AUDIO_SAMPLE_SIZE) + ((fnum == 1) ? 1 : 0)
+        return (fnum > TOTAL_FRAMES) ? Math.ceil(audioRemaining / MP2_PACKETSIZE) : r
+    }
+}
+
+let audioSamplesWrote = 0
 for (let f = 1; ; f++) {
 
     // insert sync packet
@@ -78,29 +97,48 @@ for (let f = 1; ; f++) {
     if (audioRemaining > 0) {
 
         // first frame gets two audio packets
-        for (let repeat = 0; repeat < ((f == 1) ? 2 : 1); repeat++) {
+        for (let repeat = 0; repeat < getRepeatCount(f); repeat++) {
 
-    //        print(`Frame ${f}/${TOTAL_FRAMES} (ADPCM) ->`)
+            print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`)
-            print(`Frame ${f}/${TOTAL_FRAMES} (PCMu8) ->`)
+            serial.print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`)
 
-            const actualBytesToRead = Math.min(
+            // read a chunk/mpeg-frame
-                (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2,
+            let actualBytesToRead;
-                audioRemaining
+            if ("PCMu8" == AUDIOFORMAT) {
-            )
+                actualBytesToRead = Math.min(
-            audioFile.pread(infile, actualBytesToRead, audioBytesRead)
+                    (f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2,
+                    audioRemaining
+                )
+                audioFile.pread(infile, actualBytesToRead, audioBytesRead)
+            }
+            else if ("MP2fr" == AUDIOFORMAT) {
+                if (!MP2_PACKETSIZE) {
+                    audioFile.pread(infile, 3, 0)
+                    MP2_PACKETSIZE = audio.mp2GetInitialFrameSize([sys.peek(infile),sys.peek(infile+1),sys.peek(infile+2)])
+                }
 
-            let pcmSize = [
+                actualBytesToRead = Math.min(MP2_PACKETSIZE, audioRemaining)
+                audioFile.pread(infile, actualBytesToRead, audioBytesRead)
+                audioSamplesWrote += 2304
+            }
+            else throw Error("Unknown audio format: " + AUDIOFORMAT)
+
+            // writeout
+            let audioSize = [
                 (actualBytesToRead >>> 0) & 255,
                 (actualBytesToRead >>> 8) & 255,
                 (actualBytesToRead >>> 16) & 255,
                 (actualBytesToRead >>> 24) & 255
             ]
 
-            appendToOutfile(audioPacket)
+            appendToOutfile(audioPacketType)
-            appendToOutfile(pcmSize)
+            appendToOutfile(audioSize)
             appendToOutfilePtr(infile, actualBytesToRead)
 
+
+
             print(` ${actualBytesToRead} bytes\n`)
+            serial.print(` ${actualBytesToRead} bytes\n`)
 
             audioBytesRead += actualBytesToRead
             audioRemaining -= actualBytesToRead
@@ -117,6 +155,7 @@ for (let f = 1; ; f++) {
         let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea)
 
         print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
+        serial.print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
 
     //    graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1)
         ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f)
@@ -130,11 +169,14 @@ for (let f = 1; ; f++) {
             (gzlen >>> 24) & 255
         ]
 
-        appendToOutfile(packetType)
+        appendToOutfile(videoPacketType)
         appendToOutfile(frameSize)
         appendToOutfilePtr(gzippedImage, gzlen)
 
         print(` ${gzlen} bytes\n`)
+        serial.print(` ${gzlen} bytes\n`)
+
+        audioSamplesWrote -= AUDIO_SAMPLE_SIZE
     }
 
     // if there is no video and audio remaining, exit the loop

assets/disk0/tvdos/bin/playmov.js

@@ -5,7 +5,7 @@ const HEIGHT = 448
 const FBUF_SIZE = WIDTH * HEIGHT
 const AUTO_BGCOLOUR_CHANGE = true
 const MAGIC = [0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56]
-
+const pcm = require("pcm")
 
 const fullFilePath = _G.shell.resolvePathInput(exec_args[1])
 const FILE_LENGTH = files.open(fullFilePath.full).size
@@ -41,14 +41,18 @@ let fps = seqread.readShort(); if (fps == 0) fps = 9999
 
 //fps = 9999
 
-let frameTime = 1.0 / fps
+const FRAME_TIME = 1.0 / fps
-let frameCount = seqread.readInt() % 16777216
+const FRAME_COUNT = seqread.readInt() % 16777216
-let globalType = seqread.readShort()
+const globalType = seqread.readShort()
-sys.free(seqread.readBytes(12)) // skip 12 bytes
+const audioQueueInfo = seqread.readShort()
-let akku = frameTime
+let AUDIO_QUEUE_LENGTH = (audioQueueInfo >> 12) + 1
+const AUDIO_QUEUE_BYTES = (audioQueueInfo & 0xFFF) << 2
+sys.free(seqread.readBytes(10)) // skip 12 bytes
+let audioQueuePos = 0
+let akku = FRAME_TIME
 let framesRendered = 0
 //serial.println(seqread.getReadCount()) // must say 18
-//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${frameCount}`)
+//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${FRAME_COUNT}`)
 
 /*if (type != 4 && type != 5 && type != 260 && type != 261) {
     printerrln("Not an iPF mov")
@@ -58,6 +62,10 @@ if (globalType != 255) {
     printerrln(`Unsupported MOV type (${globalType})`)
     return 1
 }
+// MP2 stuffs
+let mp2context;
+let samplePtrL;
+let samplePtrR;
 
 
 let ipfbuf = sys.malloc(FBUF_SIZE)
@@ -66,12 +74,27 @@ graphics.setGraphicsMode(4)
 let startTime = sys.nanoTime()
 let framesRead = 0
 let audioFired = false
+let audioQueue = (AUDIO_QUEUE_LENGTH < 1) ? undefined : new Int32Array(AUDIO_QUEUE_LENGTH)
+if (AUDIO_QUEUE_BYTES > 0 && AUDIO_QUEUE_LENGTH > 1) {
+    for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) {
+        audioQueue[i] = sys.malloc(AUDIO_QUEUE_BYTES)
+    }
+}
 
 audio.resetParams(0)
 audio.purgeQueue(0)
 audio.setPcmMode(0)
 audio.setMasterVolume(0, 255)
 
+function s16StTou8St(inPtrL, inPtrR, outPtr, length) {
+    for (let k = 0; k < length; k+=2) {
+        let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8))
+        let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
+        sys.poke(outPtr + k, pcm.s16Tou8(sample1))
+        sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2))
+    }
+}
+
 function getRGBfromScr(x, y) {
     let offset = y * WIDTH + x
     let rg = sys.peek(-1048577 - offset)
@@ -89,14 +112,16 @@ if (interactive) {
 let notifHideTimer = 0
 const NOTIF_SHOWUPTIME = 3000000000
 let [cy, cx] = con.getyx()
+let errorlevel = 0
+try {
 let t1 = sys.nanoTime()
 renderLoop:
 while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
 
-    if (akku >= frameTime) {
+    if (akku >= FRAME_TIME) {
 
         let frameUnit = 0 // 0: no decode, 1: normal playback, 2+: skip (n-1) frames
-        while (!stopPlay && akku >= frameTime) {
+        while (!stopPlay && akku >= FRAME_TIME) {
             if (interactive) {
                 sys.poke(-40, 1)
                 if (sys.peek(-41) == 67) {
@@ -104,7 +129,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
                 }
             }
 
-            akku -= frameTime
+            akku -= FRAME_TIME
             frameUnit += 1
         }
 
@@ -144,7 +169,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
                         let decodefun = (packetType > 255) ? graphics.decodeIpf2 : graphics.decodeIpf1
                         let payloadLen = seqread.readInt()
 
-                        if (framesRead >= frameCount) {
+                        if (framesRead >= FRAME_COUNT) {
                             break renderLoop
                         }
 
@@ -156,6 +181,11 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
                             gzip.decompFromTo(gzippedPtr, payloadLen, ipfbuf) // should return FBUF_SIZE
                             decodefun(ipfbuf, -1048577, -1310721, width, height, (packetType & 255) == 5)
 
+                            // defer audio playback until a first frame is sent
+                            if (!audioFired) {
+                                audio.play(0)
+                                audioFired = true
+                            }
 
                             // calculate bgcolour from the edges of the screen
                             if (AUTO_BGCOLOUR_CHANGE) {
@@ -187,13 +217,6 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
 
                                 graphics.setBackground(Math.round(bgr * 255), Math.round(bgg * 255), Math.round(bgb * 255))
                             }
-
-
-                            // defer audio playback until a first frame is sent
-                            if (!audioFired) {
-                                audio.play(0)
-                                audioFired = true
-                            }
                         }
 
                         sys.free(gzippedPtr)
@@ -203,18 +226,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
                     }
                 }
                 // audio packets
-                else if (4096 <= packetType && packetType <= 6133) {
+                else if (4096 <= packetType && packetType <= 6143) {
-                    if (4097 == packetType) {
+                    let readLength = seqread.readInt()
-                        let readLength = seqread.readInt()
+                    if (readLength == 0) throw Error("Readlength is zero")
-                        let samples = seqread.readBytes(readLength)
 
-                        if (readLength == 0) throw Error("Readlength is zero")
+                    // MP2
+                    if (packetType == 0x1100 || packetType == 0x1101) {
+                        if (audioQueue[audioQueuePos] === undefined) {
+//                            throw Error(`Audio queue overflow: attempt to write to index ${audioQueuePos}; queue size: ${audioQueue.length}; frame: ${framesRead}`)
+                            AUDIO_QUEUE_LENGTH += 1
+                            audioQueue.push(sys.malloc(AUDIO_QUEUE_BYTES))
+                        }
+                        if (mp2context === undefined) mp2context = audio.mp2Init()
+                        if (samplePtrL === undefined) samplePtrL = sys.malloc(2304) // 16b samples
+                        if (samplePtrR === undefined) samplePtrR = sys.malloc(2304) // 16b samples
 
-                        audio.putPcmDataByPtr(samples, readLength, 0)
+                        let frame = seqread.readBytes(readLength)
+                        let [frameSize, samples] = audio.mp2DecodeFrame(mp2context, frame, true, samplePtrL, samplePtrR)
+                        s16StTou8St(samplePtrL, samplePtrR, audioQueue[audioQueuePos++], samples)
+                        sys.free(frame)
+                    }
+                    // RAW PCM packets (decode on the fly)
+                    else if (packetType == 0x1000 || packetType == 0x1001) {
+                        let frame = seqread.readBytes(readLength)
+                        audio.putPcmDataByPtr(frame, readLength, 0)
                         audio.setSampleUploadLength(0, readLength)
                         audio.startSampleUpload(0)
-
+                        sys.free(frame)
-                        sys.free(samples)
                     }
                     else {
                         throw Error(`Audio Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`)
@@ -223,10 +261,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
                 else {
                     println(`Unknown Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`)
                 }
+
+
+                // manage audio playback
+                if (audioFired && audioQueue) {
+                    if (audio.getPosition(0) < 1 && audioQueuePos > 0) {
+                        // push audio sample
+                        audio.putPcmDataByPtr(audioQueue[0], AUDIO_QUEUE_BYTES, 0)
+                        audio.setSampleUploadLength(0, AUDIO_QUEUE_BYTES)
+                        audio.startSampleUpload(0)
+
+                        // unshift the queue
+                        const tmp = audioQueue[0]
+                        for (let i = 1; i < AUDIO_QUEUE_LENGTH; i++) audioQueue[i - 1] = audioQueue[i]
+                        audioQueue[AUDIO_QUEUE_LENGTH - 1] = tmp
+
+                        audioQueuePos -= 1
+                        sys.spin()
+                    }
+                }
             }
         }
         else {
+
+            serial.println(`frameunit ${frameUnit}`)
+
             framesRendered += 1
+
         }
 
     }
@@ -237,18 +298,34 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
 
     if (interactive) {
         notifHideTimer += (t2 - t1)
-        if (notifHideTimer > (NOTIF_SHOWUPTIME + frameTime)) {
+        if (notifHideTimer > (NOTIF_SHOWUPTIME + FRAME_TIME)) {
             con.clear()
         }
     }
 
     t1 = t2
 }
-let endTime = sys.nanoTime()
+}
+catch (e) {
+    printerrln(e)
+    errorlevel = 1
+}
+finally {
+    let endTime = sys.nanoTime()
 
-sys.free(ipfbuf)
+    sys.free(ipfbuf)
-//audio.stop(0)
+    if (audioQueue) {
+        for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) {
+            sys.free(audioQueue[i])
+        }
+    }
+    if (samplePtrL !== undefined) sys.free(samplePtrL)
+    if (samplePtrR !== undefined) sys.free(samplePtrR)
+    //audio.stop(0)
 
-let timeTook = (endTime - startTime) / 1000000000.0
+    let timeTook = (endTime - startTime) / 1000000000.0
 
-//println(`Actual FPS: ${framesRendered / timeTook}`)
+    //println(`Actual FPS: ${framesRendered / timeTook}`)
+}
+
+return errorlevel

assets/disk0/tvdos/bin/playmp2.js

@@ -82,12 +82,10 @@ let decodedLength = 0
 function decodeAndResample(inPtrL, inPtrR, outPtr, inputLen) {
     // TODO resample
     for (let k = 0; k < inputLen; k+=2) {
-        let sample = [
+        let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8))
-            pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8)),
+        let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
-            pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
+        sys.poke(outPtr + k, pcm.s16Tou8(sample1))
-        ]
+        sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2))
-        sys.poke(outPtr + k, pcm.s16Tou8(sample[0]))
-        sys.poke(outPtr + k + 1, pcm.s16Tou8(sample[1]))
     }
 }
 function decodeEvent(frameSize, len) {

terranmon.txt

@@ -379,9 +379,18 @@ METADATA -
     uint16 FPS (0: play as fast as can)
     uint32 NUMBER OF FRAMES
     uint16 GLOBAL PACKET TYPE (will be deprecated; please use 255,0)
-    byte[12] RESERVED
+    uint16 AUDIO QUEUE INFO
+           when read as little endian:
+           0b nnnn bbbb bbbb bbbb
+              [byte 21] [byte 20]
+           n: size of the queue (number of entries). Allocate at least 1 more entry than the number specified!
+           b: size of each entry in bytes DIVIDED BY FOUR (all zero = 16384; always 0x240 for MP2 because MP2-VBR is not supported)
 
-    Packet Types:
+           n=0 indicates the video audio must be decoded on-the-fly instead of being queued, or has no audio packets
+    byte[10] RESERVED
+
+
+Packet Types -
     <video>
        0,0: 256-Colour frame
        1,0: 256-Colour frame with palette data
@@ -398,6 +407,8 @@ METADATA -
       1,16: Raw PCM Stereo
       2,16: ADPCM Mono
       3,16: ADPCM Stereo
+      0,17: MP2 Mono
+      1,17: MP2 Stereo
     <special>
     255,255: sync packet (wait until the next frame)
     254,255: background colour packet
@@ -406,8 +417,6 @@ METADATA -
         0..7: iPF Type 1..8
 
 
-
-
 GLOBAL TYPE 0 Packet -
     uint32 SIZE OF FRAMEDATA
     *      FRAMEDATA COMPRESSED IN GZIP