curioustorvald/tsvm
1
0
Fork 0
mirror of https://github.com/curioustorvald/tsvm.git synced 2026-03-07 19:51:51 +09:00
Code Issues Packages Projects Releases Wiki Activity

mov: mp2 audio encoding and playing

Browse Source
This commit is contained in:
minjaesong
2023-01-18 18:05:34 +09:00
parent dfe5bf3390
commit 6d19543139
4 changed files with 188 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
assets/disk0/tvdos/bin/encodemov.js
View File

@@ -6,7 +6,9 @@ let FPS = 30
let WIDTH = 560
let HEIGHT = 448
let PATHFUN = (i) => `/namu2/${(''+i).padStart(5,'0')}.png` // how can be the image file found, if a frame number (starts from 1) were given
let AUDIOTRACK = 'namu.u8'
let AUDIOTRACK = 'namu.mp2'
let AUDIOFORMAT = 'MP2fr' // PCMu8 or MP2fr
let MP2_PACKETSIZE;
// to export video to its frames:
// ffmpeg -i file.mp4 file/%05d.bmp
// the input frames must be resized (and cropped) beforehand, using ImageMagick is recommended, like so:
@@ -39,11 +41,17 @@ function appendToOutfilePtr(ptr, len) {
outfile.pappend(ptr, len)
}
const packetType = [
4, (IPFMODE - 1)
]
const syncPacket = [255, 255]
function audioFormatToAudioPacketType() {
return ("PCMu8" == AUDIOFORMAT) ? [1, 16]
: ("MP2fr" == AUDIOFORMAT) ? [1, 17]
: [255, 16]
}
const videoPacketType = [4, (IPFMODE - 1)]
const syncPacket = [255, 255]
const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo
const AUDIO_BLOCK_SIZE = ("MP2fr" == AUDIOFORMAT) ? 0x240 : 0
const AUDIO_QUEUE_SIZE = ("MP2fr" == AUDIOFORMAT) ? Math.ceil(AUDIO_SAMPLE_SIZE / 2304) + 1 : 0
// write header to the file
let headerBytes = [
0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56, // magic
@@ -52,7 +60,8 @@ let headerBytes = [
FPS & 255, (FPS >> 8) & 255, // FPS
TOTAL_FRAMES & 255, (TOTAL_FRAMES >> 8) & 255, (TOTAL_FRAMES >> 16) & 255, (TOTAL_FRAMES >> 24) & 255, // frame count
0xFF, 0x00, // new standard deprecates global type
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved
AUDIO_BLOCK_SIZE & 255, (AUDIO_BLOCK_SIZE >>> 8) | (AUDIO_QUEUE_SIZE << 4),
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // reserved
]
let ipfFun = (IPFMODE == 1) ? graphics.encodeIpf1 : (IPFMODE == 2) ? graphics.encodeIpf2 : 0
@@ -60,15 +69,25 @@ if (!ipfFun) throw Error("Unknown IPF mode "+IPFMODE)
const AUDIO_SAMPLE_SIZE = 2 * (((32000 / FPS) + 1)|0) // times 2 because stereo
let audioBytesRead = 0
const audioFile = (AUDIOTRACK) ? files.open(_G.shell.resolvePathInput(AUDIOTRACK).full) : undefined
let audioRemaining = (audioFile) ? audioFile.size : 0
const audioPacket = [1, 16]
const audioPacketType = audioFormatToAudioPacketType()
outfile.bwrite(headerBytes)
function getRepeatCount(fnum) {
if ("PCMu8" == AUDIOFORMAT) {
return (fnum == 1) ? 2 : 1
}
else if ("MP2fr" == AUDIOFORMAT) {
let r = Math.ceil((AUDIO_SAMPLE_SIZE*2 - audioSamplesWrote) / AUDIO_SAMPLE_SIZE) + ((fnum == 1) ? 1 : 0)
return (fnum > TOTAL_FRAMES) ? Math.ceil(audioRemaining / MP2_PACKETSIZE) : r
}
}
let audioSamplesWrote = 0
for (let f = 1; ; f++) {
// insert sync packet
@@ -78,29 +97,48 @@ for (let f = 1; ; f++) {
if (audioRemaining > 0) {
// first frame gets two audio packets
for (let repeat = 0; repeat < ((f == 1) ? 2 : 1); repeat++) {
for (let repeat = 0; repeat < getRepeatCount(f); repeat++) {
// print(`Frame ${f}/${TOTAL_FRAMES} (ADPCM) ->`)
print(`Frame ${f}/${TOTAL_FRAMES} (PCMu8) ->`)
print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`)
serial.print(`Frame ${f}/${TOTAL_FRAMES} (${AUDIOFORMAT}) ->`)
const actualBytesToRead = Math.min(
(f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2,
audioRemaining
)
audioFile.pread(infile, actualBytesToRead, audioBytesRead)
// read a chunk/mpeg-frame
let actualBytesToRead;
if ("PCMu8" == AUDIOFORMAT) {
actualBytesToRead = Math.min(
(f % 2 == 1) ? AUDIO_SAMPLE_SIZE : AUDIO_SAMPLE_SIZE + 2,
audioRemaining
)
audioFile.pread(infile, actualBytesToRead, audioBytesRead)
}
else if ("MP2fr" == AUDIOFORMAT) {
if (!MP2_PACKETSIZE) {
audioFile.pread(infile, 3, 0)
MP2_PACKETSIZE = audio.mp2GetInitialFrameSize([sys.peek(infile),sys.peek(infile+1),sys.peek(infile+2)])
}
let pcmSize = [
actualBytesToRead = Math.min(MP2_PACKETSIZE, audioRemaining)
audioFile.pread(infile, actualBytesToRead, audioBytesRead)
audioSamplesWrote += 2304
}
else throw Error("Unknown audio format: " + AUDIOFORMAT)
// writeout
let audioSize = [
(actualBytesToRead >>> 0) & 255,
(actualBytesToRead >>> 8) & 255,
(actualBytesToRead >>> 16) & 255,
(actualBytesToRead >>> 24) & 255
]
appendToOutfile(audioPacket)
appendToOutfile(pcmSize)
appendToOutfile(audioPacketType)
appendToOutfile(audioSize)
appendToOutfilePtr(infile, actualBytesToRead)
print(` ${actualBytesToRead} bytes\n`)
serial.print(` ${actualBytesToRead} bytes\n`)
audioBytesRead += actualBytesToRead
audioRemaining -= actualBytesToRead
@@ -117,6 +155,7 @@ for (let f = 1; ; f++) {
let [_1, _2, channels, _3] = graphics.decodeImageTo(infile, fileLen, imagearea)
print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
serial.print(`Frame ${f}/${TOTAL_FRAMES} (Ch: ${channels}) ->`)
// graphics.imageToDisplayableFormat(imagearea, decodearea, 560, 448, 3, 1)
ipfFun(imagearea, ipfarea, WIDTH, HEIGHT, channels, false, f)
@@ -130,11 +169,14 @@ for (let f = 1; ; f++) {
(gzlen >>> 24) & 255
]
appendToOutfile(packetType)
appendToOutfile(videoPacketType)
appendToOutfile(frameSize)
appendToOutfilePtr(gzippedImage, gzlen)
print(` ${gzlen} bytes\n`)
serial.print(` ${gzlen} bytes\n`)
audioSamplesWrote -= AUDIO_SAMPLE_SIZE
}
// if there is no video and audio remaining, exit the loop

141
assets/disk0/tvdos/bin/playmov.js
View File

@@ -5,7 +5,7 @@ const HEIGHT = 448
const FBUF_SIZE = WIDTH * HEIGHT
const AUTO_BGCOLOUR_CHANGE = true
const MAGIC = [0x1F, 0x54, 0x53, 0x56, 0x4D, 0x4D, 0x4F, 0x56]
const pcm = require("pcm")
const fullFilePath = _G.shell.resolvePathInput(exec_args[1])
const FILE_LENGTH = files.open(fullFilePath.full).size
@@ -41,14 +41,18 @@ let fps = seqread.readShort(); if (fps == 0) fps = 9999
//fps = 9999
let frameTime = 1.0 / fps
let frameCount = seqread.readInt() % 16777216
let globalType = seqread.readShort()
sys.free(seqread.readBytes(12)) // skip 12 bytes
let akku = frameTime
const FRAME_TIME = 1.0 / fps
const FRAME_COUNT = seqread.readInt() % 16777216
const globalType = seqread.readShort()
const audioQueueInfo = seqread.readShort()
let AUDIO_QUEUE_LENGTH = (audioQueueInfo >> 12) + 1
const AUDIO_QUEUE_BYTES = (audioQueueInfo & 0xFFF) << 2
sys.free(seqread.readBytes(10)) // skip 12 bytes
let audioQueuePos = 0
let akku = FRAME_TIME
let framesRendered = 0
//serial.println(seqread.getReadCount()) // must say 18
//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${frameCount}`)
//serial.println(`Dim: (${width}x${height}), FPS: ${fps}, Frames: ${FRAME_COUNT}`)
/*if (type != 4 && type != 5 && type != 260 && type != 261) {
printerrln("Not an iPF mov")
@@ -58,6 +62,10 @@ if (globalType != 255) {
printerrln(`Unsupported MOV type (${globalType})`)
return 1
}
// MP2 stuffs
let mp2context;
let samplePtrL;
let samplePtrR;
let ipfbuf = sys.malloc(FBUF_SIZE)
@@ -66,12 +74,27 @@ graphics.setGraphicsMode(4)
let startTime = sys.nanoTime()
let framesRead = 0
let audioFired = false
let audioQueue = (AUDIO_QUEUE_LENGTH < 1) ? undefined : new Int32Array(AUDIO_QUEUE_LENGTH)
if (AUDIO_QUEUE_BYTES > 0 && AUDIO_QUEUE_LENGTH > 1) {
for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) {
audioQueue[i] = sys.malloc(AUDIO_QUEUE_BYTES)
}
}
audio.resetParams(0)
audio.purgeQueue(0)
audio.setPcmMode(0)
audio.setMasterVolume(0, 255)
function s16StTou8St(inPtrL, inPtrR, outPtr, length) {
for (let k = 0; k < length; k+=2) {
let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8))
let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
sys.poke(outPtr + k, pcm.s16Tou8(sample1))
sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2))
}
}
function getRGBfromScr(x, y) {
let offset = y * WIDTH + x
let rg = sys.peek(-1048577 - offset)
@@ -89,14 +112,16 @@ if (interactive) {
let notifHideTimer = 0
const NOTIF_SHOWUPTIME = 3000000000
let [cy, cx] = con.getyx()
let errorlevel = 0
try {
let t1 = sys.nanoTime()
renderLoop:
while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
if (akku >= frameTime) {
if (akku >= FRAME_TIME) {
let frameUnit = 0 // 0: no decode, 1: normal playback, 2+: skip (n-1) frames
while (!stopPlay && akku >= frameTime) {
while (!stopPlay && akku >= FRAME_TIME) {
if (interactive) {
sys.poke(-40, 1)
if (sys.peek(-41) == 67) {
@@ -104,7 +129,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
}
}
akku -= frameTime
akku -= FRAME_TIME
frameUnit += 1
}
@@ -144,7 +169,7 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
let decodefun = (packetType > 255) ? graphics.decodeIpf2 : graphics.decodeIpf1
let payloadLen = seqread.readInt()
if (framesRead >= frameCount) {
if (framesRead >= FRAME_COUNT) {
break renderLoop
}
@@ -156,6 +181,11 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
gzip.decompFromTo(gzippedPtr, payloadLen, ipfbuf) // should return FBUF_SIZE
decodefun(ipfbuf, -1048577, -1310721, width, height, (packetType & 255) == 5)
// defer audio playback until a first frame is sent
if (!audioFired) {
audio.play(0)
audioFired = true
}
// calculate bgcolour from the edges of the screen
if (AUTO_BGCOLOUR_CHANGE) {
@@ -187,13 +217,6 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
graphics.setBackground(Math.round(bgr * 255), Math.round(bgg * 255), Math.round(bgb * 255))
}
// defer audio playback until a first frame is sent
if (!audioFired) {
audio.play(0)
audioFired = true
}
}
sys.free(gzippedPtr)
@@ -203,18 +226,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
}
}
// audio packets
else if (4096 <= packetType && packetType <= 6133) {
if (4097 == packetType) {
let readLength = seqread.readInt()
let samples = seqread.readBytes(readLength)
else if (4096 <= packetType && packetType <= 6143) {
let readLength = seqread.readInt()
if (readLength == 0) throw Error("Readlength is zero")
if (readLength == 0) throw Error("Readlength is zero")
// MP2
if (packetType == 0x1100 || packetType == 0x1101) {
if (audioQueue[audioQueuePos] === undefined) {
// throw Error(`Audio queue overflow: attempt to write to index ${audioQueuePos}; queue size: ${audioQueue.length}; frame: ${framesRead}`)
AUDIO_QUEUE_LENGTH += 1
audioQueue.push(sys.malloc(AUDIO_QUEUE_BYTES))
}
if (mp2context === undefined) mp2context = audio.mp2Init()
if (samplePtrL === undefined) samplePtrL = sys.malloc(2304) // 16b samples
if (samplePtrR === undefined) samplePtrR = sys.malloc(2304) // 16b samples
audio.putPcmDataByPtr(samples, readLength, 0)
let frame = seqread.readBytes(readLength)
let [frameSize, samples] = audio.mp2DecodeFrame(mp2context, frame, true, samplePtrL, samplePtrR)
s16StTou8St(samplePtrL, samplePtrR, audioQueue[audioQueuePos++], samples)
sys.free(frame)
}
// RAW PCM packets (decode on the fly)
else if (packetType == 0x1000 || packetType == 0x1001) {
let frame = seqread.readBytes(readLength)
audio.putPcmDataByPtr(frame, readLength, 0)
audio.setSampleUploadLength(0, readLength)
audio.startSampleUpload(0)
sys.free(samples)
sys.free(frame)
}
else {
throw Error(`Audio Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`)
@@ -223,10 +261,33 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
else {
println(`Unknown Packet with type ${packetType} at offset ${seqread.getReadCount() - 2}`)
}
// manage audio playback
if (audioFired && audioQueue) {
if (audio.getPosition(0) < 1 && audioQueuePos > 0) {
// push audio sample
audio.putPcmDataByPtr(audioQueue[0], AUDIO_QUEUE_BYTES, 0)
audio.setSampleUploadLength(0, AUDIO_QUEUE_BYTES)
audio.startSampleUpload(0)
// unshift the queue
const tmp = audioQueue[0]
for (let i = 1; i < AUDIO_QUEUE_LENGTH; i++) audioQueue[i - 1] = audioQueue[i]
audioQueue[AUDIO_QUEUE_LENGTH - 1] = tmp
audioQueuePos -= 1
sys.spin()
}
}
}
}
else {
serial.println(`frameunit ${frameUnit}`)
framesRendered += 1
}
}
@@ -237,18 +298,34 @@ while (!stopPlay && seqread.getReadCount() < FILE_LENGTH) {
if (interactive) {
notifHideTimer += (t2 - t1)
if (notifHideTimer > (NOTIF_SHOWUPTIME + frameTime)) {
if (notifHideTimer > (NOTIF_SHOWUPTIME + FRAME_TIME)) {
con.clear()
}
}
t1 = t2
}
let endTime = sys.nanoTime()
}
catch (e) {
printerrln(e)
errorlevel = 1
}
finally {
let endTime = sys.nanoTime()
sys.free(ipfbuf)
//audio.stop(0)
sys.free(ipfbuf)
if (audioQueue) {
for (let i = 0; i < AUDIO_QUEUE_LENGTH; i++) {
sys.free(audioQueue[i])
}
}
if (samplePtrL !== undefined) sys.free(samplePtrL)
if (samplePtrR !== undefined) sys.free(samplePtrR)
//audio.stop(0)
let timeTook = (endTime - startTime) / 1000000000.0
let timeTook = (endTime - startTime) / 1000000000.0
//println(`Actual FPS: ${framesRendered / timeTook}`)
//println(`Actual FPS: ${framesRendered / timeTook}`)
}
return errorlevel

10
assets/disk0/tvdos/bin/playmp2.js
View File

@@ -82,12 +82,10 @@ let decodedLength = 0
function decodeAndResample(inPtrL, inPtrR, outPtr, inputLen) {
// TODO resample
for (let k = 0; k < inputLen; k+=2) {
let sample = [
pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8)),
pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
]
sys.poke(outPtr + k, pcm.s16Tou8(sample[0]))
sys.poke(outPtr + k + 1, pcm.s16Tou8(sample[1]))
let sample1 = pcm.u16Tos16(sys.peek(inPtrL + k + 0) | (sys.peek(inPtrL + k + 1) << 8))
let sample2 = pcm.u16Tos16(sys.peek(inPtrR + k + 0) | (sys.peek(inPtrR + k + 1) << 8))
sys.poke(outPtr + k, pcm.s16Tou8(sample1))
sys.poke(outPtr + k + 1, pcm.s16Tou8(sample2))
}
}
function decodeEvent(frameSize, len) {

17
terranmon.txt
View File

@@ -379,9 +379,18 @@ METADATA -
uint16 FPS (0: play as fast as can)
uint32 NUMBER OF FRAMES
uint16 GLOBAL PACKET TYPE (will be deprecated; please use 255,0)
byte[12] RESERVED
uint16 AUDIO QUEUE INFO
when read as little endian:
0b nnnn bbbb bbbb bbbb
[byte 21] [byte 20]
n: size of the queue (number of entries). Allocate at least 1 more entry than the number specified!
b: size of each entry in bytes DIVIDED BY FOUR (all zero = 16384; always 0x240 for MP2 because MP2-VBR is not supported)
Packet Types:
n=0 indicates the video audio must be decoded on-the-fly instead of being queued, or has no audio packets
byte[10] RESERVED
Packet Types -
<video>
0,0: 256-Colour frame
1,0: 256-Colour frame with palette data
@@ -398,6 +407,8 @@ METADATA -
1,16: Raw PCM Stereo
2,16: ADPCM Mono
3,16: ADPCM Stereo
0,17: MP2 Mono
1,17: MP2 Stereo
<special>
255,255: sync packet (wait until the next frame)
254,255: background colour packet
@@ -406,8 +417,6 @@ METADATA -
0..7: iPF Type 1..8
GLOBAL TYPE 0 Packet -
uint32 SIZE OF FRAMEDATA
* FRAMEDATA COMPRESSED IN GZIP