From 00c882aa8d7426a504485911b0a637b9152a8aa7 Mon Sep 17 00:00:00 2001 From: minjaesong Date: Thu, 6 Nov 2025 01:18:19 +0900 Subject: [PATCH] TEV/TAV: SSF-TC impl --- assets/disk0/tvdos/bin/playtav.js | 118 ++++++++++++++++- assets/disk0/tvdos/bin/playtev.js | 114 ++++++++++++++++- terranmon.txt | 8 +- .../torvald/tsvm/GraphicsJSR223Delegate.kt | 17 +-- video_encoder/encoder_tav.c | 121 +++++++++++------- video_encoder/encoder_tev.c | 107 +++++++++------- video_encoder/tav_inspector.c | 36 ++++-- 7 files changed, 403 insertions(+), 118 deletions(-) diff --git a/assets/disk0/tvdos/bin/playtav.js b/assets/disk0/tvdos/bin/playtav.js index 96173eb..e1f4bec 100644 --- a/assets/disk0/tvdos/bin/playtav.js +++ b/assets/disk0/tvdos/bin/playtav.js @@ -35,7 +35,8 @@ const TAV_PACKET_AUDIO_NATIVE = 0x21 const TAV_PACKET_AUDIO_PCM_16LE = 0x22 const TAV_PACKET_AUDIO_ADPCM = 0x23 const TAV_PACKET_AUDIO_TAD = 0x24 -const TAV_PACKET_SUBTITLE = 0x30 +const TAV_PACKET_SUBTITLE = 0x30 // Legacy SSF (frame-locked) +const TAV_PACKET_SUBTITLE_TC = 0x31 // SSF-TC (timecode-based) const TAV_PACKET_AUDIO_BUNDLED = 0x40 // Entire MP2 audio file in single packet const TAV_PACKET_EXTENDED_HDR = 0xEF const TAV_PACKET_GOP_SYNC = 0xFC // GOP sync (N frames decoded from GOP block) @@ -64,6 +65,11 @@ let subtitleVisible = false let subtitleText = "" let subtitlePosition = 0 // 0=bottom center (default) +// SSF-TC subtitle event buffer +let subtitleEvents = [] // Array of {timecode_ns, index, opcode, text} +let nextSubtitleEventIndex = 0 // Next event to check +let currentTimecodeNs = 0 // Current playback timecode + // Parse command line options let interactive = false let filmGrainLevel = null @@ -123,7 +129,7 @@ if (fullFilePathStr.startsWith('$:/TAPE') || fullFilePathStr.startsWith('$:\\TAP con.clear() con.curs_set(0) graphics.setGraphicsMode(4) // initially set to 4bpp mode -graphics.setGraphicsMode(5) // set to 8bpp mode. If GPU don't support it, mode will remain to 4 +//graphics.setGraphicsMode(5) // set to 8bpp mode. If GPU don't support it, mode will remain to 4 graphics.clearPixels(0) graphics.clearPixels2(0) graphics.clearPixels3(0) @@ -140,6 +146,99 @@ audio.setMasterVolume(0, 255) // set colour zero as half-opaque black graphics.setPalette(0, 0, 0, 0, 7) +// Parse SSF-TC subtitle packet and add to event buffer (0x31) +function parseSubtitlePacketTC(packetSize) { + // Read subtitle index (24-bit, little-endian) + let indexByte0 = seqread.readOneByte() + let indexByte1 = seqread.readOneByte() + let indexByte2 = seqread.readOneByte() + let index = indexByte0 | (indexByte1 << 8) | (indexByte2 << 16) + + // Read timecode (64-bit, little-endian) + let timecode_ns = 0 + for (let i = 0; i < 8; i++) { + let byte = seqread.readOneByte() + timecode_ns += byte * Math.pow(2, i * 8) + } + + // Read opcode + let opcode = seqread.readOneByte() + let remainingBytes = packetSize - 12 // Subtract 3 (index) + 8 (timecode) + 1 (opcode) + + // Read text if present + let text = null + if (remainingBytes > 1 && (opcode === SSF_OP_SHOW || (opcode >= 0x10 && opcode <= 0x2F))) { + let textBytes = seqread.readBytes(remainingBytes) + text = "" + for (let i = 0; i < remainingBytes - 1; i++) { // -1 for null terminator + let byte = sys.peek(textBytes + i) + if (byte === 0) break + text += String.fromCharCode(byte) + } + sys.free(textBytes) + } else if (remainingBytes > 0) { + // Skip remaining bytes + let skipBytes = seqread.readBytes(remainingBytes) + sys.free(skipBytes) + } + + // Add event to buffer + subtitleEvents.push({ + timecode_ns: timecode_ns, + index: index, + opcode: opcode, + text: text + }) +} + +// Process subtitle events based on current playback time +function processSubtitleEvents(currentTimeNs) { + // Process all events whose timecode has been reached + while (nextSubtitleEventIndex < subtitleEvents.length) { + let event = subtitleEvents[nextSubtitleEventIndex] + + if (event.timecode_ns > currentTimeNs) { + break // Haven't reached this event yet + } + + // Execute the subtitle event + switch (event.opcode) { + case SSF_OP_SHOW: + subtitleText = event.text || "" + subtitleVisible = true + gui.displaySubtitle(subtitleText, fontUploaded, subtitlePosition) + break + + case SSF_OP_HIDE: + subtitleVisible = false + subtitleText = "" + gui.clearSubtitleArea() + break + + case SSF_OP_MOVE: + if (event.text && event.text.length > 0) { + let newPosition = event.text.charCodeAt(0) + if (newPosition >= 0 && newPosition <= 8) { + subtitlePosition = newPosition + if (subtitleVisible && subtitleText.length > 0) { + gui.clearSubtitleArea() + gui.displaySubtitle(subtitleText, fontUploaded, subtitlePosition) + } + } + } + break + + case SSF_OP_UPLOAD_LOW_FONT: + case SSF_OP_UPLOAD_HIGH_FONT: + // Font upload handled during packet parsing + break + } + + nextSubtitleEventIndex++ + } +} + +// Process legacy frame-locked subtitle packet (0x30) function processSubtitlePacket(packetSize) { // Read subtitle packet data according to SSF format @@ -1378,10 +1477,15 @@ try { sys.free(pcmPtr) } else if (packetType === TAV_PACKET_SUBTITLE) { - // Subtitle packet - same format as TEV + // Legacy frame-locked subtitle packet (0x30) let packetSize = seqread.readInt() processSubtitlePacket(packetSize) } + else if (packetType === TAV_PACKET_SUBTITLE_TC) { + // SSF-TC subtitle packet (0x31) - parse and buffer for later playback + let packetSize = seqread.readInt() + parseSubtitlePacketTC(packetSize) + } else if (packetType === TAV_PACKET_EXTENDED_HDR) { // Extended header packet - metadata key-value pairs let numPairs = seqread.readShort() @@ -1444,6 +1548,14 @@ try { let timecodeHigh = seqread.readInt() let timecodeNs = timecodeHigh * 0x100000000 + (timecodeLow >>> 0) + // Update current timecode and process subtitle events + currentTimecodeNs = timecodeNs + + // Process SSF-TC subtitle events based on current playback time + if (subtitleEvents.length > 0) { + processSubtitleEvents(currentTimecodeNs) + } + // Optionally display timecode in interactive mode (can be verbose) // Uncomment for debugging: // if (interactive && frameCount % 60 === 0) { diff --git a/assets/disk0/tvdos/bin/playtev.js b/assets/disk0/tvdos/bin/playtev.js index 6c77115..a40b217 100644 --- a/assets/disk0/tvdos/bin/playtev.js +++ b/assets/disk0/tvdos/bin/playtev.js @@ -26,7 +26,8 @@ const TEV_MODE_MOTION = 0x03 const TEV_PACKET_IFRAME = 0x10 const TEV_PACKET_PFRAME = 0x11 const TEV_PACKET_AUDIO_MP2 = 0x20 -const TEV_PACKET_SUBTITLE = 0x30 +const TEV_PACKET_SUBTITLE = 0x30 // Legacy SSF (frame-locked) +const TEV_PACKET_SUBTITLE_TC = 0x31 // SSF-TC (timecode-based) const TEV_PACKET_SYNC = 0xFF // Subtitle opcodes (SSF format) @@ -42,6 +43,10 @@ let subtitleVisible = false let subtitleText = "" let subtitlePosition = 0 // 0=bottom center (default) +// SSF-TC subtitle event buffer +let subtitleEvents = [] // Array of {timecode_ns, index, opcode, text} +let nextSubtitleEventIndex = 0 // Next event to check + // Parse command line options let interactive = false let debugMotionVectors = false @@ -274,6 +279,99 @@ function displaySubtitle(text, position = 0) { con.color_pair(oldFgColor, oldBgColor) } +// Parse SSF-TC subtitle packet and add to event buffer (0x31) +function parseSubtitlePacketTC(packetSize) { + // Read subtitle index (24-bit, little-endian) + let indexByte0 = seqread.readOneByte() + let indexByte1 = seqread.readOneByte() + let indexByte2 = seqread.readOneByte() + let index = indexByte0 | (indexByte1 << 8) | (indexByte2 << 16) + + // Read timecode (64-bit, little-endian) + let timecode_ns = 0 + for (let i = 0; i < 8; i++) { + let byte = seqread.readOneByte() + timecode_ns += byte * Math.pow(2, i * 8) + } + + // Read opcode + let opcode = seqread.readOneByte() + let remainingBytes = packetSize - 12 // Subtract 3 (index) + 8 (timecode) + 1 (opcode) + + // Read text if present + let text = null + if (remainingBytes > 1 && (opcode === SSF_OP_SHOW || (opcode >= 0x10 && opcode <= 0x2F))) { + let textBytes = seqread.readBytes(remainingBytes) + text = "" + for (let i = 0; i < remainingBytes - 1; i++) { // -1 for null terminator + let byte = sys.peek(textBytes + i) + if (byte === 0) break + text += String.fromCharCode(byte) + } + sys.free(textBytes) + } else if (remainingBytes > 0) { + // Skip remaining bytes + let skipBytes = seqread.readBytes(remainingBytes) + sys.free(skipBytes) + } + + // Add event to buffer + subtitleEvents.push({ + timecode_ns: timecode_ns, + index: index, + opcode: opcode, + text: text + }) +} + +// Process subtitle events based on current playback time +function processSubtitleEvents(currentTimeNs) { + // Process all events whose timecode has been reached + while (nextSubtitleEventIndex < subtitleEvents.length) { + let event = subtitleEvents[nextSubtitleEventIndex] + + if (event.timecode_ns > currentTimeNs) { + break // Haven't reached this event yet + } + + // Execute the subtitle event + switch (event.opcode) { + case SSF_OP_SHOW: + subtitleText = event.text || "" + subtitleVisible = true + displaySubtitle(subtitleText, subtitlePosition) + break + + case SSF_OP_HIDE: + subtitleVisible = false + subtitleText = "" + clearSubtitleArea() + break + + case SSF_OP_MOVE: + if (event.text && event.text.length > 0) { + let newPosition = event.text.charCodeAt(0) + if (newPosition >= 0 && newPosition <= 8) { + subtitlePosition = newPosition + if (subtitleVisible && subtitleText.length > 0) { + clearSubtitleArea() + displaySubtitle(subtitleText, subtitlePosition) + } + } + } + break + + case SSF_OP_UPLOAD_LOW_FONT: + case SSF_OP_UPLOAD_HIGH_FONT: + // Font upload handled during packet parsing + break + } + + nextSubtitleEventIndex++ + } +} + +// Process legacy frame-locked subtitle packet (0x30) function processSubtitlePacket(packetSize) { // Read subtitle packet data according to SSF format // uint24 index + uint8 opcode + variable arguments @@ -450,6 +548,7 @@ function getVideoRate() { } let FRAME_TIME = 1.0 / fps +let FRAME_TIME_NS = (1000000000.0 / fps) // Frame time in nanoseconds for subtitle timing // Ultra-fast approach: always render to display, use dedicated previous frame buffer const FRAME_PIXELS = width * height @@ -682,6 +781,12 @@ try { serial.println(`Frame ${frameCount}: Duplicating previous frame`) } + // Process SSF-TC subtitle events based on current playback time + if (subtitleEvents.length > 0) { + let currentTimeNs = frameCount * FRAME_TIME_NS + processSubtitleEvents(currentTimeNs) + } + // Defer audio playback until a first frame is sent if (isInterlaced) { // fire audio after frame 1 @@ -729,9 +834,14 @@ try { audio.mp2UploadDecoded(0) } else if (packetType == TEV_PACKET_SUBTITLE) { - // Subtitle packet + // Legacy frame-locked subtitle packet (0x30) let packetSize = seqread.readInt() processSubtitlePacket(packetSize) + + } else if (packetType == TEV_PACKET_SUBTITLE_TC) { + // SSF-TC subtitle packet (0x31) - parse and buffer for later playback + let packetSize = seqread.readInt() + parseSubtitlePacketTC(packetSize) } else if (packetType == 0x00) { // Silently discard, faulty subtitle creation can cause this as 0x00 is used as an argument terminator } else { diff --git a/terranmon.txt b/terranmon.txt index fc7d14b..cd8cd63 100644 --- a/terranmon.txt +++ b/terranmon.txt @@ -818,7 +818,7 @@ When SSF is interleaved with MP2 audio, the payload must be inserted in-between ## SSF Packet Structure uint24 Subtitle object ID (used to specify target subtitle object) - uint64 Timecode in nanoseconds (only present for SSF-TC format; regular SSF must not write these bytes) + uint64 Timecode in nanoseconds (only present on SSF-TC format; regular SSF must not write these bytes) uint8 opcode 0x00 = , is NOP when used here 0x01 = show (arguments: UTF-8 text) @@ -854,7 +854,7 @@ When KSF is interleaved with MP2 audio, the payload must be inserted in-between on appropriate timings. uint24 Subtitle object ID (used to specify target subtitle object) - uint64 Timecode in nanoseconds (only present for KSF-TC format; regular KSF must not write these bytes) + uint64 Timecode in nanoseconds (only present on KSF-TC format; regular KSF must not write these bytes) uint8 opcode 0x00 = , is NOP when used here @@ -1012,12 +1012,14 @@ transmission capability, and region-of-interest coding. 1. TAV Extended header (if any) 2. Standard metadata payloads (if any) 3. SSF-TC/KSF-TC packets (if any) + When time-coded subtitles are used, the entire subtitle bytes must precede the first video frame. + Think of it as tacking the whole subtitle file before the actual video. Frame group: 1. TC Packet (0xFD) or Next TAV File (0x1F) [mutually exclusive!] 2. Loop point packet (if any) 3. Audio packets (if any) - 4. Subtitle packets (if any) + 4. Subtitle packets (if any) [mutually exclusive with SSF-TC/KSF-TC packets] 5. Main video packets (0x10-0x1E) 6. Multiplexed video packets (0x70-7F; if any) diff --git a/tsvm_core/src/net/torvald/tsvm/GraphicsJSR223Delegate.kt b/tsvm_core/src/net/torvald/tsvm/GraphicsJSR223Delegate.kt index b58a796..604cc13 100644 --- a/tsvm_core/src/net/torvald/tsvm/GraphicsJSR223Delegate.kt +++ b/tsvm_core/src/net/torvald/tsvm/GraphicsJSR223Delegate.kt @@ -536,7 +536,7 @@ class GraphicsJSR223Delegate(private val vm: VM) { 3f/16f,5f/16f,1f/16f, ) - private val bayerKernels = arrayOf( + private val bayerKernelsInt = arrayOf( intArrayOf( 0,8,2,10, 12,4,14,6, @@ -561,7 +561,9 @@ class GraphicsJSR223Delegate(private val vm: VM) { 12,4,14,6, 3,11,1,9, ) - ).map{ it.map { (it.toFloat() + 0.5f) / 16f }.toFloatArray() } + ) + + private val bayerKernels = bayerKernelsInt.map{ it.map { (it.toFloat() + 0.5f) / 16f }.toFloatArray() } /** * This method always assume that you're using the default palette @@ -6583,13 +6585,6 @@ class GraphicsJSR223Delegate(private val vm: VM) { val offsetY = (nativeHeight - height) / 2 // Dithering pattern for 8bpp → 4bpp conversion - val bayerMatrix = arrayOf( - intArrayOf(0, 8, 2, 10), - intArrayOf(12, 4, 14, 6), - intArrayOf(3, 11, 1, 9), - intArrayOf(15, 7, 13, 5) - ) - // Process row by row for (y in 0 until height) { val screenY = y + offsetY @@ -6611,13 +6606,13 @@ class GraphicsJSR223Delegate(private val vm: VM) { if (graphicsMode == 4) { // 4bpp mode: dithered RGB (RG in fb1, B_ in fb2) - val threshold = bayerMatrix[y % 4][x % 4] + val threshold = bayerKernelsInt[frameCount % 4][4 * (y % 4) + (x % 4)] val rDithered = ((r + (threshold - 8)) shr 4).coerceIn(0, 15) val gDithered = ((g + (threshold - 8)) shr 4).coerceIn(0, 15) val bDithered = ((b + (threshold - 8)) shr 4).coerceIn(0, 15) gpu.framebuffer[screenPixelIdx] = ((rDithered shl 4) or gDithered).toByte() - gpu.framebuffer2?.set(screenPixelIdx, (bDithered shl 4).toByte()) + gpu.framebuffer2?.set(screenPixelIdx, ((bDithered shl 4) or 15).toByte()) } else if (graphicsMode == 5) { // 8bpp mode: full RGB (R in fb1, G in fb2, B in fb3) gpu.framebuffer[screenPixelIdx] = r.toByte() diff --git a/video_encoder/encoder_tav.c b/video_encoder/encoder_tav.c index f73078c..70df404 100644 --- a/video_encoder/encoder_tav.c +++ b/video_encoder/encoder_tav.c @@ -18,7 +18,7 @@ #include #include -#define ENCODER_VENDOR_STRING "Encoder-TAV 20251031 (3d-dwt,tad)" +#define ENCODER_VENDOR_STRING "Encoder-TAV 20251106 (3d-dwt,tad,ssf-tc)" // TSVM Advanced Video (TAV) format constants #define TAV_MAGIC "\x1F\x54\x53\x56\x4D\x54\x41\x56" // "\x1FTSVM TAV" @@ -56,7 +56,7 @@ #define TAV_PACKET_AUDIO_MP2 0x20 // MP2 audio #define TAV_PACKET_AUDIO_PCM8 0x21 // 8-bit PCM audio (zstd compressed) #define TAV_PACKET_AUDIO_TAD 0x24 // TAD audio (DWT-based perceptual codec) -#define TAV_PACKET_SUBTITLE 0x30 // Subtitle packet +#define TAV_PACKET_SUBTITLE_TC 0x31 // Subtitle packet with timecode (SSF-TC format) #define TAV_PACKET_AUDIO_TRACK 0x40 // Separate audio track (full MP2 file) #define TAV_PACKET_EXTENDED_HDR 0xEF // Extended header packet #define TAV_PACKET_GOP_SYNC 0xFC // GOP sync packet (N frames decoded) @@ -8349,30 +8349,42 @@ static void free_subtitle_list(subtitle_entry_t *list) { } // Write subtitle packet (copied from TEV) -static int write_subtitle_packet(FILE *output, uint32_t index, uint8_t opcode, const char *text) { - // Calculate packet size +// Write SSF-TC subtitle packet to output +static int write_subtitle_packet_tc(FILE *output, uint32_t index, uint8_t opcode, const char *text, uint64_t timecode_ns) { + // Calculate packet size: index (3 bytes) + timecode (8 bytes) + opcode (1 byte) + text + null terminator size_t text_len = text ? strlen(text) : 0; - size_t packet_size = 3 + 1 + text_len + 1; // index (3 bytes) + opcode + text + null terminator + size_t packet_size = 3 + 8 + 1 + text_len + 1; // Write packet type and size - uint8_t packet_type = TAV_PACKET_SUBTITLE; + uint8_t packet_type = TAV_PACKET_SUBTITLE_TC; fwrite(&packet_type, 1, 1, output); uint32_t size32 = (uint32_t)packet_size; fwrite(&size32, 4, 1, output); - // Write subtitle data + // Write subtitle index (24-bit, little-endian) uint8_t index_bytes[3] = { (uint8_t)(index & 0xFF), (uint8_t)((index >> 8) & 0xFF), (uint8_t)((index >> 16) & 0xFF) }; fwrite(index_bytes, 3, 1, output); + + // Write timecode (64-bit, little-endian) + uint8_t timecode_bytes[8]; + for (int i = 0; i < 8; i++) { + timecode_bytes[i] = (timecode_ns >> (i * 8)) & 0xFF; + } + fwrite(timecode_bytes, 8, 1, output); + + // Write opcode fwrite(&opcode, 1, 1, output); + // Write text if present if (text && text_len > 0) { fwrite(text, 1, text_len, output); } + // Write null terminator uint8_t null_terminator = 0; fwrite(&null_terminator, 1, 1, output); @@ -9034,47 +9046,59 @@ static int process_audio_for_gop(tav_encoder_t *enc, int *frame_numbers, int num return 1; } -// Process subtitles for current frame (copied and adapted from TEV) -static int process_subtitles(tav_encoder_t *enc, int frame_num, FILE *output) { - if (!enc->subtitles) { - return 1; // No subtitles to process - } +// Write all subtitles upfront in SSF-TC format (called before first frame) +static int write_all_subtitles_tc(tav_encoder_t *enc, FILE *output) { + if (!enc->subtitles) return 0; int bytes_written = 0; + int subtitle_count = 0; - // Check if we need to show a new subtitle - if (!enc->subtitle_visible) { - subtitle_entry_t *sub = enc->current_subtitle; - if (!sub) sub = enc->subtitles; // Start from beginning if not set - - // Find next subtitle to show - while (sub && sub->start_frame <= frame_num) { - if (sub->end_frame > frame_num) { - // This subtitle should be shown - if (sub != enc->current_subtitle) { - enc->current_subtitle = sub; - enc->subtitle_visible = 1; - bytes_written += write_subtitle_packet(output, 0, 0x01, sub->text); - if (enc->verbose) { - printf("Frame %d: Showing subtitle: %.50s%s\n", - frame_num, sub->text, strlen(sub->text) > 50 ? "..." : ""); - } - } - break; - } - sub = sub->next; - } + // Convert frame timing to nanoseconds + // For NTSC: frame_time = 1001000000 / 30000 nanoseconds + // For others: frame_time = 1e9 / fps nanoseconds + uint64_t frame_time_ns; + if (enc->is_ntsc_framerate) { + frame_time_ns = 1001000000ULL / 30000ULL; // NTSC: 30000/1001 fps + } else { + frame_time_ns = (uint64_t)(1000000000.0 / enc->fps); } - // Check if we need to hide current subtitle - if (enc->subtitle_visible && enc->current_subtitle) { - if (frame_num >= enc->current_subtitle->end_frame) { - enc->subtitle_visible = 0; - bytes_written += write_subtitle_packet(output, 0, 0x02, NULL); - if (enc->verbose) { - printf("Frame %d: Hiding subtitle\n", frame_num); - } + // Iterate through all subtitles and write them with timecodes + subtitle_entry_t *sub = enc->subtitles; + while (sub) { + // Calculate timecodes for show and hide events + uint64_t show_timecode; + uint64_t hide_timecode; + + if (enc->is_ntsc_framerate) { + // NTSC: time = frame * 1001000000 / 30000 + show_timecode = ((uint64_t)sub->start_frame * 1001000000ULL) / 30000ULL; + hide_timecode = ((uint64_t)sub->end_frame * 1001000000ULL) / 30000ULL; + } else { + show_timecode = (uint64_t)sub->start_frame * frame_time_ns; + hide_timecode = (uint64_t)sub->end_frame * frame_time_ns; } + + // Write show subtitle event + bytes_written += write_subtitle_packet_tc(output, 0, 0x01, sub->text, show_timecode); + + // Write hide subtitle event + bytes_written += write_subtitle_packet_tc(output, 0, 0x02, NULL, hide_timecode); + + subtitle_count++; + if (enc->verbose) { + printf("SSF-TC: Subtitle %d: show at %.3fs, hide at %.3fs: %.50s%s\n", + subtitle_count, + show_timecode / 1000000000.0, + hide_timecode / 1000000000.0, + sub->text, strlen(sub->text) > 50 ? "..." : ""); + } + + sub = sub->next; + } + + if (enc->verbose && subtitle_count > 0) { + printf("Wrote %d SSF-TC subtitle events (%d bytes)\n", subtitle_count * 2, bytes_written); } return bytes_written; @@ -10330,6 +10354,11 @@ int main(int argc, char *argv[]) { } } + // Write all subtitles upfront in SSF-TC format (before first frame) + if (enc->subtitles) { + write_all_subtitles_tc(enc, enc->output_fp); + } + if (enc->output_fps != enc->fps) { printf("Frame rate conversion enabled: %d fps output\n", enc->output_fps); } @@ -10544,8 +10573,8 @@ int main(int argc, char *argv[]) { // Process audio for this frame process_audio(enc, true_frame_count, enc->output_fp); - // Process subtitles for this frame - process_subtitles(enc, true_frame_count, enc->output_fp); + // Note: Subtitles are now written upfront in SSF-TC format (see write_all_subtitles_tc) + // process_subtitles() is no longer called here } if (enc->enable_temporal_dwt) { @@ -10965,9 +10994,9 @@ int main(int argc, char *argv[]) { // Skip when temporal DWT is enabled (audio handled in GOP flush) if (!enc->enable_temporal_dwt && enc->is_ntsc_framerate && (frame_count % 1000 == 500)) { true_frame_count++; - // Process audio and subtitles for the duplicated frame to maintain sync + // Process audio for the duplicated frame to maintain sync process_audio(enc, true_frame_count, enc->output_fp); - process_subtitles(enc, true_frame_count, enc->output_fp); + // Note: Subtitles are now written upfront in SSF-TC format (see write_all_subtitles_tc) uint8_t sync_packet_ntsc = TAV_PACKET_SYNC_NTSC; fwrite(&sync_packet_ntsc, 1, 1, enc->output_fp); diff --git a/video_encoder/encoder_tev.c b/video_encoder/encoder_tev.c index 0e70242..3bb18cf 100644 --- a/video_encoder/encoder_tev.c +++ b/video_encoder/encoder_tev.c @@ -33,7 +33,7 @@ #define TEV_PACKET_IFRAME 0x10 // Intra frame (keyframe) #define TEV_PACKET_PFRAME 0x11 // Predicted frame #define TEV_PACKET_AUDIO_MP2 0x20 // MP2 audio -#define TEV_PACKET_SUBTITLE 0x30 // Subtitle packet +#define TEV_PACKET_SUBTITLE_TC 0x31 // Subtitle packet with timecode (SSF-TC format) #define TEV_PACKET_SYNC 0xFF // Sync packet // Utility macros @@ -1834,71 +1834,86 @@ static void free_subtitle_list(subtitle_entry_t *list) { } } -// Write subtitle packet to output -static int write_subtitle_packet(FILE *output, uint32_t index, uint8_t opcode, const char *text) { - // Calculate packet size +// Write SSF-TC subtitle packet to output +static int write_subtitle_packet_tc(FILE *output, uint32_t index, uint8_t opcode, const char *text, uint64_t timecode_ns) { + // Calculate packet size: index (3 bytes) + timecode (8 bytes) + opcode (1 byte) + text + null terminator size_t text_len = text ? strlen(text) : 0; - size_t packet_size = 3 + 1 + text_len + 1; // index (3 bytes) + opcode + text + null terminator - + size_t packet_size = 3 + 8 + 1 + text_len + 1; + // Write packet type and size - uint8_t packet_type = TEV_PACKET_SUBTITLE; + uint8_t packet_type = TEV_PACKET_SUBTITLE_TC; fwrite(&packet_type, 1, 1, output); fwrite(&packet_size, 4, 1, output); - - // Write subtitle packet data + + // Write subtitle index (24-bit, little-endian) uint8_t index_bytes[3]; index_bytes[0] = index & 0xFF; index_bytes[1] = (index >> 8) & 0xFF; index_bytes[2] = (index >> 16) & 0xFF; fwrite(index_bytes, 1, 3, output); - + + // Write timecode (64-bit, little-endian) + uint8_t timecode_bytes[8]; + for (int i = 0; i < 8; i++) { + timecode_bytes[i] = (timecode_ns >> (i * 8)) & 0xFF; + } + fwrite(timecode_bytes, 1, 8, output); + + // Write opcode fwrite(&opcode, 1, 1, output); - + + // Write text if present if (text && text_len > 0) { fwrite(text, 1, text_len, output); } - + // Write null terminator uint8_t null_term = 0x00; fwrite(&null_term, 1, 1, output); - + return packet_size + 5; // packet_size + packet_type + size field } -// Process subtitles for the current frame -static int process_subtitles(tev_encoder_t *enc, int frame_num, FILE *output) { +// Write all subtitles upfront in SSF-TC format (called before first frame) +static int write_all_subtitles_tc(tev_encoder_t *enc, FILE *output) { if (!enc->has_subtitles) return 0; int bytes_written = 0; - - // Check if any subtitles need to be shown at this frame - subtitle_entry_t *sub = enc->current_subtitle; - while (sub && sub->start_frame <= frame_num) { - if (sub->start_frame == frame_num) { - // Show subtitle - bytes_written += write_subtitle_packet(output, 0, 0x01, sub->text); - if (enc->verbose) { - printf("Frame %d: Showing subtitle: %.50s%s\n", - frame_num, sub->text, strlen(sub->text) > 50 ? "..." : ""); - } + int subtitle_count = 0; + + // Convert frame timing to nanoseconds + // Frame time = 1e9 / fps nanoseconds + uint64_t frame_time_ns = (uint64_t)(1000000000.0 / enc->output_fps); + + // Iterate through all subtitles and write them with timecodes + subtitle_entry_t *sub = enc->subtitle_list; + while (sub) { + // Calculate timecodes for show and hide events + uint64_t show_timecode = (uint64_t)sub->start_frame * frame_time_ns; + uint64_t hide_timecode = (uint64_t)sub->end_frame * frame_time_ns; + + // Write show subtitle event + bytes_written += write_subtitle_packet_tc(output, 0, 0x01, sub->text, show_timecode); + + // Write hide subtitle event + bytes_written += write_subtitle_packet_tc(output, 0, 0x02, NULL, hide_timecode); + + subtitle_count++; + if (enc->verbose) { + printf("SSF-TC: Subtitle %d: show at %.3fs, hide at %.3fs: %.50s%s\n", + subtitle_count, + show_timecode / 1000000000.0, + hide_timecode / 1000000000.0, + sub->text, strlen(sub->text) > 50 ? "..." : ""); } - - if (sub->end_frame == frame_num) { - // Hide subtitle - bytes_written += write_subtitle_packet(output, 0, 0x02, NULL); - if (enc->verbose) { - printf("Frame %d: Hiding subtitle\n", frame_num); - } - } - - // Move to next subtitle if we're past the end of current one - if (sub->end_frame <= frame_num) { - enc->current_subtitle = sub->next; - } - + sub = sub->next; } - + + if (enc->verbose && subtitle_count > 0) { + printf("Wrote %d SSF-TC subtitle events (%d bytes)\n", subtitle_count * 2, bytes_written); + } + return bytes_written; } @@ -2868,6 +2883,12 @@ int main(int argc, char *argv[]) { // Write TEV header write_tev_header(output, enc); + + // Write all subtitles upfront in SSF-TC format (before first frame) + if (enc->has_subtitles) { + write_all_subtitles_tc(enc, output); + } + gettimeofday(&enc->start_time, NULL); printf("Encoding video with %s 4:2:0 format...\n", enc->ictcp_mode ? "ICtCp" : "YCoCg-R"); @@ -2962,8 +2983,8 @@ int main(int argc, char *argv[]) { // Process audio for this frame process_audio(enc, frame_count, output); - // Process subtitles for this frame - process_subtitles(enc, frame_count, output); + // Note: Subtitles are now written upfront in SSF-TC format (see write_all_subtitles_tc) + // process_subtitles() is no longer called here // Encode frame // Pass field parity for interlaced mode, -1 for progressive mode diff --git a/video_encoder/tav_inspector.c b/video_encoder/tav_inspector.c index 2b49907..8eb8aa6 100644 --- a/video_encoder/tav_inspector.c +++ b/video_encoder/tav_inspector.c @@ -26,8 +26,8 @@ #define TAV_PACKET_AUDIO_MP2 0x20 #define TAV_PACKET_AUDIO_PCM8 0x21 #define TAV_PACKET_AUDIO_TAD 0x24 -#define TAV_PACKET_SUBTITLE 0x30 -#define TAV_PACKET_SUBTITLE_KAR 0x31 +#define TAV_PACKET_SUBTITLE 0x30 // Legacy SSF (frame-locked) +#define TAV_PACKET_SUBTITLE_TC 0x31 // SSF-TC (timecode-based) #define TAV_PACKET_AUDIO_TRACK 0x40 #define TAV_PACKET_VIDEO_CH2_I 0x70 #define TAV_PACKET_VIDEO_CH2_P 0x71 @@ -119,8 +119,8 @@ const char* get_packet_type_name(uint8_t type) { case TAV_PACKET_AUDIO_MP2: return "AUDIO MP2"; case TAV_PACKET_AUDIO_PCM8: return "AUDIO PCM8 (zstd)"; case TAV_PACKET_AUDIO_TAD: return "AUDIO TAD (zstd)"; - case TAV_PACKET_SUBTITLE: return "SUBTITLE (Simple)"; - case TAV_PACKET_SUBTITLE_KAR: return "SUBTITLE (Karaoke)"; + case TAV_PACKET_SUBTITLE: return "SUBTITLE (SSF frame-locked)"; + case TAV_PACKET_SUBTITLE_TC: return "SUBTITLE (SSF-TC timecoded)"; case TAV_PACKET_AUDIO_TRACK: return "AUDIO TRACK (Separate MP2)"; case TAV_PACKET_EXIF: return "METADATA (EXIF)"; case TAV_PACKET_ID3V1: return "METADATA (ID3v1)"; @@ -151,7 +151,7 @@ int should_display_packet(uint8_t type, display_options_t *opts) { (type >= 0x70 && type <= 0x7F))) return 1; if (opts->show_audio && (type == TAV_PACKET_AUDIO_MP2 || type == TAV_PACKET_AUDIO_PCM8 || type == TAV_PACKET_AUDIO_TAD || type == TAV_PACKET_AUDIO_TRACK)) return 1; - if (opts->show_subtitles && (type == TAV_PACKET_SUBTITLE || type == TAV_PACKET_SUBTITLE_KAR)) return 1; + if (opts->show_subtitles && (type == TAV_PACKET_SUBTITLE || type == TAV_PACKET_SUBTITLE_TC)) return 1; if (opts->show_timecode && type == TAV_PACKET_TIMECODE) return 1; if (opts->show_metadata && (type >= 0xE0 && type <= 0xE4)) return 1; if (opts->show_sync && (type == TAV_PACKET_SYNC || type == TAV_PACKET_SYNC_NTSC)) return 1; @@ -160,7 +160,7 @@ int should_display_packet(uint8_t type, display_options_t *opts) { return 0; } -void print_subtitle_packet(FILE *fp, uint32_t size, int is_karaoke, int verbose) { +void print_subtitle_packet(FILE *fp, uint32_t size, int is_timecoded, int verbose) { if (!verbose) { fseek(fp, size, SEEK_CUR); return; @@ -174,10 +174,26 @@ void print_subtitle_packet(FILE *fp, uint32_t size, int is_karaoke, int verbose) index |= (byte << (i * 8)); } + // Read timecode if SSF-TC (0x31) + uint64_t timecode_ns = 0; + int header_size = 4; // 3 bytes index + 1 byte opcode + if (is_timecoded) { + uint8_t timecode_bytes[8]; + if (fread(timecode_bytes, 1, 8, fp) != 8) return; + for (int i = 0; i < 8; i++) { + timecode_ns |= ((uint64_t)timecode_bytes[i]) << (i * 8); + } + header_size += 8; // Add 8 bytes for timecode + } + uint8_t opcode; if (fread(&opcode, 1, 1, fp) != 1) return; - printf(" [Index=%u, Opcode=0x%02X", index, opcode); + printf(" [Index=%u", index); + if (is_timecoded) { + printf(", Time=%.3fs", timecode_ns / 1000000000.0); + } + printf(", Opcode=0x%02X", opcode); switch (opcode) { case 0x01: printf(" (SHOW)"); break; @@ -193,7 +209,7 @@ void print_subtitle_packet(FILE *fp, uint32_t size, int is_karaoke, int verbose) printf("]"); // Read and display text content for SHOW commands - int remaining = size - 4; // Already read 3 (index) + 1 (opcode) + int remaining = size - header_size; // Already read index + timecode (if any) + opcode if ((opcode == 0x01 || (opcode >= 0x10 && opcode <= 0x2F) || (opcode >= 0x30 && opcode <= 0x41)) && remaining > 0) { char *text = malloc(remaining + 1); if (text && fread(text, 1, remaining, fp) == remaining) { @@ -788,14 +804,14 @@ static const char* VERDESC[] = {"null", "YCoCg tiled, uniform", "ICtCp tiled, un } case TAV_PACKET_SUBTITLE: - case TAV_PACKET_SUBTITLE_KAR: { + case TAV_PACKET_SUBTITLE_TC: { stats.subtitle_count++; uint32_t size; if (fread(&size, sizeof(uint32_t), 1, fp) != 1) break; if (!opts.summary_only && display) { printf(" - size=%u bytes", size); - print_subtitle_packet(fp, size, packet_type == TAV_PACKET_SUBTITLE_KAR, opts.verbose); + print_subtitle_packet(fp, size, packet_type == TAV_PACKET_SUBTITLE_TC, opts.verbose); } else { fseek(fp, size, SEEK_CUR); }