curioustorvald/tsvm
1
0
Fork 0
mirror of https://github.com/curioustorvald/tsvm.git synced 2026-06-12 23:54:04 +09:00
Code Issues Packages Projects Releases Wiki Activity

Changed video format; added TEV version 3 (XYB colour space)

Browse Source
This commit is contained in:
minjaesong
2025-08-26 22:17:45 +09:00
parent 6d982a9786
commit 33e77e378e
7 changed files with 2485 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
assets/disk0/tvdos/bin/playtev.js
View File

@@ -7,7 +7,8 @@ const WIDTH = 560
const HEIGHT = 448 const HEIGHT = 448
const BLOCK_SIZE = 16 // 16x16 blocks for YCoCg-R const BLOCK_SIZE = 16 // 16x16 blocks for YCoCg-R
const TEV_MAGIC = [0x1F, 0x54, 0x53, 0x56, 0x4D, 0x54, 0x45, 0x56] // "\x1FTSVM TEV" const TEV_MAGIC = [0x1F, 0x54, 0x53, 0x56, 0x4D, 0x54, 0x45, 0x56] // "\x1FTSVM TEV"
const TEV_VERSION = 2 // YCoCg-R version const TEV_VERSION_YCOCG = 2 // YCoCg-R version
const TEV_VERSION_XYB = 3 // XYB version
const SND_BASE_ADDR = audio.getBaseAddr() const SND_BASE_ADDR = audio.getBaseAddr()
const pcm = require("pcm") const pcm = require("pcm")
const MP2_FRAME_SIZE = [144,216,252,288,360,432,504,576,720,864,1008,1152,1440,1728] const MP2_FRAME_SIZE = [144,216,252,288,360,432,504,576,720,864,1008,1152,1440,1728]
@@ -35,11 +36,6 @@ let notifHideTimer = 0
const NOTIF_SHOWUPTIME = 3000000000 const NOTIF_SHOWUPTIME = 3000000000
let [cy, cx] = con.getyx() let [cy, cx] = con.getyx()
if (interactive) {
con.move(1,1)
println("Push and hold Backspace to exit")
}
let seqreadserial = require("seqread") let seqreadserial = require("seqread")
let seqreadtape = require("seqreadtape") let seqreadtape = require("seqreadtape")
let seqread = undefined let seqread = undefined
@@ -285,11 +281,17 @@ if (!magicMatching) {
// Read header // Read header
let version = seqread.readOneByte() let version = seqread.readOneByte()
if (version !== TEV_VERSION) { if (version !== TEV_VERSION_YCOCG && version !== TEV_VERSION_XYB) {
println(`Unsupported TEV version: ${version} (expected ${TEV_VERSION})`) println(`Unsupported TEV version: ${version} (expected ${TEV_VERSION_YCOCG} for YCoCg-R or ${TEV_VERSION_XYB} for XYB)`)
return 1 return 1
} }
let colorSpace = (version === TEV_VERSION_XYB) ? "XYB" : "YCoCg-R"
if (interactive) {
con.move(1,1)
println(`Push and hold Backspace to exit | TEV Format ${version} (${colorSpace})`)
}
let width = seqread.readShort() let width = seqread.readShort()
let height = seqread.readShort() let height = seqread.readShort()
let fps = seqread.readOneByte() let fps = seqread.readOneByte()
@@ -353,6 +355,8 @@ let biasTime = 0
const BIAS_LIGHTING_MIN = 1.0 / 16.0 const BIAS_LIGHTING_MIN = 1.0 / 16.0
let oldBgcol = [BIAS_LIGHTING_MIN, BIAS_LIGHTING_MIN, BIAS_LIGHTING_MIN] let oldBgcol = [BIAS_LIGHTING_MIN, BIAS_LIGHTING_MIN, BIAS_LIGHTING_MIN]
let notifHidden = false
function getRGBfromScr(x, y) { function getRGBfromScr(x, y) {
let offset = y * WIDTH + x let offset = y * WIDTH + x
let rg = sys.peek(-1048577 - offset) let rg = sys.peek(-1048577 - offset)
@@ -425,18 +429,6 @@ try {
} else if (packetType == TEV_PACKET_IFRAME || packetType == TEV_PACKET_PFRAME) { } else if (packetType == TEV_PACKET_IFRAME || packetType == TEV_PACKET_PFRAME) {
// Video frame packet (always includes rate control factor) // Video frame packet (always includes rate control factor)
let payloadLen = seqread.readInt() let payloadLen = seqread.readInt()
// Always read rate control factor (4 bytes, little-endian float)
let rateFactorBytes = seqread.readBytes(4)
let view = new DataView(new ArrayBuffer(4))
for (let i = 0; i < 4; i++) {
view.setUint8(i, sys.peek(rateFactorBytes + i))
}
let rateControlFactor = view.getFloat32(0, true) // true = little-endian
//serial.println(`rateControlFactor = ${rateControlFactor}`)
sys.free(rateFactorBytes)
payloadLen -= 4 // Subtract rate factor size from payload
let compressedPtr = seqread.readBytes(payloadLen) let compressedPtr = seqread.readBytes(payloadLen)
updateDataRateBin(payloadLen) updateDataRateBin(payloadLen)
@@ -469,10 +461,10 @@ try {
continue continue
} }
// Hardware-accelerated TEV YCoCg-R decoding to RGB buffers (with rate control factor) // Hardware-accelerated TEV decoding to RGB buffers (YCoCg-R or XYB based on version)
try { try {
let decodeStart = sys.nanoTime() let decodeStart = sys.nanoTime()
graphics.tevDecode(blockDataPtr, CURRENT_RGB_ADDR, PREV_RGB_ADDR, width, height, quality, debugMotionVectors, rateControlFactor) graphics.tevDecode(blockDataPtr, CURRENT_RGB_ADDR, PREV_RGB_ADDR, width, height, quality, debugMotionVectors, version)
decodeTime = (sys.nanoTime() - decodeStart) / 1000000.0 // Convert to milliseconds decodeTime = (sys.nanoTime() - decodeStart) / 1000000.0 // Convert to milliseconds
// Upload RGB buffer to display framebuffer with dithering // Upload RGB buffer to display framebuffer with dithering
@@ -487,7 +479,7 @@ try {
audioFired = true audioFired = true
} }
} catch (e) { } catch (e) {
serial.println(`Frame ${frameCount}: Hardware YCoCg-R decode failed: ${e}`) serial.println(`Frame ${frameCount}: Hardware ${colorSpace} decode failed: ${e}`)
} }
sys.free(compressedPtr) sys.free(compressedPtr)
@@ -531,6 +523,12 @@ try {
// Simple progress display // Simple progress display
if (interactive) { if (interactive) {
notifHideTimer += (t2 - t1)
if (!notifHidden && notifHideTimer > (NOTIF_SHOWUPTIME + FRAME_TIME)) {
con.clear()
notifHidden = true
}
con.move(31, 1) con.move(31, 1)
graphics.setTextFore(161) graphics.setTextFore(161)
print(`Frame: ${frameCount}/${totalFrames} (${((frameCount / akku2 * 100)|0) / 100}f) `) print(`Frame: ${frameCount}/${totalFrames} (${((frameCount / akku2 * 100)|0) / 100}f) `)
@@ -544,7 +542,7 @@ try {
} }
} }
catch (e) { catch (e) {
printerrln(`TEV YCoCg-R decode error: ${e}`) printerrln(`TEV ${colorSpace} decode error: ${e}`)
errorlevel = 1 errorlevel = 1
} }
finally { finally {

17
terranmon.txt
View File

@@ -683,6 +683,8 @@ DCT-based compression, motion compensation, and efficient temporal coding.
- Version 2.1: Added Rate Control Factor to all video packets (breaking change) - Version 2.1: Added Rate Control Factor to all video packets (breaking change)
* Enables bitrate-constrained encoding alongside quality modes * Enables bitrate-constrained encoding alongside quality modes
* All video frames now include 4-byte rate control factor after payload size * All video frames now include 4-byte rate control factor after payload size
- Version 3.0: Additional support of XYB Colour space
* Increased encoding efficiency, decreased decoding performance
# File Structure # File Structure
\x1F T S V M T E V \x1F T S V M T E V
@@ -692,16 +694,15 @@ DCT-based compression, motion compensation, and efficient temporal coding.
[PACKET 2] [PACKET 2]
... ...
## Header (24 bytes) ## Header (20 bytes)
uint8 Magic[8]: "\x1FTSVM TEV" uint8 Magic[8]: "\x1FTSVM TEV"
uint8 Version: 2 uint8 Version: 2 or 3
uint8 Flags: bit 0 = has audio
uint16 Width: video width in pixels uint16 Width: video width in pixels
uint16 Height: video height in pixels uint16 Height: video height in pixels
uint16 FPS: frames per second uint8 FPS: frames per second
uint32 Total Frames: number of video frames uint32 Total Frames: number of video frames
uint8 Quality: quantization quality (0-4, higher = better) uint8 Quality: quantization quality (0-4, higher = better)
byte[5] Reserved uint8 Flags: bit 0 = has audio
## Packet Types ## Packet Types
0x10: I-frame (intra-coded frame) 0x10: I-frame (intra-coded frame)
@@ -713,7 +714,6 @@ DCT-based compression, motion compensation, and efficient temporal coding.
## Video Packet Structure ## Video Packet Structure
uint8 Packet Type uint8 Packet Type
uint32 Compressed Size (includes rate control factor size) uint32 Compressed Size (includes rate control factor size)
float Rate Control Factor (4 bytes, little-endian)
* Gzip-compressed Block Data * Gzip-compressed Block Data
## Block Data (per 16x16 block) ## Block Data (per 16x16 block)
@@ -724,6 +724,7 @@ DCT-based compression, motion compensation, and efficient temporal coding.
0x03 = MOTION (motion vector only) 0x03 = MOTION (motion vector only)
int16 Motion Vector X ("capable of" 1/4 pixel precision, integer precision for now) int16 Motion Vector X ("capable of" 1/4 pixel precision, integer precision for now)
int16 Motion Vector Y ("capable of" 1/4 pixel precision, integer precision for now) int16 Motion Vector Y ("capable of" 1/4 pixel precision, integer precision for now)
float32 Rate Control Factor (4 bytes, little-endian)
uint16 Coded Block Pattern (which 8x8 have non-zero coeffs) uint16 Coded Block Pattern (which 8x8 have non-zero coeffs)
int16[256] DCT Coefficients Y int16[256] DCT Coefficients Y
int16[64] DCT Coefficients Co (subsampled by two) int16[64] DCT Coefficients Co (subsampled by two)
@@ -731,7 +732,7 @@ DCT-based compression, motion compensation, and efficient temporal coding.
For SKIP and MOTION mode, DCT coefficients are filled with zero For SKIP and MOTION mode, DCT coefficients are filled with zero
## DCT Quantization and Rate Control ## DCT Quantization and Rate Control
TEV uses 8 quality levels (0=lowest, 7=highest) with progressive quantization TEV uses 5 quality levels (0=lowest, 4=highest) with progressive quantization
tables optimized for perceptual quality. DC coefficients use fixed quantizer tables optimized for perceptual quality. DC coefficients use fixed quantizer
of 8, while AC coefficients are quantized according to quality tables. of 8, while AC coefficients are quantized according to quality tables.

252
tsvm_core/src/net/torvald/tsvm/GraphicsJSR223Delegate.kt
View File

@@ -2,14 +2,15 @@ package net.torvald.tsvm
import com.badlogic.gdx.graphics.Pixmap import com.badlogic.gdx.graphics.Pixmap
import com.badlogic.gdx.math.MathUtils.* import com.badlogic.gdx.math.MathUtils.*
import com.badlogic.gdx.math.MathUtils.PI
import com.badlogic.gdx.math.MathUtils.ceil
import com.badlogic.gdx.math.MathUtils.floor
import com.badlogic.gdx.math.MathUtils.round
import net.torvald.UnsafeHelper import net.torvald.UnsafeHelper
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.toUint import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.toUint
import net.torvald.tsvm.peripheral.GraphicsAdapter import net.torvald.tsvm.peripheral.GraphicsAdapter
import net.torvald.tsvm.peripheral.fmod import net.torvald.tsvm.peripheral.fmod
import kotlin.math.abs import kotlin.math.*
import kotlin.math.cos
import kotlin.math.roundToInt
import kotlin.math.sqrt
class GraphicsJSR223Delegate(private val vm: VM) { class GraphicsJSR223Delegate(private val vm: VM) {
@@ -1605,6 +1606,138 @@ class GraphicsJSR223Delegate(private val vm: VM) {
return ycocgData return ycocgData
} }
// XYB conversion constants from JPEG XL specification
private val XYB_BIAS = 0.00379307325527544933
private val CBRT_BIAS = 0.155954200549248620 // cbrt(XYB_BIAS)
// RGB to LMS mixing coefficients
private val RGB_TO_LMS = arrayOf(
doubleArrayOf(0.3, 0.622, 0.078), // L coefficients
doubleArrayOf(0.23, 0.692, 0.078), // M coefficients
doubleArrayOf(0.24342268924547819, 0.20476744424496821, 0.55180986650955360) // S coefficients
)
// LMS to RGB inverse matrix
private val LMS_TO_RGB = arrayOf(
doubleArrayOf(11.0315669046, -9.8669439081, -0.1646229965),
doubleArrayOf(-3.2541473811, 4.4187703776, -0.1646229965),
doubleArrayOf(-3.6588512867, 2.7129230459, 1.9459282408)
)
// sRGB linearization functions
private fun srgbLinearise(value: Double): Double {
return if (value > 0.04045) {
Math.pow((value + 0.055) / 1.055, 2.4)
} else {
value / 12.92
}
}
private fun srgbUnlinearise(value: Double): Double {
return if (value > 0.0031308) {
1.055 * Math.pow(value, 1.0 / 2.4) - 0.055
} else {
value * 12.92
}
}
// XYB to RGB conversion for hardware decoding
fun tevXybToRGB(yBlock: IntArray, xBlock: IntArray, bBlock: IntArray): IntArray {
val rgbData = IntArray(16 * 16 * 3) // R,G,B for 16x16 pixels
for (py in 0 until 16) {
for (px in 0 until 16) {
val yIdx = py * 16 + px
val y = yBlock[yIdx]
// Get chroma values from subsampled 8x8 blocks (nearest neighbor upsampling)
val xbIdx = (py / 2) * 8 + (px / 2)
val x = xBlock[xbIdx]
val b = bBlock[xbIdx]
// Dequantize from integer ranges
val yVal = (y - 128.0) / 255.0
val xVal = x / 255.0
val bVal = b / 255.0
// XYB to LMS gamma
val lgamma = xVal + yVal
val mgamma = yVal - xVal
val sgamma = bVal
// Remove gamma correction
val lmix = (lgamma + CBRT_BIAS).pow(3.0) - XYB_BIAS
val mmix = (mgamma + CBRT_BIAS).pow(3.0) - XYB_BIAS
val smix = (sgamma + CBRT_BIAS).pow(3.0) - XYB_BIAS
// LMS to linear RGB using inverse matrix
val rLinear = (LMS_TO_RGB[0][0] * lmix + LMS_TO_RGB[0][1] * mmix + LMS_TO_RGB[0][2] * smix).coerceIn(0.0, 1.0)
val gLinear = (LMS_TO_RGB[1][0] * lmix + LMS_TO_RGB[1][1] * mmix + LMS_TO_RGB[1][2] * smix).coerceIn(0.0, 1.0)
val bLinear = (LMS_TO_RGB[2][0] * lmix + LMS_TO_RGB[2][1] * mmix + LMS_TO_RGB[2][2] * smix).coerceIn(0.0, 1.0)
// Convert back to sRGB gamma and 0-255 range
val r = (srgbUnlinearise(rLinear) * 255.0 + 0.5).toInt().coerceIn(0, 255)
val g = (srgbUnlinearise(gLinear) * 255.0 + 0.5).toInt().coerceIn(0, 255)
val bRgb = (srgbUnlinearise(bLinear) * 255.0 + 0.5).toInt().coerceIn(0, 255)
// Store RGB
val baseIdx = (py * 16 + px) * 3
rgbData[baseIdx] = r // R
rgbData[baseIdx + 1] = g // G
rgbData[baseIdx + 2] = bRgb // B
}
}
return rgbData
}
// RGB to XYB conversion for INTER mode residual calculation
fun tevRGBToXyb(rgbBlock: IntArray): IntArray {
val xybData = IntArray(16 * 16 * 3) // Y,X,B for 16x16 pixels
for (py in 0 until 16) {
for (px in 0 until 16) {
val baseIdx = (py * 16 + px) * 3
val r = rgbBlock[baseIdx]
val g = rgbBlock[baseIdx + 1]
val b = rgbBlock[baseIdx + 2]
// Convert RGB to 0-1 range and linearise sRGB
val rNorm = srgbLinearise(r / 255.0)
val gNorm = srgbLinearise(g / 255.0)
val bNorm = srgbLinearise(b / 255.0)
// RGB to LMS mixing with bias
val lmix = RGB_TO_LMS[0][0] * rNorm + RGB_TO_LMS[0][1] * gNorm + RGB_TO_LMS[0][2] * bNorm + XYB_BIAS
val mmix = RGB_TO_LMS[1][0] * rNorm + RGB_TO_LMS[1][1] * gNorm + RGB_TO_LMS[1][2] * bNorm + XYB_BIAS
val smix = RGB_TO_LMS[2][0] * rNorm + RGB_TO_LMS[2][1] * gNorm + RGB_TO_LMS[2][2] * bNorm + XYB_BIAS
// Apply gamma correction (cube root)
val lgamma = lmix.pow(1.0 / 3.0) - CBRT_BIAS
val mgamma = mmix.pow(1.0 / 3.0) - CBRT_BIAS
val sgamma = smix.pow(1.0 / 3.0) - CBRT_BIAS
// LMS to XYB transformation
val xVal = (lgamma - mgamma) / 2.0
val yVal = (lgamma + mgamma) / 2.0
val bVal = sgamma
// Quantize to integer ranges suitable for TEV
val yQuant = (yVal * 255.0 + 128.0).toInt().coerceIn(0, 255) // Y: 0-255 (like YCoCg Y)
val xQuant = (xVal * 255.0).toInt().coerceIn(-128, 127) // X: -128 to +127 (like Co)
val bQuant = (bVal * 255.0).toInt().coerceIn(-128, 127) // B: -128 to +127 (like Cg, aggressively quantized)
// Store XYB values
val yIdx = py * 16 + px
xybData[yIdx * 3] = yQuant // Y
xybData[yIdx * 3 + 1] = xQuant // X
xybData[yIdx * 3 + 2] = bQuant // B
}
}
return xybData
}
/** /**
* Hardware-accelerated TEV frame decoder for YCoCg-R 4:2:0 format * Hardware-accelerated TEV frame decoder for YCoCg-R 4:2:0 format
@@ -1620,7 +1753,7 @@ class GraphicsJSR223Delegate(private val vm: VM) {
*/ */
fun tevDecode(blockDataPtr: Long, currentRGBAddr: Long, prevRGBAddr: Long, fun tevDecode(blockDataPtr: Long, currentRGBAddr: Long, prevRGBAddr: Long,
width: Int, height: Int, quality: Int, debugMotionVectors: Boolean = false, width: Int, height: Int, quality: Int, debugMotionVectors: Boolean = false,
rateControlFactor: Float = 1.0f) { tevVersion: Int = 2) {
val blocksX = (width + 15) / 16 // 16x16 blocks now val blocksX = (width + 15) / 16 // 16x16 blocks now
val blocksY = (height + 15) / 16 val blocksY = (height + 15) / 16
@@ -1630,21 +1763,9 @@ class GraphicsJSR223Delegate(private val vm: VM) {
val quantCGmult = QUANT_MULT_CG[quality] val quantCGmult = QUANT_MULT_CG[quality]
// Apply rate control factor to quantization tables (if not ~1.0, skip optimization) // Apply rate control factor to quantization tables (if not ~1.0, skip optimization)
val quantTableY = if (rateControlFactor in 0.999f..1.001f) { val quantTableY = QUANT_TABLE_Y.map { it * quantYmult }.toIntArray()
QUANT_TABLE_Y.map { it * quantYmult }.toIntArray() val quantTableCo = QUANT_TABLE_C.map { it * quantCOmult }.toIntArray()
} else { val quantTableCg = QUANT_TABLE_C.map { it * quantCGmult }.toIntArray()
QUANT_TABLE_Y.map { (it * quantYmult * rateControlFactor).toInt() }.toIntArray()
}
val quantTableCo = if (rateControlFactor in 0.999f..1.001f) {
QUANT_TABLE_C.map { it * quantCOmult }.toIntArray()
} else {
QUANT_TABLE_C.map { (it * quantCOmult * rateControlFactor).toInt() }.toIntArray()
}
val quantTableCg = if (rateControlFactor in 0.999f..1.001f) {
QUANT_TABLE_C.map { it * quantCGmult }.toIntArray()
} else {
QUANT_TABLE_C.map { (it * quantCGmult * rateControlFactor).toInt() }.toIntArray()
}
var readPtr = blockDataPtr var readPtr = blockDataPtr
@@ -1664,7 +1785,11 @@ class GraphicsJSR223Delegate(private val vm: VM) {
((vm.peek(readPtr + 2)!!.toUint()) shl 8)).toShort().toInt() ((vm.peek(readPtr + 2)!!.toUint()) shl 8)).toShort().toInt()
val mvY = ((vm.peek(readPtr + 3)!!.toUint()) or val mvY = ((vm.peek(readPtr + 3)!!.toUint()) or
((vm.peek(readPtr + 4)!!.toUint()) shl 8)).toShort().toInt() ((vm.peek(readPtr + 4)!!.toUint()) shl 8)).toShort().toInt()
readPtr += 7 // Skip CBP field val rateControlFactor = Float.fromBits((vm.peek(readPtr + 5)!!.toUint()) or
((vm.peek(readPtr + 6)!!.toUint()) shl 8) or
((vm.peek(readPtr + 7)!!.toUint()) shl 16) or
((vm.peek(readPtr + 8)!!.toUint()) shl 24))
readPtr += 11 // Skip CBP field
when (mode) { when (mode) {
@@ -1784,8 +1909,12 @@ class GraphicsJSR223Delegate(private val vm: VM) {
val coBlock = tevIdct8x8_fast(coCoeffs, quantTableCo, true) val coBlock = tevIdct8x8_fast(coCoeffs, quantTableCo, true)
val cgBlock = tevIdct8x8_fast(cgCoeffs, quantTableCg, true) val cgBlock = tevIdct8x8_fast(cgCoeffs, quantTableCg, true)
// Convert YCoCg-R to RGB // Convert to RGB (YCoCg-R for v2, XYB for v3)
val rgbData = tevYcocgToRGB(yBlock, coBlock, cgBlock) val rgbData = if (tevVersion == 3) {
tevXybToRGB(yBlock, coBlock, cgBlock) // XYB format (v3)
} else {
tevYcocgToRGB(yBlock, coBlock, cgBlock) // YCoCg-R format (v2)
}
// Store RGB data to frame buffer (complete replacement) // Store RGB data to frame buffer (complete replacement)
for (dy in 0 until 16) { for (dy in 0 until 16) {
@@ -1943,8 +2072,12 @@ class GraphicsJSR223Delegate(private val vm: VM) {
} }
} }
// Step 4: Convert final YCoCg-R to RGB // Step 4: Convert final data to RGB (YCoCg-R for v2, XYB for v3)
val finalRgb = tevYcocgToRGB(finalY, finalCo, finalCg) val finalRgb = if (tevVersion == 3) {
tevXybToRGB(finalY, finalCo, finalCg) // XYB format (v3)
} else {
tevYcocgToRGB(finalY, finalCo, finalCg) // YCoCg-R format (v2)
}
// Step 5: Store final RGB data to frame buffer // Step 5: Store final RGB data to frame buffer
for (dy in 0 until 16) { for (dy in 0 until 16) {
@@ -2002,71 +2135,6 @@ class GraphicsJSR223Delegate(private val vm: VM) {
} }
} }
// YCoCg-R transform for 16x16 Y blocks and 8x8 chroma blocks (4:2:0 subsampling)
fun blockEncodeToYCoCgR16x16(blockX: Int, blockY: Int, srcPtr: Int, width: Int, height: Int): List<IntArray> {
val yBlock = IntArray(16 * 16) // 16x16 Y
val coBlock = IntArray(8 * 8) // 8x8 Co (subsampled)
val cgBlock = IntArray(8 * 8) // 8x8 Cg (subsampled)
val incVec = if (srcPtr >= 0) 1L else -1L
// Process 16x16 Y block
for (py in 0 until 16) {
for (px in 0 until 16) {
val ox = blockX * 16 + px
val oy = blockY * 16 + py
if (ox < width && oy < height) {
val offset = 3 * (oy * width + ox)
val r = vm.peek(srcPtr + offset * incVec)!!.toUint()
val g = vm.peek(srcPtr + (offset + 1) * incVec)!!.toUint()
val b = vm.peek(srcPtr + (offset + 2) * incVec)!!.toUint()
// YCoCg-R transform
val co = r - b
val tmp = b + (co / 2)
val cg = g - tmp
val y = tmp + (cg / 2)
yBlock[py * 16 + px] = y
}
}
}
// Process 8x8 Co/Cg blocks with 4:2:0 subsampling (average 2x2 pixels)
for (py in 0 until 8) {
for (px in 0 until 8) {
var coSum = 0
var cgSum = 0
var count = 0
// Average 2x2 block of pixels for chroma subsampling
for (dy in 0 until 2) {
for (dx in 0 until 2) {
val ox = blockX * 16 + px * 2 + dx
val oy = blockY * 16 + py * 2 + dy
if (ox < width && oy < height) {
val offset = 3 * (oy * width + ox)
val r = vm.peek(srcPtr + offset * incVec)!!.toUint()
val g = vm.peek(srcPtr + (offset + 1) * incVec)!!.toUint()
val b = vm.peek(srcPtr + (offset + 2) * incVec)!!.toUint()
val co = r - b
val tmp = b + (co / 2)
val cg = g - tmp
coSum += co
cgSum += cg
count++
}
}
}
if (count > 0) {
coBlock[py * 8 + px] = coSum / count
cgBlock[py * 8 + px] = cgSum / count
}
}
}
return listOf(yBlock, coBlock, cgBlock)
}
} }

40
video_encoder/Makefile
View File

@@ -5,26 +5,37 @@ CC = gcc
CFLAGS = -std=c99 -Wall -Wextra -O2 -D_GNU_SOURCE CFLAGS = -std=c99 -Wall -Wextra -O2 -D_GNU_SOURCE
LIBS = -lm -lz LIBS = -lm -lz
# Source files # Source files and targets
SOURCES = encoder_tev.c SOURCES = encoder_tev.c encoder_tev_xyb.c
TARGET = encoder_tev TARGETS = encoder_tev encoder_tev_xyb
# Build encoder # Build all encoders
$(TARGET): $(SOURCES) all: $(TARGETS)
rm -f $(TARGET)
# Build main encoder
encoder_tev: encoder_tev.c
rm -f encoder_tev
$(CC) $(CFLAGS) -o $@ $< $(LIBS) $(CC) $(CFLAGS) -o $@ $< $(LIBS)
# Build XYB encoder
encoder_tev_xyb: encoder_tev_xyb.c
rm -f encoder_tev_xyb
$(CC) $(CFLAGS) -o $@ $< $(LIBS)
# Default target
$(TARGETS): all
# Build with debug symbols # Build with debug symbols
debug: CFLAGS += -g -DDEBUG debug: CFLAGS += -g -DDEBUG
debug: $(TARGET) debug: $(TARGETS)
# Clean build artifacts # Clean build artifacts
clean: clean:
rm -f $(TARGET) rm -f $(TARGETS)
# Install (copy to PATH) # Install (copy to PATH)
install: $(TARGET) install: $(TARGETS)
cp $(TARGET) /usr/local/bin/ cp $(TARGETS) /usr/local/bin/
# Check for required dependencies # Check for required dependencies
check-deps: check-deps:
@@ -38,7 +49,9 @@ help:
@echo "TSVM Enhanced Video (TEV) Encoder" @echo "TSVM Enhanced Video (TEV) Encoder"
@echo "" @echo ""
@echo "Targets:" @echo "Targets:"
@echo " encoder_tev - Build the encoder (default)" @echo " all - Build both encoders (default)"
@echo " encoder_tev - Build the main TEV encoder"
@echo " encoder_tev_xyb - Build the XYB color space encoder"
@echo " debug - Build with debug symbols" @echo " debug - Build with debug symbols"
@echo " clean - Remove build artifacts" @echo " clean - Remove build artifacts"
@echo " install - Install to /usr/local/bin" @echo " install - Install to /usr/local/bin"
@@ -46,7 +59,8 @@ help:
@echo " help - Show this help" @echo " help - Show this help"
@echo "" @echo ""
@echo "Usage:" @echo "Usage:"
@echo " make" @echo " make # Build both encoders"
@echo " ./encoder_tev input.mp4 -o output.tev" @echo " ./encoder_tev input.mp4 -o output.tev"
@echo " ./encoder_tev_xyb input.mp4 -o output.tev"
.PHONY: clean install check-deps help debug .PHONY: all clean install check-deps help debug

15
video_encoder/encoder_tev.c
View File

@@ -95,6 +95,7 @@ int KEYFRAME_INTERVAL = 60;
typedef struct __attribute__((packed)) { typedef struct __attribute__((packed)) {
uint8_t mode; // Block encoding mode uint8_t mode; // Block encoding mode
int16_t mv_x, mv_y; // Motion vector (1/4 pixel precision) int16_t mv_x, mv_y; // Motion vector (1/4 pixel precision)
float rate_control_factor; // Rate control factor (4 bytes, little-endian)
uint16_t cbp; // Coded block pattern (which channels have non-zero coeffs) uint16_t cbp; // Coded block pattern (which channels have non-zero coeffs)
int16_t y_coeffs[256]; // quantised Y DCT coefficients (16x16) int16_t y_coeffs[256]; // quantised Y DCT coefficients (16x16)
int16_t co_coeffs[64]; // quantised Co DCT coefficients (8x8) int16_t co_coeffs[64]; // quantised Co DCT coefficients (8x8)
@@ -666,6 +667,7 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
// Intra coding for keyframes // Intra coding for keyframes
block->mode = TEV_MODE_INTRA; block->mode = TEV_MODE_INTRA;
block->mv_x = block->mv_y = 0; block->mv_x = block->mv_y = 0;
block->rate_control_factor = enc->rate_control_factor;
enc->blocks_intra++; enc->blocks_intra++;
} else { } else {
// Implement proper mode decision for P-frames // Implement proper mode decision for P-frames
@@ -749,6 +751,7 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
block->mode = TEV_MODE_SKIP; block->mode = TEV_MODE_SKIP;
block->mv_x = 0; block->mv_x = 0;
block->mv_y = 0; block->mv_y = 0;
block->rate_control_factor = enc->rate_control_factor;
block->cbp = 0x00; // No coefficients present block->cbp = 0x00; // No coefficients present
// Zero out DCT coefficients for consistent format // Zero out DCT coefficients for consistent format
memset(block->y_coeffs, 0, sizeof(block->y_coeffs)); memset(block->y_coeffs, 0, sizeof(block->y_coeffs));
@@ -760,6 +763,7 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
(abs(block->mv_x) > 0 || abs(block->mv_y) > 0)) { (abs(block->mv_x) > 0 || abs(block->mv_y) > 0)) {
// Good motion prediction - use motion-only mode // Good motion prediction - use motion-only mode
block->mode = TEV_MODE_MOTION; block->mode = TEV_MODE_MOTION;
block->rate_control_factor = enc->rate_control_factor;
block->cbp = 0x00; // No coefficients present block->cbp = 0x00; // No coefficients present
// Zero out DCT coefficients for consistent format // Zero out DCT coefficients for consistent format
memset(block->y_coeffs, 0, sizeof(block->y_coeffs)); memset(block->y_coeffs, 0, sizeof(block->y_coeffs));
@@ -772,6 +776,7 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
// Motion compensation with threshold // Motion compensation with threshold
if (motion_sad <= 1024) { if (motion_sad <= 1024) {
block->mode = TEV_MODE_MOTION; block->mode = TEV_MODE_MOTION;
block->rate_control_factor = enc->rate_control_factor;
block->cbp = 0x00; // No coefficients present block->cbp = 0x00; // No coefficients present
memset(block->y_coeffs, 0, sizeof(block->y_coeffs)); memset(block->y_coeffs, 0, sizeof(block->y_coeffs));
memset(block->co_coeffs, 0, sizeof(block->co_coeffs)); memset(block->co_coeffs, 0, sizeof(block->co_coeffs));
@@ -783,10 +788,12 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
// Use INTER mode with motion vector and residuals // Use INTER mode with motion vector and residuals
if (abs(block->mv_x) <= 24 && abs(block->mv_y) <= 24) { if (abs(block->mv_x) <= 24 && abs(block->mv_y) <= 24) {
block->mode = TEV_MODE_INTER; block->mode = TEV_MODE_INTER;
block->rate_control_factor = enc->rate_control_factor;
enc->blocks_inter++; enc->blocks_inter++;
} else { } else {
// Motion vector too large, fall back to INTRA // Motion vector too large, fall back to INTRA
block->mode = TEV_MODE_INTRA; block->mode = TEV_MODE_INTRA;
block->rate_control_factor = enc->rate_control_factor;
block->mv_x = 0; block->mv_x = 0;
block->mv_y = 0; block->mv_y = 0;
enc->blocks_intra++; enc->blocks_intra++;
@@ -795,6 +802,7 @@ static void encode_block(tev_encoder_t *enc, int block_x, int block_y, int is_ke
} else { } else {
// No good motion prediction - use intra mode // No good motion prediction - use intra mode
block->mode = TEV_MODE_INTRA; block->mode = TEV_MODE_INTRA;
block->rate_control_factor = enc->rate_control_factor;
block->mv_x = 0; block->mv_x = 0;
block->mv_y = 0; block->mv_y = 0;
enc->blocks_intra++; enc->blocks_intra++;
@@ -1293,20 +1301,19 @@ static int encode_frame(tev_encoder_t *enc, FILE *output, int frame_num) {
// Clean up frame stream // Clean up frame stream
deflateEnd(&frame_stream); deflateEnd(&frame_stream);
// Write frame packet header (always include rate control factor) // Write frame packet header (rate control factor now per-block)
uint8_t packet_type = is_keyframe ? TEV_PACKET_IFRAME : TEV_PACKET_PFRAME; uint8_t packet_type = is_keyframe ? TEV_PACKET_IFRAME : TEV_PACKET_PFRAME;
uint32_t payload_size = compressed_size + 4; // +4 bytes for rate control factor (always) uint32_t payload_size = compressed_size; // Rate control factor now per-block, not per-packet
fwrite(&packet_type, 1, 1, output); fwrite(&packet_type, 1, 1, output);
fwrite(&payload_size, 4, 1, output); fwrite(&payload_size, 4, 1, output);
fwrite(&enc->rate_control_factor, 4, 1, output); // Always store rate control factor
fwrite(enc->compressed_buffer, 1, compressed_size, output); fwrite(enc->compressed_buffer, 1, compressed_size, output);
if (enc->verbose) { if (enc->verbose) {
printf("rateControlFactor=%.6f\n", enc->rate_control_factor); printf("rateControlFactor=%.6f\n", enc->rate_control_factor);
} }
enc->total_output_bytes += 5 + 4 + compressed_size; // packet + size + rate_factor + data enc->total_output_bytes += 5 + compressed_size; // packet + size + data (rate_factor now per-block)
// Update rate control for next frame // Update rate control for next frame
if (enc->bitrate_mode > 0) { if (enc->bitrate_mode > 0) {

2056
video_encoder/encoder_tev_xyb.c Normal file
View File

File diff suppressed because it is too large Load Diff

200
video_encoder/xyb_conversion.c Normal file
View File

@@ -0,0 +1,200 @@
// XYB Color Space Conversion Functions for TEV
// Based on JPEG XL XYB specification with proper sRGB linearization
// test with:
//// gcc -DXYB_TEST_MAIN -o test_xyb xyb_conversion.c -lm && ./test_xyb
#include <stdio.h>
#include <math.h>
#include <stdint.h>
#include <stdlib.h>
#define CLAMP(x, min, max) ((x) < (min) ? (min) : ((x) > (max) ? (max) : (x)))
// XYB conversion constants from JPEG XL specification
static const double XYB_BIAS = 0.00379307325527544933;
static const double CBRT_BIAS = 0.01558; // cbrt(XYB_BIAS)
// RGB to LMS mixing coefficients
static const double RGB_TO_LMS[3][3] = {
{0.3, 0.622, 0.078}, // L coefficients
{0.23, 0.692, 0.078}, // M coefficients
{0.24342268924547819, 0.20476744424496821, 0.55180986650955360} // S coefficients
};
// LMS to RGB inverse matrix (calculated via matrix inversion)
static const double LMS_TO_RGB[3][3] = {
{11.0315669046, -9.8669439081, -0.1646229965},
{-3.2541473811, 4.4187703776, -0.1646229965},
{-3.6588512867, 2.7129230459, 1.9459282408}
};
// sRGB linearization (0..1 range)
static inline double srgb_linearize(double val) {
if (val > 0.04045) {
return pow((val + 0.055) / 1.055, 2.4);
} else {
return val / 12.92;
}
}
// sRGB unlinearization (0..1 range)
static inline double srgb_unlinearize(double val) {
if (val > 0.0031308) {
return 1.055 * pow(val, 1.0 / 2.4) - 0.055;
} else {
return val * 12.92;
}
}
// Fast cube root approximation for performance
static inline double fast_cbrt(double x) {
if (x < 0) return -cbrt(-x);
return cbrt(x);
}
// RGB to XYB conversion with proper sRGB linearization
void rgb_to_xyb(uint8_t r, uint8_t g, uint8_t b, double *x, double *y, double *xyb_b) {
// Convert RGB to 0-1 range and linearize sRGB
double r_norm = srgb_linearize(r / 255.0);
double g_norm = srgb_linearize(g / 255.0);
double b_norm = srgb_linearize(b / 255.0);
// RGB to LMS mixing with bias
double lmix = RGB_TO_LMS[0][0] * r_norm + RGB_TO_LMS[0][1] * g_norm + RGB_TO_LMS[0][2] * b_norm + XYB_BIAS;
double mmix = RGB_TO_LMS[1][0] * r_norm + RGB_TO_LMS[1][1] * g_norm + RGB_TO_LMS[1][2] * b_norm + XYB_BIAS;
double smix = RGB_TO_LMS[2][0] * r_norm + RGB_TO_LMS[2][1] * g_norm + RGB_TO_LMS[2][2] * b_norm + XYB_BIAS;
// Apply gamma correction (cube root)
double lgamma = fast_cbrt(lmix) - CBRT_BIAS;
double mgamma = fast_cbrt(mmix) - CBRT_BIAS;
double sgamma = fast_cbrt(smix) - CBRT_BIAS;
// LMS to XYB transformation
*x = (lgamma - mgamma) / 2.0;
*y = (lgamma + mgamma) / 2.0;
*xyb_b = sgamma;
}
// XYB to RGB conversion with proper sRGB unlinearization
void xyb_to_rgb(double x, double y, double xyb_b, uint8_t *r, uint8_t *g, uint8_t *b) {
// XYB to LMS gamma
double lgamma = x + y;
double mgamma = y - x;
double sgamma = xyb_b;
// Remove gamma correction
double lmix = pow(lgamma + CBRT_BIAS, 3.0) - XYB_BIAS;
double mmix = pow(mgamma + CBRT_BIAS, 3.0) - XYB_BIAS;
double smix = pow(sgamma + CBRT_BIAS, 3.0) - XYB_BIAS;
// LMS to linear RGB using inverse matrix
double r_linear = LMS_TO_RGB[0][0] * lmix + LMS_TO_RGB[0][1] * mmix + LMS_TO_RGB[0][2] * smix;
double g_linear = LMS_TO_RGB[1][0] * lmix + LMS_TO_RGB[1][1] * mmix + LMS_TO_RGB[1][2] * smix;
double b_linear = LMS_TO_RGB[2][0] * lmix + LMS_TO_RGB[2][1] * mmix + LMS_TO_RGB[2][2] * smix;
// Clamp linear RGB to valid range
r_linear = CLAMP(r_linear, 0.0, 1.0);
g_linear = CLAMP(g_linear, 0.0, 1.0);
b_linear = CLAMP(b_linear, 0.0, 1.0);
// Convert back to sRGB gamma and 0-255 range
*r = CLAMP((int)(srgb_unlinearize(r_linear) * 255.0 + 0.5), 0, 255);
*g = CLAMP((int)(srgb_unlinearize(g_linear) * 255.0 + 0.5), 0, 255);
*b = CLAMP((int)(srgb_unlinearize(b_linear) * 255.0 + 0.5), 0, 255);
}
// Convert RGB to XYB with integer quantization suitable for TEV format
void rgb_to_xyb_quantized(uint8_t r, uint8_t g, uint8_t b, int *x_quant, int *y_quant, int *b_quant) {
double x, y, xyb_b;
rgb_to_xyb(r, g, b, &x, &y, &xyb_b);
// Quantize to suitable integer ranges for TEV
// Y channel: 0-255 (similar to current Y in YCoCg)
*y_quant = CLAMP((int)(y * 255.0 + 128.0), 0, 255);
// X channel: -128 to +127 (similar to Co range)
*x_quant = CLAMP((int)(x * 255.0), -128, 127);
// B channel: -128 to +127 (similar to Cg, can be aggressively quantized)
*b_quant = CLAMP((int)(xyb_b * 255.0), -128, 127);
}
// Test function to verify conversion accuracy
int test_xyb_conversion() {
printf("Testing XYB conversion accuracy with sRGB linearization...\n");
// Test with various RGB values
uint8_t test_colors[][3] = {
{255, 0, 0}, // Red
{0, 255, 0}, // Green
{0, 0, 255}, // Blue
{255, 255, 255}, // White
{0, 0, 0}, // Black
{128, 128, 128}, // Gray
{255, 255, 0}, // Yellow
{255, 0, 255}, // Magenta
{0, 255, 255}, // Cyan
// MacBeth chart colours converted to sRGB
{0x73,0x52,0x44},
{0xc2,0x96,0x82},
{0x62,0x7a,0x9d},
{0x57,0x6c,0x43},
{0x85,0x80,0xb1},
{0x67,0xbd,0xaa},
{0xd6,0x7e,0x2c},
{0x50,0x5b,0xa6},
{0xc1,0x5a,0x63},
{0x5e,0x3c,0x6c},
{0x9d,0xbc,0x40},
{0xe0,0xa3,0x2e},
{0x38,0x3d,0x96},
{0x46,0x94,0x49},
{0xaf,0x36,0x3c},
{0xe7,0xc7,0x1f},
{0xbb,0x56,0x95},
{0x08,0x85,0xa1},
{0xf3,0xf3,0xf3},
{0xc8,0xc8,0xc8},
{0xa0,0xa0,0xa0},
{0x7a,0x7a,0x7a},
{0x55,0x55,0x55},
{0x34,0x34,0x34}
};
int num_tests = sizeof(test_colors) / sizeof(test_colors[0]);
int errors = 0;
for (int i = 0; i < num_tests; i++) {
uint8_t r_orig = test_colors[i][0];
uint8_t g_orig = test_colors[i][1];
uint8_t b_orig = test_colors[i][2];
double x, y, xyb_b;
uint8_t r_conv, g_conv, b_conv;
// Forward and reverse conversion
rgb_to_xyb(r_orig, g_orig, b_orig, &x, &y, &xyb_b);
xyb_to_rgb(x, y, xyb_b, &r_conv, &g_conv, &b_conv);
// Check accuracy (allow small rounding errors)
int r_error = abs((int)r_orig - (int)r_conv);
int g_error = abs((int)g_orig - (int)g_conv);
int b_error = abs((int)b_orig - (int)b_conv);
printf("RGB(%3d,%3d,%3d) -> XYB(%6.3f,%6.3f,%6.3f) -> RGB(%3d,%3d,%3d) [Error: %d,%d,%d]\n",
r_orig, g_orig, b_orig, x, y, xyb_b, r_conv, g_conv, b_conv, r_error, g_error, b_error);
if (r_error > 2 || g_error > 2 || b_error > 2) {
errors++;
}
}
printf("Test completed: %d/%d passed\n", num_tests - errors, num_tests);
return errors == 0;
}
#ifdef XYB_TEST_MAIN
int main() {
return test_xyb_conversion() ? 0 : 1;
}
#endif