taut typesetter is now tvdos package

CLAUDE.md

@@ -116,6 +116,16 @@ Use the build scripts in `buildapp/`:
 - `My_BASIC_Programs/`: Example BASIC programs for testing
 - TVDOS filesystem uses custom format with specialised drivers
 
+### TSVM JavaScript Source Encoding
+
+**Do not normalise `\uXXXX` or `\xXX` escapes in .js / .mjs files that run inside
+TSVM.** TSVM's character set is not Unicode, and the JS string literal parser
+behaves differently for raw bytes vs. escape sequences. Both forms appear in
+existing code intentionally — leave each one as-is. When writing new content,
+prefer raw UTF-8 characters in string literals (e.g. write the character `ù`
+directly, rather than a `\uXXXX`-style escape) unless you are matching a
+pattern already established in the surrounding code.
+
 ## Videotron2K
 
 The Videotron2K is a specialised video display controller with:

assets/disk0/tvdos/bin/command.js

@@ -77,56 +77,31 @@ function printmotd() {
     let motd = motdFile.sread().trim()
     let width = con.getmaxyx()[1]
 
+    let ts = require("typesetter")
 
     if (goFancy) {
         let margin = 4
         let internalWidth = width - 2*margin
+        let textWidth = internalWidth - 2  // one space of padding inside each ribbon edge
 
-        con.color_pair(255,253) // white text, transparent back (initial ribbon)
-
-        let [cy, cx] = con.getyx()
-
-        con.mvaddch(cy, 4, 16);con.curs_right();print(' ')
-
-        const PCX_INIT = margin - 2
-        let tcnt = 0
-        let pcx = PCX_INIT
-        con.color_pair(240,253) // black text, white back (first line of text)
-        while (tcnt <= motd.length) {
-            let char = motd.charAt(tcnt)
-
-            if (char != '\n') {
-                // prevent the line starting from ' '
-                if (pcx != PCX_INIT || char != ' ') {
-                    print(motd.charAt(tcnt))
-                }
-                pcx += 1
-            }
-
-            if ('\n' == char || pcx % internalWidth == 0 && pcx != 0 || tcnt == motd.length) {
-                // current line ending
-                let [_, ncx] = con.getyx()
-                for (let k = 0; k < width - margin - ncx + 1; k++) print(' ')
-                con.color_pair(255,253) // white text, transparent back
-                con.addch(17);println()
-
-                if (tcnt == motd.length) break
-
-                // next line header
-                let [ncy, __] = con.getyx()
-                con.color_pair(255,253) // white text, transparent back
-                con.mvaddch(ncy, 4, 16);con.curs_right();print(' ');con.color_pair(240,253) // black text, white back (subsequent lines of the text)
-                pcx = PCX_INIT
-            }
-
-            tcnt += 1
-        }
-
+        let lines = ts.typeset(motd, textWidth)
+        lines.forEach(line => {
+            let [cy, _cx] = con.getyx()
+            con.color_pair(255,253) // ribbon edge: white text, transparent back
+            con.mvaddch(cy, margin, 16); con.curs_right()
+            print(' ')
+            con.color_pair(240,253) // body: black text, white back
+            print(line)
+            con.color_pair(255,253)
+            print(' ')
+            con.addch(17); println()
+        })
         con.reset_graphics()
     }
     else {
         println()
-        println(motd)
+        let lines = ts.typeset(motd, width)
+        lines.forEach(line => println(line))
     }
 
     println()

assets/disk0/tvdos/bin/taut.js

@@ -2297,7 +2297,7 @@ function ordersInput(wo, event) {
 // PATTERN EDITOR PANEL
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-// Returns the visual width of a TSVM string (handles „Nnu escape sequences)
+// Returns the visual width of a TSVM string (handles \u0084Nnu escape sequences)
 function visWidth(s) {
     let w = 0, i = 0
     while (i < s.length) {

assets/disk0/tvdos/bin/taut_helpmsg.js

@@ -1,6 +1,8 @@
 if (!_G.TAUT) _G.TAUT = {};
 let help = {}
 
+let ts = require("typesetter")
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /*
@@ -90,7 +92,7 @@ Timeline has two distinct modes: view and edit mode. Two modes are toggled using
 &bul;<b>W</b>&mdot;<b>E</b>&mdot;<b>R</b> : <O>toggles timeline view mode. W-most detailed, R-most abridged</O>
 &bul;<b>n</b> : <O>toggles soloing of the selected voice</O>
 &bul;<b>m</b> : <O>toggles muting of the selected voice</O>
-&bul;<b>[</b>&mdot;<b>]</b> : <O>changes tick rate of playhead</O> 
+&bul;<b>[</b>&mdot;<b>]</b> : <O>changes tick rate of playhead</O>
 
 <b>&nbsp;EDIT MODE</b>
 <b>\u00B7${'\u00B8'.repeat(9)}\u00B9</b>
@@ -143,259 +145,12 @@ Mixer flags define how should the mixer behave.
 
 // assemble help text pieces to complete help message
 
-const SCRW = con.getmaxyx()[1]
 const HRULE = '\u00B4\u00B5'.repeat((_G.TAUT.HELPMSG_WIDTH) >>> 1) + '\n'
 
-// Display-command palette. taut.js's popup uses (HELP_COL_TEXT on background) as the
-// default colour pair, so embedded `\x1B[38;5;Nm` codes switch foreground only.
-const HELP_COL_TEXT      = 239 // popup body default (== colWHITE)
-const HELP_COL_EMPH      = 230 // <b>...</b> highlight (== colVoiceHdr)
-const HELP_COL_BRAND     = 211 // first half of "Microtone"
-const HELP_COL_BRAND_DIM = 239 // second half of "Microtone"
-
-const fgEsc = (n) => `\x1B[38;5;${n}m`
-const ESC_DEFAULT = fgEsc(HELP_COL_TEXT)
-const ESC_EMPH    = fgEsc(HELP_COL_EMPH)
-const MICROTONE   = `${fgEsc(HELP_COL_BRAND)}Micro${fgEsc(HELP_COL_BRAND_DIM)}tone${ESC_DEFAULT}`
-
-// Replace &xxx; entities with their final printable representations.
-function expandEntities(s) {
-    return s
-        .replaceAll('&microtone;',  MICROTONE)
-        .replaceAll('&bul;',        '\u00F9')
-        .replaceAll('&ddot;',       '\u008419u')
-        .replaceAll('&mdot;',       '\u00FA')
-        .replaceAll('&updn;',       '\u008418u')
-        .replaceAll('&udlr;',       '\u008428u\u008429u')
-        .replaceAll('&keyoffsym;',  '\u00A0\u00B1\u00B1\u00A1')
-        .replaceAll('&notecutsym;', '\u00A4\u00A4\u00A4\u00A4')
-        .replaceAll('&nbsp;',       '\u007F')
-        .replaceAll('&shy;',        '')
-        .replaceAll('&lt;',        '<')
-        .replaceAll('&gt;',        '>')
-        .replaceAll('&demisharp;',   '\u0080\u0081')
-        .replaceAll('&sharp;',       '\u0082\u0083')
-        .replaceAll('&sesquisharp;', '\u0084132u\u0085')
-        .replaceAll('&doublesharp;', '\u0086\u0087')
-        .replaceAll('&triplesharp;', '\u0088\u0089')
-        .replaceAll('&quadsharp;',   '\u008A\u008B')
-        .replaceAll('&demiflat;',    '\u008C\u008D')
-        .replaceAll('&flat;',        '\u008E\u008F')
-        .replaceAll('&sesquiflat;',  '\u0090\u0091')
-        .replaceAll('&doubleflat;',  '\u0092\u0093')
-        .replaceAll('&tripleflat;',  '\u0094\u0095')
-        .replaceAll('&quadflat;',    '\u0096\u0097')
-        .replaceAll('&accuptick;',    '\u009A')
-        .replaceAll('&accdntick;',    '\u009B')
-        .replaceAll('&accupup;',    '\u009C')
-        .replaceAll('&accdndn;',    '\u009D')
-}
-
-// Tokenise a (post-entity-expansion) line. Returns an array of:
-//   {type:'word',   text:String, w:int}  - non-breakable run of visible chars (may carry ANSI escapes)
-//   {type:'sp'}                          - a single soft space (eligible for break/expansion)
-//   {type:'anchor', open:Boolean}        - <o>/</o> markers (zero width)
-//
-// Width accounting:
-//   - ANSI escapes (`\x1B[...m`)         : 0 visible chars
-//   - TSVM unicode escapes (`„..u`) : 1 visible char
-//   - non-breaking space ( )        : 1 visible char (consumed as part of a word)
-//   - soft hyphen (­)               : dropped (not implemented as a break point)
-//   - everything else                    : 1 visible char
-function tokenise(line) {
-    const tokens = []
-    let buf = ''
-    let bufW = 0
-    let i = 0
-
-    const flushWord = () => {
-        if (buf.length > 0) {
-            tokens.push({type: 'word', text: buf, w: bufW})
-            buf = ''
-            bufW = 0
-        }
-    }
-
-    while (i < line.length) {
-        // inline tags (case-sensitive for <b>, case-insensitive for <o>)
-        if (line.slice(i, i + 3) === '<b>')  { buf += ESC_EMPH;    i += 3; continue }
-        if (line.slice(i, i + 4) === '</b>') { buf += ESC_DEFAULT; i += 4; continue }
-        const head3 = line.slice(i, i + 3).toLowerCase()
-        const head4 = line.slice(i, i + 4).toLowerCase()
-        if (head3 === '<o>')  { flushWord(); tokens.push({type: 'anchor', open: true});  i += 3; continue }
-        if (head4 === '</o>') { flushWord(); tokens.push({type: 'anchor', open: false}); i += 4; continue }
-
-        const c  = line[i]
-        const cc = line.charCodeAt(i)
-
-        if (cc === 0x1B) {
-            // pre-existing ANSI escape - copy verbatim, zero visible width
-            const m = line.indexOf('m', i)
-            const end = (m < 0) ? line.length : m + 1
-            buf += line.slice(i, end)
-            i = end
-        }
-        else if (cc === 0x84) {
-            // TSVM „<digits>u escape - copy verbatim, one visible char
-            const u = line.indexOf('u', i)
-            const end = (u < 0) ? line.length : u + 1
-            buf += line.slice(i, end)
-            bufW += 1
-            i = end
-        }
-        else if (c === ' ') {
-            flushWord()
-            tokens.push({type: 'sp'})
-            i += 1
-        }
-        else if (cc === 0x00AD) {
-            // soft hyphen: drop (no break-point handling for now)
-            i += 1
-        }
-        else {
-            buf += c
-            bufW += 1
-            i += 1
-        }
-    }
-    flushWord()
-    return tokens
-}
-
-// Build wrapped lines from a token stream then format each one according to alignment.
-// Returns an array of strings, each exactly `width` visible chars wide (padded with
-// trailing spaces) so the caller can blit them without further math.
-function wrapAndAlign(tokens, width, alignment) {
-    const lines = [] // each: {tokens, indent, contentW}
-    let curTokens = []
-    let curW = 0
-    let curIndent = 0
-    let nextIndent = 0 // indent the *next* flushed line should use
-
-    const flushLine = () => {
-        // strip trailing soft spaces
-        while (curTokens.length > 0 && curTokens[curTokens.length - 1].type === 'sp') {
-            curTokens.pop()
-            curW -= 1
-        }
-        lines.push({tokens: curTokens, indent: curIndent, contentW: curW})
-        curTokens = []
-        curW = 0
-        curIndent = nextIndent
-    }
-
-    for (const tok of tokens) {
-        if (tok.type === 'anchor') {
-            // anchor opens at the current visible column (accounting for indent)
-            if (tok.open) nextIndent = curIndent + curW
-            else          nextIndent = 0
-            continue
-        }
-
-        if (tok.type === 'sp') {
-            // ignore leading soft spaces on a fresh line
-            if (curW === 0) continue
-            // hard wrap if the line is already at the right edge
-            if (curIndent + curW + 1 > width) { flushLine(); continue }
-            curTokens.push(tok)
-            curW += 1
-            continue
-        }
-
-        // word
-        const tw = tok.w
-        if (curIndent + curW + tw > width) {
-            flushLine()
-            // word too wide for the wrapped line: emit it on its own row (possibly clipped by terminal)
-            if (curIndent + tw > width) {
-                curTokens.push(tok)
-                curW += tw
-                flushLine()
-                continue
-            }
-        }
-        curTokens.push(tok)
-        curW += tw
-    }
-
-    if (curTokens.length > 0 || lines.length === 0) flushLine()
-
-    return lines.map((line, i) => formatLine(line, width, alignment, i === lines.length - 1))
-}
-
-function formatLine(line, totalWidth, alignment, isLast) {
-    if (line.tokens.length === 0) return ' '.repeat(totalWidth)
-
-    const indent    = ' '.repeat(line.indent)
-    const remaining = totalWidth - line.indent - line.contentW
-    const pad       = (n) => (n > 0) ? ' '.repeat(n) : ''
-    const flatText  = () => line.tokens.map(t => (t.type === 'sp') ? ' ' : t.text).join('')
-
-    if (alignment === 'c') {
-        const left = remaining >> 1
-        return indent + pad(left) + flatText() + pad(remaining - left)
-    }
-    if (alignment === 'r') return indent + pad(remaining) + flatText()
-    if (alignment === 'l') return indent + flatText() + pad(remaining)
-
-    // justified: only expand spaces when there's slack and we're not on the
-    // last (or single) wrapped line
-    if (isLast || remaining <= 0) return indent + flatText() + pad(remaining)
-
-    const spaceCount = line.tokens.reduce((n, t) => n + (t.type === 'sp' ? 1 : 0), 0)
-    if (spaceCount === 0) return indent + flatText() + pad(remaining)
-
-    const baseExtra = (remaining / spaceCount) | 0
-    let leftover    = remaining - baseExtra * spaceCount
-
-    let out = indent
-    for (const tok of line.tokens) {
-        if (tok.type === 'sp') {
-            const extra = baseExtra + (leftover > 0 ? 1 : 0)
-            if (leftover > 0) leftover -= 1
-            out += ' '.repeat(1 + extra)
-        } else {
-            out += tok.text
-        }
-    }
-    return out
-}
-
-// Process a single source line: peel a leading <c>/<r>/<l> alignment tag (if present),
-// strip its matching close tag, then tokenise + wrap.
-function typesetSourceLine(line, width) {
-    if (line.length === 0) return [' '.repeat(width)]
-
-    let alignment = 'j' // justified default
-    const startMatch = line.match(/^<([crl])>/i)
-    if (startMatch) {
-        alignment = startMatch[1].toLowerCase()
-        line = line.slice(startMatch[0].length)
-        const closeRe = new RegExp(`</${alignment}>$`, 'i')
-        line = line.replace(closeRe, '')
-    }
-
-    const tokens = tokenise(line)
-    return wrapAndAlign(tokens, width, alignment)
-}
-
-function typesetText(text, width) {
-    text = expandEntities(text)
-    const out = []
-    for (const srcLine of text.split('\n')) {
-        for (const outLine of typesetSourceLine(srcLine, width)) out.push(outLine)
-    }
-    return out
-}
-
-function typeset(text, customWidth) {
-    let typesetWidth = customWidth
-    if (typesetWidth === undefined) typesetWidth = _G.TAUT.HELPMSG_WIDTH
-    if (typesetWidth === undefined) {
-        const currentPosX = con.getyx()[1] // 1-indexed
-        typesetWidth = SCRW - currentPosX + 1
-    }
-    return typesetText(text, typesetWidth)
+// taut.js's popup uses (HELP_COL_TEXT on background) as the default colour pair.
+// The shared typesetter module owns the palette and the markup expander.
+function typeset(text) {
+    return ts.typeset(text, _G.TAUT.HELPMSG_WIDTH)
 }
 
 let helpMessages = [ // index: taut.js PANEL_NAMES
@@ -410,7 +165,7 @@ let helpMessages = [ // index: taut.js PANEL_NAMES
 
 help.MSG_BY_TABS = helpMessages.map(it => typeset(it))
 help.typeset     = typeset
-help.COL_TEXT    = HELP_COL_TEXT
-help.COL_EMPH    = HELP_COL_EMPH
+help.COL_TEXT    = ts.COL_TEXT
+help.COL_EMPH    = ts.COL_EMPH
 
 if (!_G.TAUT.HELPMSG) _G.TAUT.HELPMSG=help;

assets/disk0/tvdos/hopper/textedit.hop.per

@@ -0,0 +1,12 @@
+HopperManifestVersion:1
+HopperPackageName:textedit
+HopperPackageVersion:1.0.0
+HopperPackageMaintainer:CuriousTorvald
+HopperProvides:edit;
+HopperRequires:tvdos 1.*
+ProperName:edit.js
+ProperAuthor:CuriousTorvald
+ProperDescription:TVDOS default text editor
+Licence:MIT
+SupportMe:https://github.com/sponsors/curioustorvald/
+SystemPackagePath:/tvdos/bin/edit.js

assets/disk0/tvdos/hopper/tvdos.hop.per

@@ -9,4 +9,4 @@ ProperAuthor:CuriousTorvald
 ProperDescription:TSVM Disk Operating System
 Licence:MIT
 SupportMe:https://github.com/sponsors/curioustorvald/
-SystemPackagePath:/tvdos/TVDOS.SYS
+SystemPackagePath:/tvdos/TVDOS.SYS;/tvdos/hyve.SYS;/tvdos/HSDPADRV.SYS;/tvdos/bin/command.js;/tvdos/sbin/sysctl.js;/tvdos/include/font.mjs;/tvdos/include/keysym.mjs;/tvdos/include/mload.mjs;/tvdos/include/playgui.mjs;/tvdos/include/typesetter.mjs