maths ops

Updated funding options to include GitHub Sponsors and a PayPal link.

.claude/settings.local.json

@@ -0,0 +1,30 @@
+{
+  "permissions": {
+    "allow": [
+      "WebFetch(domain:github.com)",
+      "WebSearch",
+      "Bash(head:*)",
+      "WebFetch(domain:gitlab.com)",
+      "Bash(java:*)",
+      "Bash(ls:*)",
+      "Bash(jar tf:*)",
+      "Bash(chmod +x:*)",
+      "WebFetch(domain:fontforge.org)",
+      "WebFetch(domain:fonttools.readthedocs.io)",
+      "Bash(grep:*)",
+      "Bash(tail:*)",
+      "Bash(python3:*)",
+      "Bash(make:*)",
+      "Bash(git stash:*)",
+      "Bash(./autokem*)",
+      "Bash(cmp:*)",
+      "Bash(wc:*)",
+      "Bash(pip3:*)",
+      "Bash(pip install:*)",
+      "Bash(.venv/bin/python3:*)",
+      "Bash(.venv/bin/python:*)",
+      "Bash(find:*)",
+      "Skill(update-config)"
+    ]
+  }
+}

.claude/skills/add-unicode-block/SKILL.md

@@ -0,0 +1,93 @@
+---
+name: add-unicode-block
+description: Add a new Unicode script/block to the Terrarum Sans Bitmap font engine.
+---
+# Add Unicode Block
+
+## Required inputs
+
+The user must supply:
+- **Script name** — human-readable name used in constant/function names (e.g. `Ogham`, `LatinExtE`)
+- **TGA filename** — the sprite sheet filename without path (e.g. `ogham_variable.tga`)
+- **Unicode range** — start and end codepoints inclusive (e.g. `U+1680..U+169F`)
+
+If any of these are missing, ask for them before proceeding.
+
+## Step 1 — Determine the next sheet index
+
+Read the sheet index constants from both files to find the current highest index (excluding `SHEET_UNKNOWN = 254`):
+
+- Kotlin: `src/net/torvald/terrarumsansbitmap/gdx/TerrarumSansBitmap.kt` — grep for `internal const val SHEET_`
+- Python: `OTFbuild/sheet_config.py` — grep for `^SHEET_`
+
+The new index = highest existing index + 1.
+
+## Step 2 — Derive identifiers
+
+From the script name, derive:
+- **Kotlin constant**: `SHEET_<UPPER_SNAKE>_VARW` (e.g. `SHEET_OGHAM_VARW`)
+- **Kotlin indexY function**: `<camelCase>IndexY` (e.g. `oghamIndexY`)
+- **Python constant**: same as Kotlin constant
+- **Range start hex**: the lower bound codepoint as a `0x`-prefixed Kotlin/Python literal
+
+## Step 3 — Edit both files
+
+Make all 6 edits. Read each section before editing.
+
+### Kotlin: `src/net/torvald/terrarumsansbitmap/gdx/TerrarumSansBitmap.kt`
+
+**a) Sheet index constant** — find the block of `internal const val SHEET_*` constants (just before `SHEET_UNKNOWN = 254`) and append:
+```kotlin
+internal const val SHEET_<NAME>_VARW = <INDEX>
+```
+
+**b) fileList entry** — find `internal val fileList` array and append before the closing `)`:
+```kotlin
+"<tga_filename>",
+```
+
+**c) codeRange entry** — find `internal val codeRange` array and append before the closing `)`:
+```kotlin
+0x<START>..<0x<END>, // SHEET_<NAME>_VARW
+```
+Use `+` to combine non-contiguous ranges if needed.
+
+**d) getSheetwisePosition when-branch** — find the `when` block that dispatches to indexY functions (just before `else -> ch / 16`) and append:
+```kotlin
+SHEET_<NAME>_VARW -> <camelCase>IndexY(ch)
+```
+
+**e) indexY function** — find the block of private `*IndexY` functions near the bottom of the companion object and append:
+```kotlin
+private fun <camelCase>IndexY(c: CodePoint) = (c - 0x<START>) / 16
+```
+
+### Python: `OTFbuild/sheet_config.py`
+
+**f) Sheet index constant** — find the block of `SHEET_* = <n>` constants (just before `SHEET_UNKNOWN = 254`) and append:
+```python
+SHEET_<NAME>_VARW = <INDEX>
+```
+
+**g) FILE_LIST entry** — find `FILE_LIST = [` array and append before the closing `]`:
+```python
+"<tga_filename>",
+```
+
+**h) CODE_RANGE entry** — find `CODE_RANGE = [` array and append before the closing `]`:
+```python
+list(range(0x<START>, 0x<END+1>)),                                                  # <INDEX>: <ScriptName>
+```
+
+**i) index_y lambda** — find the dict in `get_index_y(sheet_index, c)` (just before `SHEET_HANGUL: lambda: 0`) and append:
+```python
+SHEET_<NAME>_VARW: lambda: (c - 0x<START>) // 16,
+```
+
+## Step 4 — Verify
+
+After all edits, confirm:
+1. The Kotlin constant, fileList, codeRange, when-branch, and indexY function are all present and consistent.
+2. The Python constant, FILE_LIST, CODE_RANGE, and index_y lambda are all present and consistent.
+3. The indices in both files match.
+4. The range end in `CODE_RANGE` is `end + 1` (Python `range` is exclusive).

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [curioustorvald]
+custom: ["https://paypal.me/curioustorvald"]

Autokem/apply.c

@@ -2,6 +2,7 @@
 #include "tga.h"
 #include "nn.h"
 #include "safetensor.h"
+#include "unicode_filter.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -75,7 +76,8 @@ int apply_model(const char *tga_path) {
     int rows = img->height / cell_h;
     int total_cells = cols * rows;
 
-    int processed = 0, updated = 0, skipped = 0;
+    int start_code = sheet_start_code(basename);
+    int processed = 0, updated = 0, skipped = 0, fixed_lm = 0;
 
     for (int index = 0; index < total_cells; index++) {
         int cell_x, cell_y;
@@ -107,6 +109,21 @@ int apply_model(const char *tga_path) {
         int opcode = (int)((dir_pixel >> 24) & 0xFF);
         if (opcode != 0) { skipped++; continue; }
 
+        /* Modifier letters: fixed kern pixel, skip inference */
+        if (start_code >= 0 && is_modifier_letter(start_code + index)) {
+            if (is_subscript_modifier(start_code + index)) {
+                /* Subscript: CDEFGHJK(B), lowheight=1 */
+                tga_write_pixel(tga_path, img, tag_x, tag_y + 5, 0xFFFFFFFF);
+                tga_write_pixel(tga_path, img, tag_x, tag_y + 6, 0x00C03FFF);
+            } else {
+                /* Superscript: ABCDEF(B), lowheight=0 */
+                tga_write_pixel(tga_path, img, tag_x, tag_y + 5, 0x00000000);
+                tga_write_pixel(tga_path, img, tag_x, tag_y + 6, 0x0000FCFF);
+            }
+            processed++; updated++; fixed_lm++;
+            continue;
+        }
+
         /* Extract 15x20 binary input */
         float input[300];
         for (int gy = 0; gy < 20; gy++) {
@@ -155,8 +172,8 @@ int apply_model(const char *tga_path) {
         updated++;
     }
 
-    printf("Processed: %d cells, Updated: %d, Skipped: %d (of %d total)\n",
-           processed, updated, skipped, total_cells);
+    printf("Processed: %d cells, Updated: %d, Skipped: %d, Fixed Lm: %d (of %d total)\n",
+           processed, updated, skipped, fixed_lm, total_cells);
 
     tga_free(img);
     network_free(net);

Autokem/eval.sh

@@ -7,7 +7,7 @@
 set -euo pipefail
 cd "$(dirname "$0")"
 
-RUNS="${1:-10}"
+RUNS="${1:-42}"
 shift 2>/dev/null || true
 EXTRA_ARGS="$*"
 PYTHON="${PYTHON:-.venv/bin/python3}"

Autokem/sheet_stats.py

@@ -0,0 +1,631 @@
+#!/usr/bin/env python3
+"""
+Spritesheet statistics generator for TerrarumSansBitmap.
+
+Scans all *_variable.tga sheets and reports:
+  - Width distribution
+  - Compiler directives (replaceWith breakdown)
+  - Kerning shape distribution
+  - Lowheight count
+  - Diacritics (anchors, writeOnTop, stacking)
+  - Glyphs missing kerning data
+  - Dot removal directives
+  - Nudge usage
+  - Alignment modes
+  - Per-sheet summary
+
+Usage:
+    python sheet_stats.py [assets_dir]
+    python sheet_stats.py ../src/assets
+"""
+
+import os
+import struct
+import sys
+from collections import Counter, defaultdict
+
+# ---- TGA reader ----
+
+class TgaImage:
+    __slots__ = ('width', 'height', 'pixels')
+
+    def __init__(self, width, height, pixels):
+        self.width = width
+        self.height = height
+        self.pixels = pixels
+
+    def get_pixel(self, x, y):
+        if x < 0 or x >= self.width or y < 0 or y >= self.height:
+            return 0
+        return self.pixels[y * self.width + x]
+
+
+def read_tga(path):
+    with open(path, 'rb') as f:
+        data = f.read()
+    pos = 0
+    id_length = data[pos]; pos += 1
+    pos += 1  # colour_map_type
+    image_type = data[pos]; pos += 1
+    pos += 5
+    pos += 4  # x/y origin
+    width = struct.unpack_from('<H', data, pos)[0]; pos += 2
+    height = struct.unpack_from('<H', data, pos)[0]; pos += 2
+    bits_per_pixel = data[pos]; pos += 1
+    descriptor = data[pos]; pos += 1
+    top_to_bottom = (descriptor & 0x20) != 0
+    bpp = bits_per_pixel // 8
+    pos += id_length
+    if image_type != 2 or bpp not in (3, 4):
+        raise ValueError(f"Unsupported TGA: type={image_type}, bpp={bits_per_pixel}")
+    pixels = [0] * (width * height)
+    for row in range(height):
+        y = row if top_to_bottom else (height - 1 - row)
+        for x in range(width):
+            b = data[pos]; g = data[pos+1]; r = data[pos+2]
+            a = data[pos+3] if bpp == 4 else 0xFF
+            pos += bpp
+            pixels[y * width + x] = (r << 24) | (g << 16) | (b << 8) | a
+    return TgaImage(width, height, pixels)
+
+
+def tagify(pixel):
+    return 0 if (pixel & 0xFF) == 0 else pixel
+
+
+def signed_byte(val):
+    return val - 256 if val >= 128 else val
+
+
+# ---- Unicode range classification ----
+
+# Ranges to EXCLUDE from "missing kern" report
+EXCLUDE_KERN_RANGES = [
+    (0x3400, 0xA000, 'CJK Unified Ideographs'),
+    (0x1100, 0x1200, 'Hangul Jamo'),
+    (0xA960, 0xA980, 'Hangul Jamo Extended-A'),
+    (0xD7B0, 0xD800, 'Hangul Jamo Extended-B'),
+    (0x3130, 0x3190, 'Hangul Compatibility Jamo'),
+    (0xAC00, 0xD7A4, 'Hangul Syllables'),
+    (0xE000, 0xE100, 'Custom Symbols (PUA)'),
+    (0xF0000, 0xF0600, 'Internal PUA'),
+    (0xFFE00, 0x100000, 'Internal control/PUA'),
+    (0x2800, 0x2900, 'Braille'),
+    (0x1FB00, 0x1FC00, 'Legacy Computing Symbols'),
+    (0x2400, 0x2440, 'Control Pictures'),
+    (0x3000, 0x3040, 'CJK Punctuation'),
+    (0x3040, 0x3100, 'Hiragana/Katakana'),
+    (0x31F0, 0x3200, 'Katakana Phonetic Ext'),
+    (0xFF00, 0x10000, 'Halfwidth/Fullwidth'),
+    (0x16A0, 0x1700, 'Runic'),
+    (0x300, 0x370, 'Combining Diacritical Marks'),
+    (0x1B000, 0x1B170, 'Hentaigana'),
+]
+
+
+def is_excluded_from_kern(cp):
+    for lo, hi, _ in EXCLUDE_KERN_RANGES:
+        if lo <= cp < hi:
+            return True
+    return False
+
+
+def unicode_block_name(cp):
+    """Rough Unicode block classification for display."""
+    blocks = [
+        (0x0000, 0x0080, 'Basic Latin'),
+        (0x0080, 0x0100, 'Latin-1 Supplement'),
+        (0x0100, 0x0180, 'Latin Extended-A'),
+        (0x0180, 0x0250, 'Latin Extended-B'),
+        (0x0250, 0x02B0, 'IPA Extensions'),
+        (0x02B0, 0x0300, 'Spacing Modifier Letters'),
+        (0x0300, 0x0370, 'Combining Diacritical Marks'),
+        (0x0370, 0x0400, 'Greek and Coptic'),
+        (0x0400, 0x0530, 'Cyrillic'),
+        (0x0530, 0x0590, 'Armenian'),
+        (0x0900, 0x0980, 'Devanagari'),
+        (0x0980, 0x0A00, 'Bengali'),
+        (0x0B80, 0x0C00, 'Tamil'),
+        (0x0E00, 0x0E80, 'Thai'),
+        (0x10D0, 0x1100, 'Georgian'),
+        (0x1100, 0x1200, 'Hangul Jamo'),
+        (0x13A0, 0x13F6, 'Cherokee'),
+        (0x1B80, 0x1BC0, 'Sundanese'),
+        (0x1C80, 0x1CC0, 'Cyrillic Extended'),
+        (0x1D00, 0x1DC0, 'Phonetic Extensions'),
+        (0x1E00, 0x1F00, 'Latin Extended Additional'),
+        (0x1F00, 0x2000, 'Greek Extended'),
+        (0x2000, 0x2070, 'General Punctuation'),
+        (0x20A0, 0x20D0, 'Currency Symbols'),
+        (0x2100, 0x2200, 'Letterlike Symbols'),
+        (0x2C60, 0x2C80, 'Latin Extended-C'),
+        (0x2DE0, 0x2E00, 'Cyrillic Extended-A'),
+        (0xA640, 0xA6A0, 'Cyrillic Extended-B'),
+        (0xA720, 0xA800, 'Latin Extended-D'),
+        (0xFB00, 0xFB50, 'Alphabetic Presentation Forms'),
+        (0x1F100, 0x1F200, 'Enclosed Alphanumeric Supplement'),
+        (0xF0000, 0xF0060, 'PUA Bulgarian'),
+        (0xF0060, 0xF00C0, 'PUA Serbian'),
+        (0xF0100, 0xF0500, 'PUA Devanagari Internal'),
+        (0xF0500, 0xF0600, 'PUA Sundanese/Codestyle'),
+    ]
+    for lo, hi, name in blocks:
+        if lo <= cp < hi:
+            return name
+    return f'U+{cp:04X}'
+
+
+# ---- Code ranges (from sheet_config.py) ----
+
+CODE_RANGE = [
+    list(range(0x00, 0x100)),
+    list(range(0x1100, 0x1200)) + list(range(0xA960, 0xA980)) + list(range(0xD7B0, 0xD800)),
+    list(range(0x100, 0x180)),
+    list(range(0x180, 0x250)),
+    list(range(0x3040, 0x3100)) + list(range(0x31F0, 0x3200)),
+    list(range(0x3000, 0x3040)),
+    list(range(0x3400, 0xA000)),
+    list(range(0x400, 0x530)),
+    list(range(0xFF00, 0x10000)),
+    list(range(0x2000, 0x20A0)),
+    list(range(0x370, 0x3CF)),
+    list(range(0xE00, 0xE60)),
+    list(range(0x530, 0x590)),
+    list(range(0x10D0, 0x1100)),
+    list(range(0x250, 0x300)),
+    list(range(0x16A0, 0x1700)),
+    list(range(0x1E00, 0x1F00)),
+    list(range(0xE000, 0xE100)),
+    list(range(0xF0000, 0xF0060)),
+    list(range(0xF0060, 0xF00C0)),
+    list(range(0x13A0, 0x13F6)),
+    list(range(0x1D00, 0x1DC0)),
+    list(range(0x900, 0x980)) + list(range(0xF0100, 0xF0500)),
+    list(range(0x1C90, 0x1CC0)),
+    list(range(0x300, 0x370)),
+    list(range(0x1F00, 0x2000)),
+    list(range(0x2C60, 0x2C80)),
+    list(range(0xA720, 0xA800)),
+    list(range(0x20A0, 0x20D0)),
+    list(range(0xFFE00, 0xFFFA0)),
+    list(range(0x2100, 0x2200)),
+    list(range(0x1F100, 0x1F200)),
+    list(range(0x0B80, 0x0C00)) + list(range(0xF00C0, 0xF0100)),
+    list(range(0x980, 0xA00)),
+    list(range(0x2800, 0x2900)),
+    list(range(0x1B80, 0x1BC0)) + list(range(0x1CC0, 0x1CD0)) + list(range(0xF0500, 0xF0510)),
+    list(range(0xF0110, 0xF0130)),
+    list(range(0xF0520, 0xF0580)),
+    list(range(0xFB00, 0xFB18)),
+    list(range(0x1B000, 0x1B170)),
+    list(range(0x2400, 0x2440)),
+    list(range(0x1FB00, 0x1FC00)),
+    list(range(0xA640, 0xA6A0)),
+    list(range(0x2DE0, 0x2E00)),
+    list(range(0x1C80, 0x1C8F)),
+]
+
+FILE_LIST = [
+    "ascii_variable.tga",
+    "hangul_johab.tga",
+    "latinExtA_variable.tga",
+    "latinExtB_variable.tga",
+    "kana_variable.tga",
+    "cjkpunct_variable.tga",
+    "wenquanyi.tga",
+    "cyrilic_variable.tga",
+    "halfwidth_fullwidth_variable.tga",
+    "unipunct_variable.tga",
+    "greek_variable.tga",
+    "thai_variable.tga",
+    "hayeren_variable.tga",
+    "kartuli_variable.tga",
+    "ipa_ext_variable.tga",
+    "futhark.tga",
+    "latinExt_additional_variable.tga",
+    "puae000-e0ff.tga",
+    "cyrilic_bulgarian_variable.tga",
+    "cyrilic_serbian_variable.tga",
+    "tsalagi_variable.tga",
+    "phonetic_extensions_variable.tga",
+    "devanagari_variable.tga",
+    "kartuli_allcaps_variable.tga",
+    "diacritical_marks_variable.tga",
+    "greek_polytonic_xyswap_variable.tga",
+    "latinExtC_variable.tga",
+    "latinExtD_variable.tga",
+    "currencies_variable.tga",
+    "internal_variable.tga",
+    "letterlike_symbols_variable.tga",
+    "enclosed_alphanumeric_supplement_variable.tga",
+    "tamil_extrawide_variable.tga",
+    "bengali_variable.tga",
+    "braille_variable.tga",
+    "sundanese_variable.tga",
+    "devanagari_internal_extrawide_variable.tga",
+    "pua_codestyle_ascii_variable.tga",
+    "alphabetic_presentation_forms_extrawide_variable.tga",
+    "hentaigana_variable.tga",
+    "control_pictures_variable.tga",
+    "symbols_for_legacy_computing_variable.tga",
+    "cyrilic_extB_variable.tga",
+    "cyrilic_extA_variable.tga",
+    "cyrilic_extC_variable.tga",
+]
+
+
+def is_variable(fn):
+    return fn.endswith('_variable.tga')
+
+
+def is_extra_wide(fn):
+    return 'extrawide' in fn.lower()
+
+
+def is_xyswap(fn):
+    return 'xyswap' in fn.lower()
+
+
+# ---- Shape tag formatting ----
+
+SHAPE_CHARS = 'ABCDEFGHJK'
+
+
+def format_shape(mask, is_ytype):
+    """Format kerning mask + ytype as keming_machine tag, e.g. 'ABCDEFGH(B)'."""
+    bits = []
+    for i, ch in enumerate(SHAPE_CHARS):
+        bit_pos = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14][i]
+        if (mask >> bit_pos) & 1:
+            bits.append(ch)
+    chars = ''.join(bits) if bits else '(empty)'
+    mode = '(Y)' if is_ytype else '(B)'
+    return f'{chars}{mode}'
+
+
+# ---- Parsing ----
+
+def parse_diacritics_anchors(img, tag_x, tag_y):
+    """Return number of defined diacritics anchors (0-6)."""
+    count = 0
+    for i in range(6):
+        y_pos = 13 - (i // 3) * 2
+        shift = (3 - (i % 3)) * 8
+        y_pixel = tagify(img.get_pixel(tag_x, tag_y + y_pos))
+        x_pixel = tagify(img.get_pixel(tag_x, tag_y + y_pos + 1))
+        y_used = ((y_pixel >> shift) & 128) != 0
+        x_used = ((x_pixel >> shift) & 128) != 0
+        if y_used or x_used:
+            count += 1
+    return count
+
+
+def parse_variable_sheet(path, code_range, is_xy, is_ew):
+    """Parse a variable-width sheet and yield per-glyph stats dicts."""
+    img = read_tga(path)
+    cell_w = 32 if is_ew else 16
+    cell_h = 20
+    cols = img.width // cell_w
+
+    for index, code in enumerate(code_range):
+        if is_xy:
+            cell_x = (index // cols) * cell_w
+            cell_y = (index % cols) * cell_h
+        else:
+            cell_x = (index % cols) * cell_w
+            cell_y = (index // cols) * cell_h
+
+        tag_x = cell_x + (cell_w - 1)
+        tag_y = cell_y
+
+        # Width
+        width = 0
+        for y in range(5):
+            if img.get_pixel(tag_x, tag_y + y) & 0xFF:
+                width |= (1 << y)
+
+        if width == 0:
+            continue  # empty cell
+
+        # Lowheight
+        is_low_height = (img.get_pixel(tag_x, tag_y + 5) & 0xFF) != 0
+
+        # Kerning data
+        kern_pixel = tagify(img.get_pixel(tag_x, tag_y + 6))
+        has_kern = (kern_pixel & 0xFF) != 0
+        is_ytype = (kern_pixel & 0x80000000) != 0 if has_kern else False
+        kern_mask = ((kern_pixel >> 8) & 0xFFFFFF) if has_kern else 0
+
+        # Dot removal (Y+7)
+        dot_pixel = tagify(img.get_pixel(tag_x, tag_y + 7))
+        has_dot_removal = dot_pixel != 0
+
+        # Compiler directive (Y+9)
+        dir_pixel = tagify(img.get_pixel(tag_x, tag_y + 9))
+        opcode = (dir_pixel >> 24) & 0xFF
+        arg1 = (dir_pixel >> 16) & 0xFF
+        arg2 = (dir_pixel >> 8) & 0xFF
+
+        # Nudge (Y+10)
+        nudge_pixel = tagify(img.get_pixel(tag_x, tag_y + 10))
+        nudge_x = signed_byte((nudge_pixel >> 24) & 0xFF) if nudge_pixel else 0
+        nudge_y = signed_byte((nudge_pixel >> 16) & 0xFF) if nudge_pixel else 0
+        has_nudge = nudge_x != 0 or nudge_y != 0
+
+        # Diacritics anchors (Y+11..Y+14)
+        n_anchors = parse_diacritics_anchors(img, tag_x, tag_y)
+
+        # Alignment (Y+15..Y+16)
+        align = 0
+        for y in range(2):
+            if img.get_pixel(tag_x, tag_y + 15 + y) & 0xFF:
+                align |= (1 << y)
+
+        # WriteOnTop (Y+17)
+        wot_raw = img.get_pixel(tag_x, tag_y + 17)
+        has_write_on_top = (wot_raw & 0xFF) != 0
+
+        # Stack (Y+18..Y+19)
+        s0 = tagify(img.get_pixel(tag_x, tag_y + 18))
+        s1 = tagify(img.get_pixel(tag_x, tag_y + 19))
+        if s0 == 0x00FF00FF and s1 == 0x00FF00FF:
+            stack_where = 4  # STACK_DONT
+        else:
+            stack_where = 0
+            for y in range(2):
+                if img.get_pixel(tag_x, tag_y + 18 + y) & 0xFF:
+                    stack_where |= (1 << y)
+
+        yield {
+            'code': code,
+            'width': width,
+            'lowheight': is_low_height,
+            'has_kern': has_kern,
+            'is_ytype': is_ytype,
+            'kern_mask': kern_mask,
+            'has_dot_removal': has_dot_removal,
+            'opcode': opcode,
+            'opcode_arg1': arg1,
+            'opcode_arg2': arg2,
+            'has_nudge': has_nudge,
+            'nudge_x': nudge_x,
+            'nudge_y': nudge_y,
+            'n_anchors': n_anchors,
+            'align': align,
+            'has_write_on_top': has_write_on_top,
+            'stack_where': stack_where,
+        }
+
+
+# ---- Main ----
+
+def main():
+    assets_dir = sys.argv[1] if len(sys.argv) > 1 else '../src/assets'
+
+    # Accumulators
+    all_glyphs = []
+    per_sheet = defaultdict(lambda: {'total': 0, 'kern': 0, 'lowh': 0, 'directives': 0})
+    sheets_scanned = 0
+
+    print(f"Scanning {assets_dir}...\n")
+
+    for sheet_idx, filename in enumerate(FILE_LIST):
+        if not is_variable(filename):
+            continue
+        if sheet_idx >= len(CODE_RANGE):
+            continue
+
+        path = os.path.join(assets_dir, filename)
+        if not os.path.exists(path):
+            continue
+
+        is_xy = is_xyswap(filename)
+        is_ew = is_extra_wide(filename)
+        code_range = CODE_RANGE[sheet_idx]
+
+        count = 0
+        for g in parse_variable_sheet(path, code_range, is_xy, is_ew):
+            g['sheet'] = filename
+            all_glyphs.append(g)
+            s = per_sheet[filename]
+            s['total'] += 1
+            if g['has_kern']:
+                s['kern'] += 1
+            if g['lowheight']:
+                s['lowh'] += 1
+            if g['opcode'] != 0:
+                s['directives'] += 1
+            count += 1
+
+        sheets_scanned += 1
+
+    total = len(all_glyphs)
+    if total == 0:
+        print("No glyphs found!")
+        return 1
+
+    print(f"Scanned {sheets_scanned} variable sheets, {total} glyphs with width > 0\n")
+
+    # ---- 1. Width distribution ----
+    width_counter = Counter(g['width'] for g in all_glyphs)
+    print("=" * 60)
+    print("WIDTH DISTRIBUTION")
+    print("=" * 60)
+    for w in sorted(width_counter):
+        c = width_counter[w]
+        bar = '#' * (c * 40 // max(width_counter.values()))
+        print(f"  w={w:2d}: {c:5d} ({100*c/total:5.1f}%)  {bar}")
+    print(f"  Total: {total}")
+
+    # ---- 2. Compiler directives ----
+    dir_glyphs = [g for g in all_glyphs if g['opcode'] != 0]
+    print(f"\n{'=' * 60}")
+    print("COMPILER DIRECTIVES")
+    print("=" * 60)
+    print(f"  Total glyphs with directives: {len(dir_glyphs)}/{total} ({100*len(dir_glyphs)/total:.1f}%)")
+
+    opcode_counter = Counter()
+    replace_counts = Counter()
+    illegal_count = 0
+    for g in dir_glyphs:
+        op = g['opcode']
+        opcode_counter[op] += 1
+        if 0x80 <= op <= 0x87:
+            n_replace = op & 0x07
+            replace_counts[n_replace] += 1
+        if op == 255:
+            illegal_count += 1
+
+    if opcode_counter:
+        print(f"\n  By opcode:")
+        for op in sorted(opcode_counter):
+            c = opcode_counter[op]
+            if 0x80 <= op <= 0x87:
+                label = f'replaceWith (n={op & 0x07})'
+            elif op == 255:
+                label = 'ILLEGAL (0xFF)'
+            else:
+                label = f'unknown'
+            print(f"    0x{op:02X} ({label}): {c}")
+
+    if replace_counts:
+        print(f"\n  replaceWith breakdown:")
+        for n in sorted(replace_counts):
+            print(f"    {n} replacement char(s): {replace_counts[n]}")
+
+    if illegal_count:
+        print(f"  Illegal glyphs: {illegal_count}")
+
+    # ---- 3. Kerning shapes ----
+    kern_glyphs = [g for g in all_glyphs if g['has_kern']]
+    print(f"\n{'=' * 60}")
+    print("KERNING SHAPES")
+    print("=" * 60)
+    print(f"  Glyphs with kern data: {len(kern_glyphs)}/{total} ({100*len(kern_glyphs)/total:.1f}%)")
+
+    shape_counter = Counter()
+    for g in kern_glyphs:
+        tag = format_shape(g['kern_mask'], g['is_ytype'])
+        shape_counter[tag] += 1
+
+    n_unique = len(shape_counter)
+    n_kern = len(kern_glyphs)
+    ytype_count = sum(1 for g in kern_glyphs if g['is_ytype'])
+    btype_count = n_kern - ytype_count
+    print(f"  Unique shapes: {n_unique}")
+    print(f"  B-type: {btype_count} ({100*btype_count/n_kern:.1f}%)")
+    print(f"  Y-type: {ytype_count} ({100*ytype_count/n_kern:.1f}%)")
+
+    # Per-bit occurrences
+    bit_names = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K']
+    bit_positions = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14]
+    print(f"\n  Per-bit occurrences ({n_kern} glyphs with kern):")
+    for name, pos in zip(bit_names, bit_positions):
+        c = sum(1 for g in kern_glyphs if (g['kern_mask'] >> pos) & 1)
+        bar = '#' * (c * 30 // n_kern)
+        print(f"    {name}: {c:5d}/{n_kern} ({100*c/n_kern:5.1f}%)  {bar}")
+
+    print(f"\n  Top shapes (of {n_unique} unique):")
+    for tag, c in shape_counter.most_common(30):
+        bar = '#' * (c * 30 // shape_counter.most_common(1)[0][1])
+        print(f"    {tag:<22s} {c:4d} ({100*c/len(kern_glyphs):5.1f}%)  {bar}")
+    if n_unique > 30:
+        remaining = sum(c for _, c in shape_counter.most_common()[30:])
+        print(f"    ... {n_unique - 30} more shapes: {remaining} glyphs")
+
+    # ---- 4. Lowheight ----
+    lowh_glyphs = [g for g in all_glyphs if g['lowheight']]
+    print(f"\n{'=' * 60}")
+    print("LOWHEIGHT")
+    print("=" * 60)
+    print(f"  Lowheight glyphs: {len(lowh_glyphs)}/{total} ({100*len(lowh_glyphs)/total:.1f}%)")
+
+    # ---- 5. Diacritics / stacking ----
+    anchor_glyphs = [g for g in all_glyphs if g['n_anchors'] > 0]
+    wot_glyphs = [g for g in all_glyphs if g['has_write_on_top']]
+    stack_names = {0: 'STACK_UP', 1: 'STACK_DOWN', 2: 'STACK_BEFORE_N_AFTER',
+                   3: 'STACK_UP_N_DOWN', 4: 'STACK_DONT'}
+    stack_counter = Counter(g['stack_where'] for g in all_glyphs if g['stack_where'] != 0)
+
+    print(f"\n{'=' * 60}")
+    print("DIACRITICS & STACKING")
+    print("=" * 60)
+    print(f"  Glyphs with diacritics anchors: {len(anchor_glyphs)}/{total} ({100*len(anchor_glyphs)/total:.1f}%)")
+    anchor_count_dist = Counter(g['n_anchors'] for g in anchor_glyphs)
+    for n in sorted(anchor_count_dist):
+        print(f"    {n} anchor(s): {anchor_count_dist[n]}")
+    print(f"  Glyphs with writeOnTop: {len(wot_glyphs)}")
+    if stack_counter:
+        print(f"  Stack modes:")
+        for sw, c in stack_counter.most_common():
+            print(f"    {stack_names.get(sw, f'?{sw}')}: {c}")
+
+    # ---- 6. Dot removal ----
+    dot_glyphs = [g for g in all_glyphs if g['has_dot_removal']]
+    print(f"\n{'=' * 60}")
+    print("DOT REMOVAL")
+    print("=" * 60)
+    print(f"  Glyphs with dot removal directive: {len(dot_glyphs)}/{total} ({100*len(dot_glyphs)/total:.1f}%)")
+
+    # ---- 7. Nudge ----
+    nudge_glyphs = [g for g in all_glyphs if g['has_nudge']]
+    print(f"\n{'=' * 60}")
+    print("NUDGE")
+    print("=" * 60)
+    print(f"  Glyphs with nudge: {len(nudge_glyphs)}/{total} ({100*len(nudge_glyphs)/total:.1f}%)")
+    if nudge_glyphs:
+        nudge_x_vals = Counter(g['nudge_x'] for g in nudge_glyphs if g['nudge_x'] != 0)
+        nudge_y_vals = Counter(g['nudge_y'] for g in nudge_glyphs if g['nudge_y'] != 0)
+        if nudge_x_vals:
+            print(f"  X nudge values: {dict(sorted(nudge_x_vals.items()))}")
+        if nudge_y_vals:
+            print(f"  Y nudge values: {dict(sorted(nudge_y_vals.items()))}")
+
+    # ---- 8. Alignment ----
+    align_names = {0: 'LEFT', 1: 'RIGHT', 2: 'CENTRE', 3: 'BEFORE'}
+    align_counter = Counter(g['align'] for g in all_glyphs if g['align'] != 0)
+    print(f"\n{'=' * 60}")
+    print("ALIGNMENT")
+    print("=" * 60)
+    if align_counter:
+        for a, c in align_counter.most_common():
+            print(f"  {align_names.get(a, f'?{a}')}: {c}")
+    else:
+        print("  All glyphs use default (LEFT) alignment")
+
+    # ---- 9. Missing kern data ----
+    missing = [g for g in all_glyphs
+               if not g['has_kern']
+               and g['opcode'] == 0
+               and not is_excluded_from_kern(g['code'])]
+    print(f"\n{'=' * 60}")
+    print("MISSING KERNING DATA")
+    print("=" * 60)
+    print(f"  Glyphs without kern (excl. CJK/Hangul/symbols/diacriticals): "
+          f"{len(missing)}/{total} ({100*len(missing)/total:.1f}%)")
+    if missing:
+        by_block = defaultdict(list)
+        for g in missing:
+            by_block[unicode_block_name(g['code'])].append(g['code'])
+        print(f"\n  By block:")
+        for block in sorted(by_block, key=lambda b: by_block[b][0]):
+            cps = by_block[block]
+            sample = ', '.join(f'U+{c:04X}' for c in cps[:8])
+            more = f' ... +{len(cps)-8}' if len(cps) > 8 else ''
+            print(f"    {block}: {len(cps)}  ({sample}{more})")
+
+    # ---- 10. Per-sheet summary ----
+    print(f"\n{'=' * 60}")
+    print("PER-SHEET SUMMARY")
+    print("=" * 60)
+    print(f"  {'Sheet':<52s} {'Total':>5s} {'Kern':>5s} {'LowH':>5s} {'Dir':>4s}")
+    print(f"  {'-'*52} {'-'*5} {'-'*5} {'-'*5} {'-'*4}")
+    for fn in sorted(per_sheet):
+        s = per_sheet[fn]
+        print(f"  {fn:<52s} {s['total']:5d} {s['kern']:5d} {s['lowh']:5d} {s['directives']:4d}")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

Autokem/train.c

@@ -2,6 +2,7 @@
 #include "tga.h"
 #include "nn.h"
 #include "safetensor.h"
+#include "unicode_filter.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -42,7 +43,8 @@ static void extract_shape_bits(int kerning_mask, float *shape) {
 
 /* ---- Collect samples from one TGA ---- */
 
-static int collect_from_sheet(const char *path, int is_xyswap, Sample *samples, int max_samples) {
+static int collect_from_sheet(const char *path, int is_xyswap, int start_code,
+                              Sample *samples, int max_samples) {
     TgaImage *img = tga_read(path);
     if (!img) {
         fprintf(stderr, "Warning: cannot read %s\n", path);
@@ -76,6 +78,10 @@ static int collect_from_sheet(const char *path, int is_xyswap, Sample *samples,
         }
         if (width == 0) continue;
 
+        /* Skip modifier letters, symbols, punctuation */
+        if (start_code >= 0 && is_excluded_from_training(start_code + index))
+            continue;
+
         /* Read kerning data pixel at Y+6 */
         uint32_t kern_pixel = tagify(tga_get_pixel(img, tag_x, tag_y + 6));
         if ((kern_pixel & 0xFF) == 0) continue; /* no kern data */
@@ -170,7 +176,9 @@ int train_model(void) {
         char fullpath[512];
         snprintf(fullpath, sizeof(fullpath), "%s/%s", assets_dir, name);
 
-        int got = collect_from_sheet(fullpath, is_xyswap, all_samples + total, max_total - total);
+        int start_code = sheet_start_code(name);
+        int got = collect_from_sheet(fullpath, is_xyswap, start_code,
+                                     all_samples + total, max_total - total);
         if (got > 0) {
             printf("  %s: %d samples\n", name, got);
             total += got;

Autokem/train_torch.py

@@ -20,10 +20,26 @@ import json
 import os
 import struct
 import sys
+import unicodedata
 from pathlib import Path
 
 import numpy as np
 
+# ---- Sheet code ranges (imported from OTFbuild/sheet_config.py) ----
+
+_otfbuild = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'OTFbuild')
+try:
+    sys.path.insert(0, _otfbuild)
+    from sheet_config import FILE_LIST as _FILE_LIST, CODE_RANGE as _CODE_RANGE
+    sys.path.pop(0)
+    _CODE_RANGE_MAP = {}
+    for _i, _fn in enumerate(_FILE_LIST):
+        if _i < len(_CODE_RANGE):
+            _CODE_RANGE_MAP[_fn] = _CODE_RANGE[_i]
+except ImportError:
+    _CODE_RANGE_MAP = {}
+
+
 # ---- TGA reader (matches OTFbuild/tga_reader.py and Autokem/tga.c) ----
 
 class TgaImage:
@@ -80,7 +96,7 @@ def tagify(pixel):
 
 # ---- Data collection (matches Autokem/train.c) ----
 
-def collect_from_sheet(path, is_xyswap):
+def collect_from_sheet(path, is_xyswap, code_range=None):
     """Extract labelled samples from a single TGA sheet."""
     img = read_tga(path)
     cell_w, cell_h = 16, 20
@@ -90,6 +106,7 @@ def collect_from_sheet(path, is_xyswap):
 
     inputs = []
     labels = []
+    skipped_lm = 0
 
     for index in range(total_cells):
         if is_xyswap:
@@ -110,6 +127,17 @@ def collect_from_sheet(path, is_xyswap):
         if width == 0:
             continue
 
+        # Skip modifier letters, symbols, punctuation
+        if code_range is not None and index < len(code_range):
+            cp = code_range[index]
+            try:
+                cat = unicodedata.category(chr(cp))
+                if cat == 'Lm' or cat[0] in ('S', 'P'):
+                    skipped_lm += 1
+                    continue
+            except (ValueError, OverflowError):
+                pass
+
         # Kern data pixel at Y+6
         kern_pixel = tagify(img.get_pixel(tag_x, tag_y + 6))
         if (kern_pixel & 0xFF) == 0:
@@ -145,7 +173,7 @@ def collect_from_sheet(path, is_xyswap):
         inputs.append(inp)
         labels.append(shape + [is_kern_ytype, is_low_height])
 
-    return inputs, labels
+    return inputs, labels, skipped_lm
 
 
 def collect_all_samples(assets_dir):
@@ -153,6 +181,7 @@ def collect_all_samples(assets_dir):
     all_inputs = []
     all_labels = []
     file_count = 0
+    total_skipped_lm = 0
 
     for name in sorted(os.listdir(assets_dir)):
         if not name.endswith('_variable.tga'):
@@ -161,14 +190,20 @@ def collect_all_samples(assets_dir):
             continue
 
         is_xyswap = 'xyswap' in name
+        code_range = _CODE_RANGE_MAP.get(name, None)
         path = os.path.join(assets_dir, name)
-        inputs, labels = collect_from_sheet(path, is_xyswap)
+        inputs, labels, skipped_lm = collect_from_sheet(path, is_xyswap, code_range)
+        total_skipped_lm += skipped_lm
         if inputs:
-            print(f"  {name}: {len(inputs)} samples")
+            suffix = f" (skipped {skipped_lm})" if skipped_lm else ""
+            print(f"  {name}: {len(inputs)} samples{suffix}")
             all_inputs.extend(inputs)
             all_labels.extend(labels)
             file_count += 1
 
+    if total_skipped_lm:
+        print(f"  Filtered (Lm/S/P): {total_skipped_lm}")
+
     return np.array(all_inputs), np.array(all_labels, dtype=np.float32), file_count
 
 
@@ -191,7 +226,7 @@ def build_model():
             self.conv1 = nn.Conv2d(1, 32, 7, padding=1)
             self.conv2 = nn.Conv2d(32, 64, 7, padding=1)
             self.fc1 = nn.Linear(64, 256)
-            # self.fc2 = nn.Linear(256, 48)
+            # self.fc2 = nn.Linear(256, 128)
             self.output = nn.Linear(256, 12)
             self.tf = nn.SiLU()
 
@@ -306,6 +341,7 @@ def load_safetensors(model, path):
 
 BIT_NAMES = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'Ytype', 'LowH']
 SHAPE_CHARS = 'ABCDEFGHJK'
+MIRROR_PAIRS = [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]  # A↔B, C↔D, E↔F, G↔H, J↔K
 
 
 def format_tag(bits_12):
@@ -359,6 +395,180 @@ def print_examples_and_accuracy(model, X_val, y_val, max_examples=8):
     print(f"  Overall: {total_correct}/{n_val*12} ({100*total_correct/(n_val*12):.2f}%)")
 
 
+# ---- Data augmentation ----
+
+def _shape_key(label):
+    """10-bit shape tuple from label (A through K)."""
+    return tuple(int(label[i]) for i in range(10))
+
+
+def _mirror_shape(key):
+    """Swap mirror pairs: A↔B, C↔D, E↔F, G↔H, J↔K."""
+    m = list(key)
+    for a, b in MIRROR_PAIRS:
+        m[a], m[b] = m[b], m[a]
+    return tuple(m)
+
+
+def _mirror_label(label):
+    """Mirror shape bits in label, keep ytype and lowheight."""
+    m = label.copy()
+    for a, b in MIRROR_PAIRS:
+        m[a], m[b] = m[b], m[a]
+    return m
+
+
+def _shift_image(img, dx, dy):
+    """Shift 2D image by (dx, dy), fill with 0."""
+    h, w = img.shape
+    shifted = np.zeros_like(img)
+    sx0, sx1 = max(0, -dx), min(w, w - dx)
+    sy0, sy1 = max(0, -dy), min(h, h - dy)
+    dx0, dx1 = max(0, dx), min(w, w + dx)
+    dy0, dy1 = max(0, dy), min(h, h + dy)
+    shifted[dy0:dy1, dx0:dx1] = img[sy0:sy1, sx0:sx1]
+    return shifted
+
+
+def _augment_one(img, label, rng):
+    """One augmented copy: random 1px shift + 1% pixel dropout."""
+    dx = rng.integers(-1, 2)  # -1, 0, or 1
+    dy = rng.integers(-1, 2)  # -1, 0, or 1
+    aug = _shift_image(img, dx, dy)
+    # mask = rng.random(aug.shape) > 0.01
+    # aug = aug * mask
+    return aug, label.copy()
+
+
+def _do_mirror_augmentation(X, y, rng):
+    """For each mirror pair (S, mirror(S)), fill deficit from the common side."""
+    shape_counts = {}
+    shape_indices = {}
+    for i in range(len(y)):
+        key = _shape_key(y[i])
+        shape_counts[key] = shape_counts.get(key, 0) + 1
+        shape_indices.setdefault(key, []).append(i)
+
+    new_X, new_y = [], []
+    done = set()  # avoid processing both directions
+
+    for key, count in shape_counts.items():
+        if key in done:
+            continue
+        mkey = _mirror_shape(key)
+        done.add(key)
+        done.add(mkey)
+        if mkey == key:
+            continue  # symmetric shape
+        mcount = shape_counts.get(mkey, 0)
+        if count == mcount:
+            continue
+        # Mirror from the larger side to fill the smaller side
+        if count > mcount:
+            src_key, deficit = key, count - mcount
+        else:
+            src_key, deficit = mkey, mcount - count
+        indices = shape_indices.get(src_key, [])
+        if not indices:
+            continue
+        chosen = rng.choice(indices, size=deficit, replace=True)
+        for idx in chosen:
+            new_X.append(np.fliplr(X[idx]).copy())
+            new_y.append(_mirror_label(y[idx]))
+
+    if new_X:
+        X = np.concatenate([X, np.array(new_X)])
+        y = np.concatenate([y, np.array(new_y)])
+    return X, y
+
+
+def _compute_rarity_weights(y):
+    """Per-sample weight: sum of inverse bit frequencies for all 12 bits.
+
+    Samples with rare bit values (e.g. J=1 at 13%, C=0 at 8%) get higher weight.
+    """
+    bit_freq = y.mean(axis=0)  # [12], P(bit=1)
+    weights = np.zeros(len(y))
+    for i in range(len(y)):
+        w = 0.0
+        for b in range(12):
+            p = bit_freq[b] if y[i, b] > 0.5 else (1.0 - bit_freq[b])
+            w += 1.0 / max(p, 0.01)
+        weights[i] = w
+    return weights
+
+
+def _do_rarity_augmentation(X, y, rng, target_new):
+    """Create target_new augmented samples, drawn proportionally to rarity weight."""
+    if target_new <= 0:
+        return X, y
+
+    weights = _compute_rarity_weights(y)
+    weights /= weights.sum()
+
+    chosen = rng.choice(len(X), size=target_new, replace=True, p=weights)
+
+    new_X, new_y = [], []
+    for idx in chosen:
+        aug_img, aug_label = _augment_one(X[idx], y[idx], rng)
+        new_X.append(aug_img)
+        new_y.append(aug_label)
+
+    X = np.concatenate([X, np.array(new_X)])
+    y = np.concatenate([y, np.array(new_y)])
+    return X, y
+
+
+def _print_bit_freq(y, label):
+    """Print per-bit frequencies for diagnostics."""
+    freq = y.mean(axis=0)
+    names = BIT_NAMES
+    parts = [f'{names[b]}:{freq[b]*100:.0f}%' for b in range(12)]
+    print(f"  {label}: {' '.join(parts)}")
+
+
+def augment_training_data(X_train, y_train, rng, aug_factor=3.0):
+    """
+    Three-phase data augmentation:
+      1. Mirror augmentation — fill deficit between mirror-paired shapes
+      2. Rarity-weighted — samples with rare bit values get more copies (shift+dropout)
+      3. Y-type boost — repeat phases 1-2 scoped to Y-type samples only
+    """
+    n0 = len(X_train)
+    _print_bit_freq(y_train, 'Before')
+
+    # Phase 1: Mirror augmentation
+    X_train, y_train = _do_mirror_augmentation(X_train, y_train, rng)
+    n1 = len(X_train)
+
+    # Phase 2: Rarity-weighted augmentation — target aug_factor × original size
+    target_new = int(n0 * aug_factor) - n1
+    X_train, y_train = _do_rarity_augmentation(X_train, y_train, rng, target_new)
+    n2 = len(X_train)
+
+    # Phase 3: Y-type boost — same pipeline for Y-type subset only
+    ytype_mask = y_train[:, 10] > 0.5
+    n_ytype_existing = int(ytype_mask.sum())
+    if n_ytype_existing > 0:
+        X_yt = X_train[ytype_mask]
+        y_yt = y_train[ytype_mask]
+        X_yt, y_yt = _do_mirror_augmentation(X_yt, y_yt, rng)
+        # Double the Y-type subset via rarity augmentation
+        yt_new = n_ytype_existing
+        X_yt, y_yt = _do_rarity_augmentation(X_yt, y_yt, rng, yt_new)
+        if len(X_yt) > n_ytype_existing:
+            X_train = np.concatenate([X_train, X_yt[n_ytype_existing:]])
+            y_train = np.concatenate([y_train, y_yt[n_ytype_existing:]])
+
+    n3 = len(X_train)
+
+    _print_bit_freq(y_train, 'After ')
+    print(f"Data augmentation: {n0} → {n3} samples ({n3/n0:.1f}×)")
+    print(f"  Mirror: +{n1 - n0}, Rarity: +{n2 - n1}, Y-type boost: +{n3 - n2}")
+
+    return X_train, y_train
+
+
 # ---- Main ----
 
 def main():
@@ -376,6 +586,10 @@ def main():
                         help='Early stopping patience (default: 10)')
     parser.add_argument('--val-split', type=float, default=0.2,
                         help='Validation split (default: 0.2)')
+    parser.add_argument('--no-augment', action='store_true',
+                        help='Disable data augmentation')
+    parser.add_argument('--aug-factor', type=float, default=3.0,
+                        help='Augmentation target multiplier (default: 3.0)')
     args = parser.parse_args()
 
     import torch
@@ -404,10 +618,17 @@ def main():
     perm = rng.permutation(total)
     X, y = X[perm], y[perm]
 
-    n_train = int(total * (1 - args.val_split))
+    n_val = int(total * args.val_split)
+    n_train = total - n_val
     X_train, X_val = X[:n_train], X[n_train:]
     y_train, y_val = y[:n_train], y[n_train:]
-    print(f"Train: {n_train}, Validation: {total - n_train}\n")
+    print(f"Train: {n_train}, Validation: {n_val}")
+
+    # Data augmentation (training set only)
+    if not args.no_augment:
+        X_train, y_train = augment_training_data(X_train, y_train, rng, args.aug_factor)
+        n_train = len(X_train)
+    print()
 
     # Convert to tensors — PyTorch conv expects [N, C, H, W]
     X_train_t = torch.from_numpy(X_train[:, np.newaxis, :, :]).to(device)  # [N,1,20,15]

Autokem/unicode_filter.h

@@ -0,0 +1,191 @@
+#ifndef UNICODE_FILTER_H
+#define UNICODE_FILTER_H
+
+#include <string.h>
+
+/*
+ * Unicode category filters for training/apply.
+ * Generated from Python unicodedata (Unicode 16.0).
+ *
+ * is_modifier_letter(cp)         — category Lm
+ * is_subscript_modifier(cp)      — Lm with <sub> decomposition
+ * is_symbol_or_punctuation(cp)   — categories S* or P*
+ * is_excluded_from_training(cp)  — Lm or S* or P*
+ */
+
+/* ---- Lm (modifier letter) ---- */
+
+static inline int is_modifier_letter(int cp) {
+    if (cp >= 0x02B0 && cp <= 0x02C1) return 1;
+    if (cp >= 0x02C6 && cp <= 0x02D1) return 1;
+    if (cp >= 0x02E0 && cp <= 0x02E4) return 1;
+    if (cp == 0x02EC) return 1;
+    if (cp == 0x02EE) return 1;
+    if (cp == 0x0374) return 1;
+    if (cp == 0x037A) return 1;
+    if (cp == 0x0559) return 1;
+    if (cp == 0x0640) return 1;
+    if (cp >= 0x06E5 && cp <= 0x06E6) return 1;
+    if (cp >= 0x07F4 && cp <= 0x07F5) return 1;
+    if (cp == 0x07FA) return 1;
+    if (cp == 0x081A) return 1;
+    if (cp == 0x0824) return 1;
+    if (cp == 0x0828) return 1;
+    if (cp == 0x08C9) return 1;
+    if (cp == 0x0971) return 1;
+    if (cp == 0x0E46) return 1;
+    if (cp == 0x0EC6) return 1;
+    if (cp == 0x10FC) return 1;
+    if (cp == 0x17D7) return 1;
+    if (cp == 0x1843) return 1;
+    if (cp == 0x1AA7) return 1;
+    if (cp >= 0x1C78 && cp <= 0x1C7D) return 1;
+    if (cp >= 0x1D2C && cp <= 0x1D6A) return 1;
+    if (cp == 0x1D78) return 1;
+    if (cp >= 0x1D9B && cp <= 0x1DBF) return 1;
+    if (cp == 0x2071) return 1;
+    if (cp == 0x207F) return 1;
+    if (cp >= 0x2090 && cp <= 0x209C) return 1;
+    if (cp >= 0x2C7C && cp <= 0x2C7D) return 1;
+    if (cp == 0x2D6F) return 1;
+    if (cp == 0x2E2F) return 1;
+    if (cp == 0x3005) return 1;
+    if (cp >= 0x3031 && cp <= 0x3035) return 1;
+    if (cp == 0x303B) return 1;
+    if (cp >= 0x309D && cp <= 0x309E) return 1;
+    if (cp >= 0x30FC && cp <= 0x30FE) return 1;
+    if (cp == 0xA015) return 1;
+    if (cp >= 0xA4F8 && cp <= 0xA4FD) return 1;
+    if (cp == 0xA60C) return 1;
+    if (cp == 0xA67F) return 1;
+    if (cp >= 0xA69C && cp <= 0xA69D) return 1;
+    if (cp >= 0xA717 && cp <= 0xA71F) return 1;
+    if (cp == 0xA770) return 1;
+    if (cp == 0xA788) return 1;
+    if (cp >= 0xA7F2 && cp <= 0xA7F4) return 1;
+    if (cp >= 0xA7F8 && cp <= 0xA7F9) return 1;
+    if (cp == 0xA9CF) return 1;
+    if (cp == 0xA9E6) return 1;
+    if (cp == 0xAA70) return 1;
+    if (cp == 0xAADD) return 1;
+    if (cp >= 0xAAF3 && cp <= 0xAAF4) return 1;
+    if (cp >= 0xAB5C && cp <= 0xAB5F) return 1;
+    if (cp == 0xAB69) return 1;
+    if (cp == 0xFF70) return 1;
+    if (cp >= 0xFF9E && cp <= 0xFF9F) return 1;
+    if (cp >= 0x10780 && cp <= 0x10785) return 1;
+    if (cp >= 0x10787 && cp <= 0x107B0) return 1;
+    if (cp >= 0x107B2 && cp <= 0x107BA) return 1;
+    if (cp >= 0x16B40 && cp <= 0x16B43) return 1;
+    if (cp >= 0x16F93 && cp <= 0x16F9F) return 1;
+    if (cp >= 0x16FE0 && cp <= 0x16FE1) return 1;
+    if (cp == 0x16FE3) return 1;
+    if (cp >= 0x1AFF0 && cp <= 0x1AFF3) return 1;
+    if (cp >= 0x1AFF5 && cp <= 0x1AFFB) return 1;
+    if (cp >= 0x1AFFD && cp <= 0x1AFFE) return 1;
+    if (cp >= 0x1E030 && cp <= 0x1E06D) return 1;
+    if (cp >= 0x1E137 && cp <= 0x1E13D) return 1;
+    if (cp == 0x1E4EB) return 1;
+    if (cp == 0x1E94B) return 1;
+    return 0;
+}
+
+static inline int is_subscript_modifier(int cp) {
+    if (cp >= 0x1D62 && cp <= 0x1D6A) return 1;
+    if (cp >= 0x2090 && cp <= 0x209C) return 1;
+    if (cp == 0x2C7C) return 1;
+    if (cp >= 0x1E051 && cp <= 0x1E06A) return 1;
+    return 0;
+}
+
+/* ---- S* (Symbol) and P* (Punctuation) ---- */
+
+/* Table of {start, end} ranges for S/P codepoints in font sheets */
+static const int sp_ranges[][2] = {
+    {0x00021, 0x0002F}, {0x0003A, 0x00040}, {0x0005B, 0x00060},
+    {0x0007B, 0x0007E}, {0x000A1, 0x000A9}, {0x000AB, 0x000AC},
+    {0x000AE, 0x000B1}, {0x000B4, 0x000B4}, {0x000B6, 0x000B8},
+    {0x000BB, 0x000BB}, {0x000BF, 0x000BF}, {0x000D7, 0x000D7},
+    {0x000F7, 0x000F7}, {0x002C2, 0x002C5}, {0x002D2, 0x002DF},
+    {0x002E5, 0x002EB}, {0x002ED, 0x002ED}, {0x002EF, 0x002FF},
+    {0x00375, 0x00375}, {0x0037E, 0x0037E}, {0x00384, 0x00385},
+    {0x00387, 0x00387}, {0x00482, 0x00482}, {0x0055A, 0x0055F},
+    {0x00589, 0x0058A}, {0x0058D, 0x0058F}, {0x00964, 0x00965},
+    {0x00970, 0x00970}, {0x009F2, 0x009F3}, {0x009FA, 0x009FB},
+    {0x009FD, 0x009FD}, {0x00BF3, 0x00BFA}, {0x00E3F, 0x00E3F},
+    {0x00E4F, 0x00E4F}, {0x00E5A, 0x00E5B}, {0x010FB, 0x010FB},
+    {0x016EB, 0x016ED}, {0x01CC0, 0x01CC7}, {0x01FBD, 0x01FBD},
+    {0x01FBF, 0x01FC1}, {0x01FCD, 0x01FCF}, {0x01FDD, 0x01FDF},
+    {0x01FED, 0x01FEF}, {0x01FFD, 0x01FFE}, {0x02010, 0x02027},
+    {0x02030, 0x0205E}, {0x0207A, 0x0207E}, {0x0208A, 0x0208E},
+    {0x020A0, 0x020C0}, {0x02100, 0x02101}, {0x02103, 0x02106},
+    {0x02108, 0x02109}, {0x02114, 0x02114}, {0x02116, 0x02118},
+    {0x0211E, 0x02123}, {0x02125, 0x02125}, {0x02127, 0x02127},
+    {0x02129, 0x02129}, {0x0212E, 0x0212E}, {0x0213A, 0x0213B},
+    {0x02140, 0x02144}, {0x0214A, 0x0214D}, {0x0214F, 0x0214F},
+    {0x0218A, 0x0218B}, {0x02190, 0x021FF}, {0x02400, 0x02426},
+    {0x02800, 0x028FF}, {0x03001, 0x03004}, {0x03008, 0x03020},
+    {0x03030, 0x03030}, {0x03036, 0x03037}, {0x0303D, 0x0303F},
+    {0x0309B, 0x0309C}, {0x030A0, 0x030A0}, {0x030FB, 0x030FB},
+    {0x04DC0, 0x04DFF}, {0x0A673, 0x0A673}, {0x0A67E, 0x0A67E},
+    {0x0A720, 0x0A721}, {0x0A789, 0x0A78A}, {0x0AB5B, 0x0AB5B},
+    {0x0AB6A, 0x0AB6B}, {0x0FF01, 0x0FF0F}, {0x0FF1A, 0x0FF20},
+    {0x0FF3B, 0x0FF40}, {0x0FF5B, 0x0FF65}, {0x0FFE0, 0x0FFE6},
+    {0x0FFE8, 0x0FFEE}, {0x0FFFC, 0x0FFFD}, {0x1F10D, 0x1F1AD},
+    {0x1F1E6, 0x1F1FF}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA},
+};
+
+static inline int is_symbol_or_punctuation(int cp) {
+    int n = (int)(sizeof(sp_ranges) / sizeof(sp_ranges[0]));
+    for (int i = 0; i < n; i++) {
+        if (cp >= sp_ranges[i][0] && cp <= sp_ranges[i][1])
+            return 1;
+    }
+    return 0;
+}
+
+/* ---- Combined filter for training exclusion ---- */
+
+static inline int is_excluded_from_training(int cp) {
+    return is_modifier_letter(cp) || is_symbol_or_punctuation(cp);
+}
+
+/* ---- Sheet filename → start codepoint ---- */
+
+static int sheet_start_code(const char *basename) {
+    if (strstr(basename, "ascii_variable"))                return 0x00;
+    if (strstr(basename, "latinExtA_variable"))            return 0x100;
+    if (strstr(basename, "latinExtB_variable"))            return 0x180;
+    if (strstr(basename, "cyrilic_extC_variable"))         return 0x1C80;
+    if (strstr(basename, "cyrilic_extB_variable"))         return 0xA640;
+    if (strstr(basename, "cyrilic_bulgarian_variable"))    return 0xF0000;
+    if (strstr(basename, "cyrilic_serbian_variable"))      return 0xF0060;
+    if (strstr(basename, "cyrilic_variable"))              return 0x400;
+    if (strstr(basename, "halfwidth_fullwidth_variable"))  return 0xFF00;
+    if (strstr(basename, "unipunct_variable"))             return 0x2000;
+    if (strstr(basename, "greek_polytonic"))               return 0x1F00;
+    if (strstr(basename, "greek_variable"))                return 0x370;
+    if (strstr(basename, "thai_variable"))                 return 0xE00;
+    if (strstr(basename, "hayeren_variable"))              return 0x530;
+    if (strstr(basename, "kartuli_allcaps_variable"))      return 0x1C90;
+    if (strstr(basename, "kartuli_variable"))              return 0x10D0;
+    if (strstr(basename, "ipa_ext_variable"))              return 0x250;
+    if (strstr(basename, "latinExt_additional_variable"))  return 0x1E00;
+    if (strstr(basename, "tsalagi_variable"))              return 0x13A0;
+    if (strstr(basename, "phonetic_extensions_variable"))  return 0x1D00;
+    if (strstr(basename, "latinExtC_variable"))            return 0x2C60;
+    if (strstr(basename, "latinExtD_variable"))            return 0xA720;
+    if (strstr(basename, "internal_variable"))             return 0xFFE00;
+    if (strstr(basename, "letterlike_symbols_variable"))   return 0x2100;
+    if (strstr(basename, "enclosed_alphanumeric"))         return 0x1F100;
+    if (strstr(basename, "sundanese_variable"))            return 0x1B80;
+    if (strstr(basename, "control_pictures_variable"))     return 0x2400;
+    if (strstr(basename, "latinExtE_variable"))            return 0xAB30;
+    if (strstr(basename, "latinExtF_variable"))            return 0x10780;
+    if (strstr(basename, "latinExtG_variable"))            return 0x1DF00;
+    if (strstr(basename, "devanagari") && !strstr(basename, "internal"))
+                                                           return 0x900;
+    return -1;
+}
+
+#endif /* UNICODE_FILTER_H */

OTFbuild/font_builder.py

@@ -28,7 +28,7 @@ from keming_machine import generate_kerning_pairs
 from opentype_features import generate_features, glyph_name
 import sheet_config as SC
 
-FONT_VERSION = "1.15"
+FONT_VERSION = "1.16"
 
 # Codepoints that get cmap entries (user-visible)
 # PUA forms used internally by GSUB get glyphs but NO cmap entries

OTFbuild/hangul.py

@@ -16,8 +16,8 @@ import sheet_config as SC
 
 # PUA range for Hangul jamo variant storage.
 # We need space for: max_col * max_row variants.
-# Using 0xF0600-0xF1E7F
-HANGUL_PUA_BASE = 0xF0600
+# Using 0x100000-0x10187F
+HANGUL_PUA_BASE = 0x100000
 
 
 def _compose_bitmaps(a, b, w, h):

OTFbuild/opentype_features.py

@@ -273,7 +273,7 @@ def _generate_hangul_gsub(glyphs, has, jamo_data):
             continue
         for f in [0, 1]:
             try:
-                row_ng = SC.get_han_initial_row(1, idx, f)
+                row_ng = SC.get_han_initial_row(2, idx, f)
             except (ValueError, KeyError):
                 continue
             jung_groups_general.setdefault((row_ng, f), []).append(jcp)

OTFbuild/sheet_config.py

@@ -75,6 +75,13 @@ SHEET_LEGACY_COMPUTING_VARW = 41
 SHEET_CYRILIC_EXTB_VARW = 42
 SHEET_CYRILIC_EXTA_VARW = 43
 SHEET_CYRILIC_EXTC_VARW = 44
+SHEET_LATIN_EXTE_VARW = 45
+SHEET_LATIN_EXTF_VARW = 46
+SHEET_LATIN_EXTG_VARW = 47
+SHEET_OGHAM_VARW = 48
+SHEET_COPTIC_VARW = 49
+SHEET_CYRILIC_EXTD_VARW = 50
+SHEET_MATHS1_VARW = 51
 
 SHEET_UNKNOWN = 254
 
@@ -124,6 +131,13 @@ FILE_LIST = [
     "cyrilic_extB_variable.tga",
     "cyrilic_extA_variable.tga",
     "cyrilic_extC_variable.tga",
+    "latinExtE_variable.tga",
+    "latinExtF_variable.tga",
+    "latinExtG_variable.tga",
+    "ogham_variable.tga",
+    "coptic_variable.tga",
+    "cyrilic_extD_variable.tga",
+    "maths1_extrawide_variable.tga",
 ]
 
 CODE_RANGE = [
@@ -137,7 +151,7 @@ CODE_RANGE = [
     list(range(0x400, 0x530)),                                                      # 7: Cyrillic
     list(range(0xFF00, 0x10000)),                                                   # 8: Halfwidth/Fullwidth
     list(range(0x2000, 0x20A0)),                                                    # 9: Uni Punct
-    list(range(0x370, 0x3CF)),                                                      # 10: Greek
+    list(range(0x370, 0x400)),                                                      # 10: Greek
     list(range(0xE00, 0xE60)),                                                      # 11: Thai
     list(range(0x530, 0x590)),                                                      # 12: Armenian
     list(range(0x10D0, 0x1100)),                                                    # 13: Georgian
@@ -167,11 +181,18 @@ CODE_RANGE = [
     list(range(0xF0520, 0xF0580)),                                                  # 37: Codestyle ASCII
     list(range(0xFB00, 0xFB18)),                                                    # 38: Alphabetic Presentation
     list(range(0x1B000, 0x1B170)),                                                  # 39: Hentaigana
-    list(range(0x2400, 0x2440)),                                                    # 40: Control Pictures
+    list(range(0x2400, 0x2450)),                                                    # 40: Control Pictures
     list(range(0x1FB00, 0x1FC00)),                                                  # 41: Legacy Computing
     list(range(0xA640, 0xA6A0)),                                                    # 42: Cyrillic Ext B
     list(range(0x2DE0, 0x2E00)),                                                    # 43: Cyrillic Ext A
-    list(range(0x1C80, 0x1C8F)),                                                    # 43: Cyrillic Ext C
+    list(range(0x1C80, 0x1C8F)),                                                    # 44: Cyrillic Ext C
+    list(range(0xAB30, 0xAB70)),                                                    # 45: Latin Ext E
+    list(range(0x10780, 0x107C0)),                                                  # 46: Latin Ext F
+    list(range(0x1DF00, 0x1E000)),                                                  # 47: Latin Ext G
+    list(range(0x1680, 0x16A0)),                                                    # 48: Ogham
+    list(range(0x2C80, 0x2D00)),                                                    # 49: Coptic
+    list(range(0x1E030, 0x1E090)),                                                  # 50: Cyrillic Ext D
+    list(range(0x2200, 0x2300)),                                                    # 51: Maths1
 ]
 
 CODE_RANGE_HANGUL_COMPAT = range(0x3130, 0x3190)
@@ -551,5 +572,12 @@ def index_y(sheet_index, c):
         SHEET_CYRILIC_EXTB_VARW: lambda: (c - 0xA640) // 16,
         SHEET_CYRILIC_EXTA_VARW: lambda: (c - 0x2DE0) // 16,
         SHEET_CYRILIC_EXTC_VARW: lambda: (c - 0x1C80) // 16,
+        SHEET_LATIN_EXTE_VARW: lambda: (c - 0xAB30) // 16,
+        SHEET_LATIN_EXTF_VARW: lambda: (c - 0x10780) // 16,
+        SHEET_LATIN_EXTG_VARW: lambda: (c - 0x1DF00) // 16,
+        SHEET_OGHAM_VARW: lambda: (c - 0x1680) // 16,
+        SHEET_COPTIC_VARW: lambda: (c - 0x2C80) // 16,
+        SHEET_CYRILIC_EXTD_VARW: lambda: (c - 0x1E030) // 16,
+        SHEET_MATHS1_VARW: lambda: (c - 0x2200) // 16,
         SHEET_HANGUL: lambda: 0,
     }.get(sheet_index, lambda: c // 16)()

demotext_unaligned.txt

@@ -26,21 +26,22 @@ How multilingual? Real multilingual!
 􏻬󿿁Под южно дърво, цъфтящо в синьо, бягаше малко пухкаво зайче󿿀􀀀
 􏻬ᎠᏍᎦᏯᎡᎦᎢᎾᎨᎢᎣᏍᏓᎤᎩᏍᏗᎥᎴᏓᎯᎲᎢᏔᎵᏕᎦᏟᏗᏖᎸᎳᏗᏗᎧᎵᎢᏘᎴᎩ ᏙᏱᏗᏜᏫᏗᏣᏚᎦᏫᏛᏄᏓᎦᏝᏃᎠᎾᏗᎭᏞᎦᎯᎦᏘᏓᏠᎨᏏᏕᏡᎬᏢᏓᏥᏩᏝᎡᎢᎪᎢ ᎠᎦᏂᏗᎮᎢᎫᎩᎬᏩᎴᎢᎠᏆᏅᏛᎫᏊᎾᎥᎠᏁᏙᎲᏐᏈᎵᎤᎩᎸᏓᏭᎷᏤᎢᏏᏉᏯᏌᏊ ᎤᏂᏋᎢᏡᎬᎢᎰᏩᎬᏤᎵᏍᏗᏱᎩᎱᎱᎤᎩᎴᎢᏦᎢᎠᏂᏧᏣᏨᎦᏥᎪᎥᏌᏊᎤᎶᏒᎢᎢᏡᎬᎢ ᎹᎦᎺᎵᏥᎻᎼᏏᎽᏗᏩᏂᎦᏘᎾᎿᎠᏁᎬᎢᏅᎩᎾᏂᎡᎢᏌᎶᎵᏎᎷᎠᏑᏍᏗᏪᎩ ᎠᎴ ᏬᏗᏲᏭᎾᏓᏍᏓᏴᏁᎢᎤᎦᏅᏮᏰᎵᏳᏂᎨᎢ􀀀
 􏻬Ѳеѡфа́нъ и҆ Алеѯі́й, ѕѣлѡ̀ возлюби́вше ѱалти́рь, воспѣ́ша при свѣ́тѣ ѕвѣ́здъ, помазꙋ́юще сщ҃е́нное мѵ́ро; серафими мн̑оꙮ҆читїи̑, ꙗ҆́кѡ ѻ҆́гнь, ѡ҆крꙋжа́хꙋ прⷭ҇то́лъ Бж҃їй, и҆ всѧ̀ землѧ̀ и҆спо́лнисѧ свѣ́та, ꙗ҆́кѡ ѕмі́й попра́нъ є҆́сть􀀀
+􏻬ⲡⲓⲝⲉⲛⲟⲥ ⲅⲁⲣ ⲁϥϫⲉⲙ ⲟⲩⲫⲱⲥ ϧⲉⲛ ⲡⲓⲍⲏⲗⲟⲥ ⲛⲧⲉ ϯⲯⲩⲭⲏ· ⲁϥϣⲱⲡⲓ ⲇⲉ ⲕⲁⲧⲁ ⲡⲓⲑⲉⲗⲏⲙⲁ· ⲁϥϭⲓ ⲛϩⲱⲃ ⲛⲓⲃⲉⲛ ⲟⲩⲟϩ ⲁϥϯⲙⲟⲧ􀀀
 􏻬Příliš žluťoučký kůň úpěl ďábelské ódy􀀀
 􏻬Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen Walther spillede på xylofon􀀀
-􏻬PACK MY BOX WITH FIVE DOZEN LIQUOR JUGS􀀀
+􏻬Sphinx of black quartz, judge my vow􀀀
 􏻬hƿæt ƿe ᵹardena inᵹear ꝺaᵹum þeoꝺ cynninᵹa þꞃym ᵹeꝼꞃumon􀀀
-􏻬Victor jagt zwölf Boxkämpfer quer über den großen Sylter Deich  GROẞEN GROẞE􀀀
+􏻬Victor jagt zwölf Boxkämpfer quer über den GROẞEN Sylter Deich􀀀
 􏻬ζαφείρι δέξου πάγκαλο, βαθῶν ψυχῆς τὸ σῆμα􀀀
-􏻬ΔΙΑΦΥΛΆΞΤΕ ΓΕΝΙΚΆ ΤΗ ΖΩΉ ΣΑΣ ΑΠΌ ΒΑΘΕΙΆ ΨΥΧΙΚΆ ΤΡΑΎΜΑΤΑ􀀀
 􏻬სწრაფი ყავისფერი მელა გადაახტა ზარმაც ძაღლს  ᲘᲜᲢᲔᲚ ᲞᲔᲜᲢᲘᲣᲛᲘ ᲛᲘᲙᲠᲝᲞᲠᲝᲪᲔᲡᲝᲠᲘ􀀀
 􏻬ऋषियों को सताने वाले दुष्ट राक्षसों के राजा रावण का सर्वनाश करने वाले विष्णुवतार भगवान श्रीराम अयोध्या के महाराज दशरथ के􀀀
 􏻬Kæmi ný öxi hér, ykist þjófum nú bæði víl og ádrepa􀀀
-􏻬Ċuaiġ bé ṁórṡáċ le dlúṫspád fíoꝛḟinn trí hata mo ḋea-ṗoꝛcáin ḃig􀀀
+􏻬Scríoḃ Fergus ⁊ a ṁáṫaır dán le peann úr􀀀
 􏻬あめつちほしそら やまかはみねたに くもきりむろこけ ひといぬうへすゑ ゆわさるおふせよ えの𛀁をなれゐて􀀀
 􏻬トリナクコヱス ユメサマセ ミヨアケワタル ヒンカシヲ ソライロハエテ オキツヘニ ホフネムレヰヌ モヤノウチ􀀀
 􏻬田居に出で 菜摘むわれをぞ 君召すと 求食り追ひゆく 山城の 打酔へる子ら 藻葉干せよ え舟繋けぬ􀀀
-􏻬정 참판 양반댁 규수 큰 교자 타고 혼례 치른 날　　찦차를 타고 온 펲시맨과 쑛다리 똠방각하􀀀
+􏻬콩고물과 우유가 들어간 빙수는 차게 먹어야 특별한 맛이 잘 표현된다　유쾌했던 땃쥐 토끼풀 쫓기 바쁨􀀀
+􏻬찦차를 타고 온 펲시맨과 쑛다리 똠방각하　왜날뷁􀀀
 􏻬쾅 ᄒᆞ는 소리 헨 “아이구 베락 털어져ᇝ인가?” 영 걷어진 쥥은 몰르곡 경헨 ᄇᆞᆰ도록 ᄌᆞᆷ ᄒᆞᆫᄌᆞᆷ들 안 잣수다􀀀
 􏻬Četri psihi faķīri vēlu vakarā zāģēja guļbūvei durvis, fonā šņācot mežam􀀀
 􏻬Įlinkdama fechtuotojo špaga sublykčiojusi pragręžė apvalų arbūzą􀀀
@@ -58,7 +59,7 @@ How multilingual? Real multilingual!
 􏻬Pijamalı hasta yağız şoföre çabucak güvendi􀀀
 􏻬Жебракують філософи при ґанку церкви в Гадячі, ще й шатро їхнє п’яне знаємо􀀀
 􏻬Do bạch kim rất quý nên sẽ dùng để lắp vô xương􀀀
-􏻬日堀油告観観藤村抄海評業庁経賃室弁市。太撮収改売週法所何都慣次現。価紙一無三洋日話転手治稿載末替付致治。􀀀
+􏻬日堀油告観観藤村抄海評業庁経賃室弁市。太撮収改売週法所何都慣次現。􀀀
 􏻬[pʰnɣɬɥi.m͡ŋχɫʍɨnaɸ.cθʊɫɯ.ɹɨɫʏ͡ɛx.ɯ͡ɣaxɲaɣɫ.ɸtʰɑɣɴ]􀀀
 􏻬⠑⠥⠊⠵⠀⠟⠫⠒⠵⠀⠓⠗⠎⠉⠂⠀⠠⠊⠗⠘⠍⠓⠎⠀⠨⠣⠩⠐⠥⠍⠑⠱⠀⠈⠪⠀⠨⠷⠎⠢⠈⠧⠀⠈⠏⠒⠐⠕⠝⠀⠕⠌⠎⠀⠊⠿⠊⠪⠶⠚⠊􀀀
 
@@ -97,7 +98,7 @@ How multilingual? Real multilingual!
 􎳌‣ Unicode fractions, also known as super/subscripts􀀀
 
 􏻬ᄀᆞᄅᆞᇝ ᄀᆞᅀᅢ 자거늘 밀므리 사ᄋᆞ리로ᄃᆡ 나거ᅀᅡ ᄌᆞᄆᆞ니ᅌᅵ다 셤 안해 자시ᇙ 제 한비 사ᄋᆞ리로ᄃᆡ 뷔어ᅀᅡ ᄌᆞᄆᆞ니ᅌᅵ다􀀀
-􏻬쾅 ᄒᆞ는 소리 헨 “아이구, 베락 털어져ᇝ인가?” 영 걷어진 쥥은 몰르곡 경헨 나왕 보고들랑 영헤연 ᄇᆞᆰ도록 ᄌᆞᆷ ᄒᆞᆫᄌᆞᆷ들 안 잣수다 이 시간 동네 사람들.􀀀
+􏻬쾅 ᄒᆞ는 소리 헨 “아이구, 베락 털어져ᇝ인가?” 영 걷어진 쥥은 몰르곡 경헨 나왕 보고들랑 영헤연 ᄇᆞᆰ도록 ᄌᆞᆷ ᄒᆞᆫᄌᆞᆷ들 안 잣수다􀀀
 
 􎳌‣ Full support for Old Korean/Jeju dialect orthography􀀀
 
@@ -114,27 +115,28 @@ How multilingual? Real multilingual!
 ⁃ Basic Latin
 ⁃ Latin-1 Supplement
 ⁃ Latin Extended Additional
-⁃ Latin Extended-A/B/C/D
+⁃ Latin Extended-A/B/C/D/E/F/G
 ⁃ Armenian
 ⁃ Arrows
 ⁃ Bengali􏿆ᶠⁱ􀀀
 ⁃ Braille Patterns
-⁃ Cherokee􏿆⁷􀀀
+⁃ Cherokee􏿆ᴬ􀀀
 ⁃ CJK Symbols and Punctuation
-⁃ CJK Unified Ideographs􏿆⁶􀀀
-⁃ CJK Unified Ideographs Extension A􏿆¹²·¹􀀀
+⁃ CJK Unified Ideographs
+⁃ CJK Unified Ideographs Extension A
 ⁃ Combining Diacritical Marks
 ⁃ Control Pictures
+⁃ Coptic
 ⁃ Currency Symbols
 ⁃ Cyrillic
 ⁃ Cyrillic Supplement
-⁃ Cyrillic Extended-A/B/C
+⁃ Cyrillic Extended-A/B/C/D
 ⁃ Devanagari
 ⁃ Enclosed Alphanumeric Supplement
 ⁃ General Punctuations
 ⁃ Georgian􏿆ჼ􀀀
 ⁃ Georgian Extended
-⁃ Greek and Coptic􏿆ᴱ􀀀
+⁃ Greek and Coptic
 ⁃ Greek Extended
 ⁃ Halfwidth and Fullwidth Forms
 ⁃ Hangul Compatibility Jamo
@@ -147,9 +149,12 @@ How multilingual? Real multilingual!
 ⁃ Katakana Phonetic Extensions
 ⁃ Kana Supplement
 ⁃ Kana Extended-A
-⁃ Number Forms
 ⁃ Small Kana Extension
 ⁃ Letterlike Symbols
+⁃ Mathematical Operators
+⁃ Number Forms
+⁃ Ogham
+⁃ Optical Character Recognition
 ⁃ Phonetic Extensions
 ⁃ Phonetic Extensions Supplement
 ⁃ Runic
@@ -160,9 +165,9 @@ How multilingual? Real multilingual!
 ⁃ Symbols for Legacy Computing
 ⁃ Tamil
 ⁃ Thai
+⁃ Yijing Hexagram Symbols
 
-  􏿆ᴱ􀀀  No support for Coptic
-  􏿆ᶠⁱ􀀀  No support for ligatures　　􏿆ჼ􀀀  Mkhedruli only
-  􏿆⁶􀀀  􏿆⁷􀀀  􏿆⁹􀀀  􏿆¹²·¹􀀀  Up to the specified Unicode version
+  􏿆ᶠⁱ􀀀  No support for ligatures
+  􏿆ᴬ􀀀   Uppercase only     􏿆ჼ􀀀  Mkhedruli only
 
 GitHub’s issue page is open! You can report any 􏽕errors􀀀, or leave 􏽕suggestions􀀀. You can help this font to be more versatile. (for more languages, more frameworks) 􏽕Clone􀀀 this repo, make changes, and make a 􏽕pull request􀀀! I appreciate any and all supports.

keming_calculator.html

@@ -579,15 +579,15 @@ const EXAMPLES = [
     { zones: 'BDFGH',     wye: false, chars: '\u027A',     desc: 'BDFGH' },
     { zones: 'BG',        wye: true,  chars: '/',          desc: 'BG(Y)' },
     { zones: 'CD',        wye: false, chars: '\u10B5',     desc: 'CD' },
-    { zones: 'CDEF',      wye: true,  chars: '\u03A6',     desc: 'CDEF(Y)' },
-    { zones: 'CDEFGH',    wye: false, chars: 'a,c,e',     desc: 'CDEFGH' },
+    { zones: 'CDEF',      wye: true,  chars: '\u03A6,v',     desc: 'CDEF(Y)' },
+    { zones: 'CDEFGH',    wye: false, chars: 'a,e',     desc: 'CDEFGH' },
     { zones: 'CDEFGHJK',  wye: false, chars: 'g',          desc: 'CDEFGHJK' },
     { zones: 'CDEFGHK',   wye: false, chars: '\u019E',     desc: 'CDEFGHK' },
+    { zones: 'CDEFGH',    wye: true,  chars: 'A',          desc: 'CDEFGH(Y)' },
+    { zones: 'CDEGH',     wye: false,  chars: 'c',          desc: 'CDEGH' },
     { zones: 'AB',        wye: true,  chars: 'Y',          desc: 'AB(Y)' },
     { zones: 'ABCD',      wye: true,  chars: 'V',          desc: 'ABCD(Y)' },
-    { zones: 'CDEF',      wye: true,  chars: 'v',          desc: 'CDEF(Y)' },
     { zones: 'EFGH',      wye: true,  chars: '\u028C',     desc: 'EFGH(Y)' },
-    { zones: 'CDEFGH',    wye: true,  chars: 'A',          desc: 'CDEFGH(Y)' },
 ];
 
 function buildExamples() {

src/net/torvald/terrarumsansbitmap/gdx/GlyphAtlas.kt

@@ -0,0 +1,134 @@
+package net.torvald.terrarumsansbitmap.gdx
+
+import com.badlogic.gdx.graphics.Pixmap
+
+data class AtlasRegion(
+    val atlasX: Int,
+    val atlasY: Int,
+    val width: Int,
+    val height: Int,
+    val offsetX: Int = 0,
+    val offsetY: Int = 0
+)
+
+class GlyphAtlas(val atlasWidth: Int, val atlasHeight: Int) {
+
+    val pixmap = Pixmap(atlasWidth, atlasHeight, Pixmap.Format.RGBA8888).also { it.blending = Pixmap.Blending.SourceOver }
+
+    private val regions = HashMap<Long, AtlasRegion>()
+
+    private var cursorX = 0
+    private var cursorY = 0
+    private var shelfHeight = 0
+
+    private val pendingCells = ArrayList<PendingCell>()
+
+    private class PendingCell(
+        val sheetID: Int,
+        val cellX: Int,
+        val cellY: Int,
+        val cropped: Pixmap,
+        val offsetX: Int,
+        val offsetY: Int
+    )
+
+    private fun atlasKey(sheetID: Int, cellX: Int, cellY: Int): Long =
+        sheetID.toLong().shl(32) or cellX.toLong().shl(16) or cellY.toLong()
+
+    /** Scans the cell for its non-transparent bounding box, crops, and queues for deferred packing. */
+    fun queueCell(sheetID: Int, cellX: Int, cellY: Int, cellPixmap: Pixmap) {
+        var minX = cellPixmap.width
+        var minY = cellPixmap.height
+        var maxX = -1
+        var maxY = -1
+
+        for (y in 0 until cellPixmap.height) {
+            for (x in 0 until cellPixmap.width) {
+                if (cellPixmap.getPixel(x, y) and 0xFF != 0) {
+                    if (x < minX) minX = x
+                    if (y < minY) minY = y
+                    if (x > maxX) maxX = x
+                    if (y > maxY) maxY = y
+                }
+            }
+        }
+
+        if (maxX < 0) return // entirely transparent, skip
+
+        val cropW = maxX - minX + 1
+        val cropH = maxY - minY + 1
+        val cropped = Pixmap(cropW, cropH, Pixmap.Format.RGBA8888)
+        cropped.drawPixmap(cellPixmap, 0, 0, minX, minY, cropW, cropH)
+
+        pendingCells.add(PendingCell(sheetID, cellX, cellY, cropped, minX, minY))
+    }
+
+    /** Sorts queued cells by height desc then width desc, and packs into shelves. */
+    fun packAllQueued() {
+        pendingCells.sortWith(
+            compareByDescending<PendingCell> { it.cropped.height }
+                .thenByDescending { it.cropped.width }
+        )
+
+        for (cell in pendingCells) {
+            val w = cell.cropped.width
+            val h = cell.cropped.height
+
+            // start new shelf if cell doesn't fit horizontally
+            if (cursorX + w > atlasWidth) {
+                cursorX = 0
+                cursorY += shelfHeight
+                shelfHeight = 0
+            }
+
+            pixmap.drawPixmap(cell.cropped, cursorX, cursorY)
+
+            regions[atlasKey(cell.sheetID, cell.cellX, cell.cellY)] =
+                AtlasRegion(cursorX, cursorY, w, h, cell.offsetX, cell.offsetY)
+
+            cursorX += w
+            if (h > shelfHeight) shelfHeight = h
+
+            cell.cropped.dispose()
+        }
+
+        pendingCells.clear()
+    }
+
+    fun blitSheet(sheetID: Int, sheetPixmap: Pixmap, cellW: Int, cellH: Int, cols: Int, rows: Int) {
+        if (cursorX > 0) {
+            cursorX = 0
+            cursorY += shelfHeight
+            shelfHeight = 0
+        }
+
+        val baseY = cursorY
+
+        pixmap.drawPixmap(sheetPixmap, 0, baseY)
+
+        for (cy in 0 until rows) {
+            for (cx in 0 until cols) {
+                regions[atlasKey(sheetID, cx, cy)] = AtlasRegion(cx * cellW, baseY + cy * cellH, cellW, cellH)
+            }
+        }
+
+        cursorY = baseY + sheetPixmap.height
+        cursorX = 0
+        shelfHeight = 0
+    }
+
+    fun getRegion(sheetID: Int, cellX: Int, cellY: Int): AtlasRegion? =
+        regions[atlasKey(sheetID, cellX, cellY)]
+
+    fun clearRegion(region: AtlasRegion) {
+        for (y in 0 until region.height) {
+            for (x in 0 until region.width) {
+                pixmap.drawPixel(region.atlasX + x, region.atlasY + y, 0)
+            }
+        }
+    }
+
+    fun dispose() {
+        pixmap.dispose()
+    }
+}

src/net/torvald/terrarumsansbitmap/gdx/TerrarumSansBitmap.kt

@@ -318,7 +318,7 @@ class TerrarumSansBitmap(
     /** Props of all printable Unicode points. */
     private val glyphProps = HashMap<CodePoint, GlyphProps>()
     private val textReplaces = HashMap<CodePoint, CodePoint>()
-    private val sheets: Array<PixmapRegionPack>
+    private lateinit var atlas: GlyphAtlas
 
 //    private var charsetOverride = 0
 
@@ -326,9 +326,10 @@ class TerrarumSansBitmap(
 //    private val tempFiles = ArrayList<String>()
 
     init {
-        val sheetsPack = ArrayList<PixmapRegionPack>()
+        atlas = GlyphAtlas(4096, 4096)
+        var unihanPixmap: Pixmap? = null
 
-        // first we create pixmap to read pixels, then make texture using pixmap
+        // first we create pixmap to read pixels, then pack into atlas
         fileList.forEachIndexed { index, it ->
             val isVariable = it.endsWith("_variable.tga")
             val isXYSwapped = it.contains("xyswap", true)
@@ -346,32 +347,6 @@ class TerrarumSansBitmap(
             else
                 dbgprn("loading texture [STATIC] $it")
 
-
-            // unpack gz if applicable
-            /*if (it.endsWith(".gz")) {
-                val tmpFilePath = tempDir + "/tmp_${it.dropLast(7)}.tga"
-
-                try {
-                    val gzi = GZIPInputStream(Gdx.files.classpath(fontParentDir + it).read(8192))
-                    val wholeFile = gzi.readBytes()
-                    gzi.close()
-                    val fos = BufferedOutputStream(FileOutputStream(tmpFilePath))
-                    fos.write(wholeFile)
-                    fos.flush()
-                    fos.close()
-
-                    pixmap = Pixmap(Gdx.files.absolute(tmpFilePath))
-//                    tempFiles.add(tmpFilePath)
-                }
-                catch (e: GdxRuntimeException) {
-                    //e.printStackTrace()
-                    dbgprn("said texture not found, skipping...")
-
-                    pixmap = Pixmap(1, 1, Pixmap.Format.RGBA8888)
-                }
-                //File(tmpFileName).delete()
-            }
-            else {*/
                 try {
                     pixmap = Pixmap(Gdx.files.classpath("assets/$it"))
                 }
@@ -387,7 +362,6 @@ class TerrarumSansBitmap(
                         System.exit(1)
                     }
                 }
-            //}
 
             if (isVariable) buildWidthTable(pixmap, codeRange[index], if (isExtraWide) 32 else 16)
             buildWidthTableFixed()
@@ -395,40 +369,63 @@ class TerrarumSansBitmap(
 
             setupDynamicTextReplacer()
 
-            /*if (!noShadow) {
-                makeShadowForSheet(pixmap)
-            }*/
+            if (index == SHEET_UNIHAN) {
+                // defer wenquanyi packing to after all other sheets
+                unihanPixmap = pixmap
+            }
+            else {
+                val texRegPack = if (isExtraWide)
+                    PixmapRegionPack(pixmap, W_WIDEVAR_INIT, H, HGAP_VAR, 0, xySwapped = isXYSwapped)
+                else if (isVariable)
+                    PixmapRegionPack(pixmap, W_VAR_INIT, H, HGAP_VAR, 0, xySwapped = isXYSwapped)
+                else if (index == SHEET_HANGUL)
+                    PixmapRegionPack(pixmap, W_HANGUL_BASE, H)
+                else if (index == SHEET_CUSTOM_SYM)
+                    PixmapRegionPack(pixmap, SIZE_CUSTOM_SYM, SIZE_CUSTOM_SYM)
+                else if (index == SHEET_RUNIC)
+                    PixmapRegionPack(pixmap, W_LATIN_WIDE, H)
+                else throw IllegalArgumentException("Unknown sheet index: $index")
 
+                // this code causes initial deva chars to be skipped from rendering
+//                val illegalCells = HashSet<Long>()
+//                for (code in codeRange[index]) {
+//                    if (glyphProps[code]?.isIllegal == true) {
+//                        val pos = getSheetwisePosition(0, code)
+//                        illegalCells.add(pos[0].toLong().shl(16) or pos[1].toLong())
+//                    }
+//                }
+//
+                for (cy in 0 until texRegPack.verticalCount) {
+                    for (cx in 0 until texRegPack.horizontalCount) {
+//                        if (cx.toLong().shl(16) or cy.toLong() in illegalCells) continue
+                        atlas.queueCell(index, cx, cy, texRegPack.get(cx, cy))
+                    }
+                }
 
-            //val texture = Texture(pixmap)
-            val texRegPack = if (isExtraWide)
-                PixmapRegionPack(pixmap, W_WIDEVAR_INIT, H, HGAP_VAR, 0, xySwapped = isXYSwapped)
-            else if (isVariable)
-                PixmapRegionPack(pixmap, W_VAR_INIT, H, HGAP_VAR, 0, xySwapped = isXYSwapped)
-            else if (index == SHEET_UNIHAN)
-                PixmapRegionPack(pixmap, W_UNIHAN, H_UNIHAN) // the only exception that is height is 16
-            // below they all have height of 20 'H'
-            else if (index == SHEET_HANGUL)
-                PixmapRegionPack(pixmap, W_HANGUL_BASE, H)
-            else if (index == SHEET_CUSTOM_SYM)
-                PixmapRegionPack(pixmap, SIZE_CUSTOM_SYM, SIZE_CUSTOM_SYM) // TODO variable
-            else if (index == SHEET_RUNIC)
-                PixmapRegionPack(pixmap, W_LATIN_WIDE, H)
-            else throw IllegalArgumentException("Unknown sheet index: $index")
-
-            //texRegPack.texture.setFilter(minFilter, magFilter)
-
-            sheetsPack.add(texRegPack)
-
-            pixmap.dispose() // you are terminated
+                texRegPack.dispose()
+                pixmap.dispose()
+            }
         }
 
-        sheets = sheetsPack.toTypedArray()
+        // sort and pack all queued cells (tight-cropped, sorted by height then width)
+        atlas.packAllQueued()
+
+        // pack wenquanyi (SHEET_UNIHAN) last as a contiguous blit
+        unihanPixmap?.let {
+            val cols = it.width / W_UNIHAN
+            val rows = it.height / H_UNIHAN
+            atlas.blitSheet(SHEET_UNIHAN, it, W_UNIHAN, H_UNIHAN, cols, rows)
+            it.dispose()
+        }
 
         // make sure null char is actually null (draws nothing and has zero width)
-        sheets[SHEET_ASCII_VARW].regions[0].setColor(0)
-        sheets[SHEET_ASCII_VARW].regions[0].fill()
+        atlas.getRegion(SHEET_ASCII_VARW, 0, 0)?.let { atlas.clearRegion(it) }
         glyphProps[0] = GlyphProps(0)
+
+        if (debug) {
+            com.badlogic.gdx.graphics.PixmapIO.writePNG(Gdx.files.absolute("$tempDir/glyph_atlas_dump.png"), atlas.pixmap)
+            dbgprn("atlas dumped to $tempDir/glyph_atlas_dump.png")
+        }
     }
 
     override fun getLineHeight(): Float = LINE_HEIGHT.toFloat() * scale
@@ -582,6 +579,9 @@ class TerrarumSansBitmap(
                         renderCol = getColour(c)
                     }
                 }
+                else if (isNoDrawChar(c) || glyphProps[c]?.isIllegal == true) {
+                    // whitespace/control/internal/invalid — no visible glyph, just advance position
+                }
                 else if (sheetID == SHEET_HANGUL) {
                     // Flookahead for {I, P, F}
 
@@ -599,39 +599,31 @@ class TerrarumSansBitmap(
 
                     val (indexCho, indexJung, indexJong) = indices
                     val (choRow, jungRow, jongRow) = rows
-                    val hangulSheet = sheets[SHEET_HANGUL]
-
-
-
-                    val choTex = hangulSheet.get(indexCho, choRow)
-                    val jungTex = hangulSheet.get(indexJung, jungRow)
-                    val jongTex = hangulSheet.get(indexJong, jongRow)
-
-                    linotypePixmap.drawPixmap(choTex,  posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
-                    linotypePixmap.drawPixmap(jungTex, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
-                    linotypePixmap.drawPixmap(jongTex, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    atlas.getRegion(SHEET_HANGUL, indexCho, choRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
+                    atlas.getRegion(SHEET_HANGUL, indexJung, jungRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
+                    atlas.getRegion(SHEET_HANGUL, indexJong, jongRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
 
 
                     index += hangulLength - 1
 
                 }
                 else {
-                    try {
-                        val posY = posmap.y[index].flipY() +
-                                if (sheetID == SHEET_UNIHAN) // evil exceptions
-                                    offsetUnihan
-                                else if (sheetID == SHEET_CUSTOM_SYM)
-                                    offsetCustomSym
-                                else 0
+                    val posY = posmap.y[index].flipY() +
+                            if (sheetID == SHEET_UNIHAN) // evil exceptions
+                                offsetUnihan
+                            else if (sheetID == SHEET_CUSTOM_SYM)
+                                offsetCustomSym
+                            else 0
 
-                        val posX = posmap.x[index]
-                        val texture = sheets[sheetID].get(sheetX, sheetY)
-
-                        linotypePixmap.drawPixmap(texture, posX + linotypePaddingX, posY + linotypePaddingY, renderCol)
-
-
-                    }
-                    catch (noSuchGlyph: ArrayIndexOutOfBoundsException) {
+                    val posX = posmap.x[index]
+                    atlas.getRegion(sheetID, sheetX, sheetY)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posX + linotypePaddingX, posY + linotypePaddingY, renderCol)
                     }
                 }
 
@@ -698,6 +690,9 @@ class TerrarumSansBitmap(
                         renderCol = getColour(c)
                     }
                 }
+                else if (isNoDrawChar(c) || glyphProps[c]?.isIllegal == true) {
+                    // whitespace/control/internal/invalid — no visible glyph, just advance position
+                }
                 else if (sheetID == SHEET_HANGUL) {
                     // Flookahead for {I, P, F}
 
@@ -715,39 +710,31 @@ class TerrarumSansBitmap(
 
                     val (indexCho, indexJung, indexJong) = indices
                     val (choRow, jungRow, jongRow) = rows
-                    val hangulSheet = sheets[SHEET_HANGUL]
-
-
-
-                    val choTex = hangulSheet.get(indexCho, choRow)
-                    val jungTex = hangulSheet.get(indexJung, jungRow)
-                    val jongTex = hangulSheet.get(indexJong, jongRow)
-
-                    linotypePixmap.drawPixmap(choTex,  posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
-                    linotypePixmap.drawPixmap(jungTex, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
-                    linotypePixmap.drawPixmap(jongTex, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    atlas.getRegion(SHEET_HANGUL, indexCho, choRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
+                    atlas.getRegion(SHEET_HANGUL, indexJung, jungRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
+                    atlas.getRegion(SHEET_HANGUL, indexJong, jongRow)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posmap.x[index] + linotypePaddingX, linotypePaddingY, renderCol)
+                    }
 
 
                     index += hangulLength - 1
 
                 }
                 else {
-                    try {
-                        val posY = posmap.y[index].flipY() +
-                                if (sheetID == SHEET_UNIHAN) // evil exceptions
-                                    offsetUnihan
-                                else if (sheetID == SHEET_CUSTOM_SYM)
-                                    offsetCustomSym
-                                else 0
+                    val posY = posmap.y[index].flipY() +
+                            if (sheetID == SHEET_UNIHAN) // evil exceptions
+                                offsetUnihan
+                            else if (sheetID == SHEET_CUSTOM_SYM)
+                                offsetCustomSym
+                            else 0
 
-                        val posX = posmap.x[index]
-                        val texture = sheets[sheetID].get(sheetX, sheetY)
-
-                        linotypePixmap.drawPixmap(texture, posX + linotypePaddingX, posY + linotypePaddingY, renderCol)
-
-
-                    }
-                    catch (noSuchGlyph: ArrayIndexOutOfBoundsException) {
+                    val posX = posmap.x[index]
+                    atlas.getRegion(sheetID, sheetX, sheetY)?.let {
+                        linotypePixmap.drawFromAtlas(atlas.pixmap, it, posX + linotypePaddingX, posY + linotypePaddingY, renderCol)
                     }
                 }
 
@@ -826,7 +813,7 @@ class TerrarumSansBitmap(
     override fun dispose() {
         super.dispose()
         textCache.values.forEach { it.dispose() }
-        sheets.forEach { it.dispose() }
+        atlas.dispose()
     }
 
     fun getSheetType(c: CodePoint): Int {
@@ -891,6 +878,13 @@ class TerrarumSansBitmap(
             SHEET_CYRILIC_EXTB_VARW -> cyrilicExtBIndexY(ch)
             SHEET_CYRILIC_EXTA_VARW -> cyrilicExtAIndexY(ch)
             SHEET_CYRILIC_EXTC_VARW -> cyrilicExtCIndexY(ch)
+            SHEET_LATIN_EXTE_VARW -> latinExtEIndexY(ch)
+            SHEET_LATIN_EXTF_VARW -> latinExtFIndexY(ch)
+            SHEET_LATIN_EXTG_VARW -> latinExtGIndexY(ch)
+            SHEET_OGHAM_VARW -> oghamIndexY(ch)
+            SHEET_COPTIC_VARW -> copticIndexY(ch)
+            SHEET_CYRILIC_EXTD_VARW -> cyrilicExtDIndexY(ch)
+            SHEET_MATHS1_VARW -> maths1IndexY(ch)
             else -> ch / 16
         }
 
@@ -2160,6 +2154,16 @@ class TerrarumSansBitmap(
         }
     }
 
+    private fun Pixmap.drawFromAtlas(atlas: Pixmap, region: AtlasRegion, xPos: Int, yPos: Int, col: Int) {
+        for (y in 0 until region.height) {
+            for (x in 0 until region.width) {
+                val pixel = atlas.getPixel(region.atlasX + x, region.atlasY + y)
+                val newPixel = pixel colorTimes col
+                this.drawPixel(xPos + x + region.offsetX, yPos + y + region.offsetY, newPixel)
+            }
+        }
+    }
+
     private fun Color.toRGBA8888() =
         (this.r * 255f).toInt().shl(24) or
                 (this.g * 255f).toInt().shl(16) or
@@ -2609,6 +2613,13 @@ class TerrarumSansBitmap(
         internal const val SHEET_CYRILIC_EXTB_VARW = 42
         internal const val SHEET_CYRILIC_EXTA_VARW = 43
         internal const val SHEET_CYRILIC_EXTC_VARW = 44
+        internal const val SHEET_LATIN_EXTE_VARW = 45
+        internal const val SHEET_LATIN_EXTF_VARW = 46
+        internal const val SHEET_LATIN_EXTG_VARW = 47
+        internal const val SHEET_OGHAM_VARW = 48
+        internal const val SHEET_COPTIC_VARW = 49
+        internal const val SHEET_CYRILIC_EXTD_VARW = 50
+        internal const val SHEET_MATHS1_VARW = 51
 
         internal const val SHEET_UNKNOWN = 254
 
@@ -2676,6 +2687,13 @@ class TerrarumSansBitmap(
             "cyrilic_extB_variable.tga",
             "cyrilic_extA_variable.tga",
             "cyrilic_extC_variable.tga",
+            "latinExtE_variable.tga",
+            "latinExtF_variable.tga",
+            "latinExtG_variable.tga",
+            "ogham_variable.tga",
+            "coptic_variable.tga",
+            "cyrilic_extD_variable.tga",
+            "maths1_extrawide_variable.tga",
         )
         internal val codeRange = arrayOf( // MUST BE MATCHING WITH SHEET INDICES!!
             0..0xFF, // SHEET_ASCII_VARW
@@ -2688,7 +2706,7 @@ class TerrarumSansBitmap(
             0x400..0x52F, // SHEET_CYRILIC_VARW
             0xFF00..0xFFFF, // SHEET_HALFWIDTH_FULLWIDTH_VARW
             0x2000..0x209F, // SHEET_UNI_PUNCT_VARW
-            0x370..0x3CE, // SHEET_GREEK_VARW
+            0x370..0x3FF, // SHEET_GREEK_VARW
             0xE00..0xE5F, // SHEET_THAI_VARW
             0x530..0x58F, // SHEET_HAYEREN_VARW
             0x10D0..0x10FF, // SHEET_KARTULI_VARW
@@ -2718,11 +2736,18 @@ class TerrarumSansBitmap(
             0xF0520..0xF057F, // SHEET_CODESTYLE_ASCII_VARW
             0xFB00..0xFB17, // SHEET_ALPHABETIC_PRESENTATION_FORMS
             0x1B000..0x1B16F, // SHEET_HENTAIGANA_VARW
-            0x2400..0x243F, // SHEET_CONTROL_PICTURES_VARW
+            0x2400..0x244F, // SHEET_CONTROL_PICTURES_VARW
             0x1FB00..0x1FBFF, // SHEET_LEGACY_COMPUTING_VARW
             0xA640..0xA69F, // SHEET_CYRILIC_EXTB_VARW
             0x2DE0..0x2DFF, // SHEET_CYRILIC_EXTA_VARW
             0x1C80..0x1C8F, // SHEET_CYRILIC_EXTC_VARW
+            0xAB30..0xAB6F, // SHEET_LATIN_EXTE_VARW
+            0x10780..0x107BF, // SHEET_LATIN_EXTF_VARW
+            0x1DF00..0x1DFFF, // SHEET_LATIN_EXTG_VARW
+            0x1680..0x169F, // SHEET_OGHAM_VARW
+            0x2C80..0x2CFF, // SHEET_COPTIC_VARW
+            0x1E030..0x1E08F, // SHEET_CYRILIC_EXTD_VARW
+            0x2200..0x22FF, // SHEET_MATHS1_VARW
         )
         private val codeRangeHangulCompat = 0x3130..0x318F
 
@@ -3018,6 +3043,13 @@ class TerrarumSansBitmap(
         private fun isBulgarian(c: CodePoint) = c in 0xF0000..0xF005F
         private fun isSerbian(c: CodePoint) = c in 0xF0060..0xF00BF
         fun isColourCode(c: CodePoint) = c == 0x100000 || c in 0x10F000..0x10FFFF
+        private fun isNoDrawChar(c: CodePoint): Boolean =
+            c <= 0x20 || c == NBSP || c == SHY || c == OBJ ||
+            c in 0x2000..0x200D ||
+            c in 0xD800..0xDFFF ||
+            c in 0xF800..0xF8FF ||
+            c in 0xFFF70..0xFFF9F ||
+            c >= 0xFFFA0
         private fun isCharsetOverride(c: CodePoint) = c in 0xFFFC0..0xFFFCF
         private fun isDevanagari(c: CodePoint) = c in codeRange[SHEET_DEVANAGARI_VARW]
         private fun isHangulCompat(c: CodePoint) = c in codeRangeHangulCompat
@@ -3071,6 +3103,13 @@ class TerrarumSansBitmap(
         private fun cyrilicExtBIndexY(c: CodePoint) = (c - 0xA640) / 16
         private fun cyrilicExtAIndexY(c: CodePoint) = (c - 0x2DE0) / 16
         private fun cyrilicExtCIndexY(c: CodePoint) = (c - 0x1C80) / 16
+        private fun latinExtEIndexY(c: CodePoint) = (c - 0xAB30) / 16
+        private fun latinExtFIndexY(c: CodePoint) = (c - 0x10780) / 16
+        private fun latinExtGIndexY(c: CodePoint) = (c - 0x1DF00) / 16
+        private fun oghamIndexY(c: CodePoint) = (c - 0x1680) / 16
+        private fun copticIndexY(c: CodePoint) = (c - 0x2C80) / 16
+        private fun cyrilicExtDIndexY(c: CodePoint) = (c - 0x1E030) / 16
+        private fun maths1IndexY(c: CodePoint) = (c - 0x2200) / 16
 
         val charsetOverrideDefault = Character.toChars(CHARSET_OVERRIDE_DEFAULT).toSurrogatedString()
         val charsetOverrideBulgarian = Character.toChars(CHARSET_OVERRIDE_BG_BG).toSurrogatedString()