Multi-byte fonts: GB2312, Big5, Shift-JIS, KS C 5601, UTF-8

Font Creator output decoded (lead/trail ranges and gaps in the header, map codes = bytes as
little-endian u16 or Unicode code points) and round-tripped byte for byte; from_ttf(chars=)
builds them; the renderer splits text like the simulator (any byte >= 0x81 starts a pair,
missing characters advance one space) -- pixel-exact for all five encodings.
Intelligent glyph data is 8-byte aligned (was 4). CLI add-font --chars/--chars-file, MCP chars.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mangavalkandClaude Opus 5.5 committed 2026-11-16 13:42:50 +01:00
1 parent c102157ac2
commit 0138a0ba56
11 files changed
+137 -26

No files matched your search

+9 -2
View File
@@ -116,7 +116,11 @@ def cmd_new_page(a):
def cmd_add_font(a):
from . import fonts, project
h = HMIFile.load(a.hmi)
f = fonts.from_ttf(a.ttf, a.height, a.encoding, a.fixed_width)
chars = a.chars
if a.chars_file:
with open(a.chars_file, encoding="utf-8") as cf:
chars = (chars or "") + cf.read()
f = fonts.from_ttf(a.ttf, a.height, a.encoding, a.fixed_width, chars=chars)
fid = project.add_font(h, f)
h.save(a.out or a.hmi)
print("added font id %d (%s, %d chars)" % (fid, f.name, len(f.glyphs)))
@@ -192,7 +196,10 @@ def main(argv=None):
s = sub.add_parser("kinds", help="list component kinds that can be added"); s.set_defaults(f=cmd_kinds)
s = sub.add_parser("add-font", help="rasterize a .ttf into a new font (needs Pillow)")
s.add_argument("hmi"); s.add_argument("ttf"); s.add_argument("height", type=int)
s.add_argument("--encoding", default="ascii", choices=["ascii", "iso-8859-1"])
s.add_argument("--encoding", default="ascii",
choices=["ascii", "iso-8859-1", "gb2312", "big5", "shift-jis", "ks_c_5601-1987", "utf-8"])
s.add_argument("--chars", help="characters to include (needed for multi-byte encodings)")
s.add_argument("--chars-file", help="UTF-8 text file with the characters to include")
s.add_argument("--fixed-width", type=int); s.add_argument("-o", "--out"); s.set_defaults(f=cmd_add_font)
s = sub.add_parser("add-picture", help="add a picture (png/jpg/bmp; needs Pillow)")
s.add_argument("hmi"); s.add_argument("image"); s.add_argument("-o", "--out")
+63 -8
View File
@@ -73,7 +73,27 @@ class Font:
glyphs: list = field(default_factory=list)
header: bytes = b"" # original 44-byte header, for fields not modelled
direction: int = 1 # display direction the glyphs are stored for (header byte 3 = 0x0A + d)
align: bool = False # Intelligent: glyph data starts 4-byte aligned (zero padding after the map)
align: bool = False # Intelligent: glyph data starts 8-byte aligned (zero padding after the map)
def codes(self, text: bytes) -> list:
"""Character codes of text bytes as this font's map stores them: UTF-8 fonts (encoding 24)
use code points; double-byte fonts (header byte 5 = 2: GB2312, Big5, Shift-JIS, KS C 5601)
take any byte >= 0x81 together with the next one, as lead | trail << 8 (measured in the
simulator: KS C hanja leads above the header's lead range still pair, and Shift-JIS
half-width katakana bytes pair up too, so they cannot be shown)."""
h = self.header
if h and h[4] == 24:
return [ord(ch) for ch in text.decode("utf-8", "replace")]
if h and h[5] == 2:
out, i = [], 0
while i < len(text):
b = text[i]
if b >= 0x81 and i + 1 < len(text):
out.append(b | text[i + 1] << 8); i += 2
else:
out.append(b); i += 1
return out
return list(text)
@property
def antialias(self) -> bool:
@@ -85,7 +105,7 @@ class Font:
# ------------------------------------------------------------------ decode
@classmethod
def parse(cls, blob: bytes) -> "Font":
if blob[:3] != b"\x04\xff\x00":
if blob[0] != 4: # bytes 1-2 are 0xFF 0x00 except for Big5 / Shift-JIS
raise ValueError("not a .zi font")
if blob[16] != 6:
raise ValueError("unsupported .zi version %d (only 6 is implemented)" % blob[16])
@@ -116,7 +136,7 @@ class Font:
for g in self.glyphs]
mp = bytearray()
off = MAP_ENTRY * len(self.glyphs)
pad = -off % 4 if self.align else 0
pad = -off % 8 if self.align else 0
off += pad
for g, b in zip(self.glyphs, bodies):
mp += struct.pack("<HB", g.code, g.width) + g.map_extra[:2].ljust(2, b"\0")
@@ -124,7 +144,7 @@ class Font:
off += len(b)
payload = name + bytes(mp) + bytes(pad) + b"".join(bodies)
hdr = bytearray(self.header or bytes(HEADER_SIZE))
hdr[0:4] = bytes([4, 0xFF, 0, 0x0A + self.direction % 4])
hdr[0:4] = bytes([4, hdr[1], hdr[2], 0x0A + self.direction % 4]) if self.header else bytes([4, 0xFF, 0, 0x0A + self.direction % 4])
struct.pack_into("<H", hdr, 4, self.encoding)
hdr[6], hdr[7], hdr[10], hdr[11] = self.width, self.height, self.first, self.last
struct.pack_into("<I", hdr, 12, len(self.glyphs))
@@ -195,18 +215,50 @@ def encode_glyph(rows: list, w: int, h: int, direction: int = 1) -> bytes:
ENCODINGS = {"ascii": (1, 0x20, 0x7E), "iso-8859-1": (3, 0x20, 0xFF)}
# Multi-byte fonts (header values from the Editor's Font Creator, fixed per encoding):
# encoding id, Python codec, header bytes 1-2 (first trail range end, trail gap), 8-9 (lead range),
# 10-11 (trail range), 0x1C-0x1D (first lead range end, lead gap), 0x20
MULTIBYTE = {
"gb2312": (2, "gb2312", (0xFF, 0x00), (0xA1, 0xF7), (0xA1, 0xFE), (0xFF, 0x00), 4),
"ks_c_5601-1987": (15, "euc_kr", (0xFF, 0x00), (0xA1, 0xC8), (0xA1, 0xFE), (0xFF, 0x00), 4),
"big5": (16, "big5", (0x7E, 0x22), (0xA0, 0xF9), (0x40, 0xFE), (0xFF, 0x00), 5),
"shift-jis": (23, "shift_jis", (0x7E, 0x01), (0x81, 0xEF), (0x40, 0xFC), (0x9F, 0x40), 5),
"utf-8": (24, "utf-8", (0xFF, 0x00), (0xFF, 0xFF), (0x00, 0xFF), (0xFF, 0x00), 5),
}
def char_code(ch: str, encoding: str) -> int:
"""Map code of one character in a font of this encoding (see Font.codes)."""
if encoding == "utf-8":
return ord(ch)
if encoding in MULTIBYTE:
b = ch.encode(MULTIBYTE[encoding][1])
return b[0] | b[1] << 8 if len(b) == 2 else b[0]
return ch.encode("latin-1")[0]
def from_ttf(path: str | None, height: int, encoding: str = "ascii", fixed_width: int | None = None,
name: str | None = None, threshold: int = 128) -> "Font":
name: str | None = None, threshold: int = 128, chars: str | None = None) -> "Font":
"""Rasterize a TrueType/OpenType font into a black/white Nextion font (needs Pillow).
``path=None`` uses Pillow's bundled default font (Aileron Regular, CC0) -- handy for tests and
demos. The font size is chosen so ascent + descent fills ``height`` pixels. Glyph widths follow
the font's advance widths unless ``fixed_width`` is given.
Multi-byte encodings (gb2312, big5, shift-jis, ks_c_5601-1987, utf-8) need ``chars``: the
characters to include (like the Font Creator's "Specified character" range), and a TrueType
file that has them.
"""
import os
from PIL import Image, ImageDraw, ImageFont
enc, first, last = ENCODINGS[encoding]
if encoding in MULTIBYTE:
if not chars:
raise ValueError("encoding %s needs chars= (the characters to include)" % encoding)
mb = MULTIBYTE[encoding]
enc, first, last = mb[0] | 2 << 8, mb[4][0], mb[4][1]
todo = sorted({(char_code(ch, encoding), ch) for ch in chars if ch not in "\r\n"})
else:
enc, first, last = ENCODINGS[encoding]
todo = [(code, bytes([code]).decode("latin-1")) for code in range(first, last + 1)]
def load(size):
return ImageFont.load_default(size=size) if path is None else ImageFont.truetype(path, size)
@@ -220,8 +272,7 @@ def from_ttf(path: str | None, height: int, encoding: str = "ascii", fixed_width
ascent, descent = f.getmetrics()
top = (height - ascent - descent) // 2
glyphs = []
for code in range(first, last + 1):
ch = bytes([code]).decode("latin-1")
for code, ch in todo:
adv = fixed_width or max(1, int(round(f.getlength(ch))))
img = Image.new("L", (adv, height), 0)
ImageDraw.Draw(img).text((0, top), ch, font=f, fill=255)
@@ -235,6 +286,10 @@ def from_ttf(path: str | None, height: int, encoding: str = "ascii", fixed_width
hdr[0:4] = b"\x04\xff\x00\x0b"
hdr[0x1C] = last
hdr[0x20] = 0x0B
if encoding in MULTIBYTE:
hdr[1:3], hdr[8:10], hdr[0x1C:0x1E] = bytes(mb[2]), bytes(mb[3]), bytes(mb[5])
hdr[0x20] = mb[6]
width = 0 # the Editor writes 0 for multi-byte fonts
hdr[18] = len(base[:255].encode("latin-1")) # name length twice, like the Editor re-saves it
struct.pack_into("<I", hdr, 0x24, len(glyphs))
return Font(base[:255], width, height, enc, first, last, glyphs, bytes(hdr))
+7 -4
View File
@@ -158,10 +158,10 @@ def t_add_picture(path, image_path, quality=100):
return {"picture_id": pid}
def t_add_font(path, height, ttf_path=None, encoding="ascii"):
def t_add_font(path, height, ttf_path=None, encoding="ascii", chars=None):
from . import fonts
h = _load(path)
f = fonts.from_ttf(ttf_path, int(height), encoding)
f = fonts.from_ttf(ttf_path, int(height), encoding, chars=chars)
fid = project.add_font(h, f)
h.save(path)
return {"font_id": fid, "name": f.name, "height": f.height}
@@ -243,8 +243,11 @@ TOOLS = {
"add_picture": (t_add_picture, "Add a picture from an image file (png/jpg/bmp); returns its picture id. "
"quality 1-99 stores a JPEG (Intelligent displays only, smaller), default 100.",
{"path": S, "image_path": S, "quality": {"type": "integer"}}, ["path", "image_path"]),
"add_font": (t_add_font, "Add a font rasterized from a TrueType file (omit ttf_path for the bundled CC0 font).",
{"path": S, "height": {"type": "integer"}, "ttf_path": S, "encoding": S}, ["path", "height"]),
"add_font": (t_add_font, "Add a font rasterized from a TrueType file (omit ttf_path for the bundled CC0 font). "
"encoding: ascii, iso-8859-1, or multi-byte gb2312/big5/shift-jis/ks_c_5601-1987/utf-8 "
"(then give chars, and a ttf_path that has those characters; the project's encoding "
"should match).",
{"path": S, "height": {"type": "integer"}, "ttf_path": S, "encoding": S, "chars": S}, ["path", "height"]),
"verify": (t_verify, "Check every checksum in the project.", {"path": S}, ["path"]),
"list_kinds": (t_kinds, "Component kinds that can be added.", {}, []),
"schema": (t_schema, "Attributes of a component kind: names, types, defaults, allowed values.", {"kind": S}, ["kind"]),
+1 -1
View File
@@ -260,7 +260,7 @@ def add_font(h: HMIFile, font) -> int:
n += 1
main = h.read("main.HMI")
font.direction = main[12] % 4 # glyphs are stored rotated like pictures
font.align = main[11] == 3 # Intelligent: glyph data 4-byte aligned
font.align = main[11] == 3 # Intelligent: glyph data 8-byte aligned
h.write("%d.zi" % n, font.pack())
h.write("main.HMI", main_insert_resource(h.read("main.HMI"), "zi", "%d.zi" % n))
return n
+3 -1
View File
@@ -256,13 +256,15 @@ class Renderer:
ycen = c.get("ycen") if ycen is None else ycen
wrap = c.get("isbr") if wrap is None else wrap
glyphs = {g.code: g for g in font.glyphs}
missing = glyphs[0x20].width if 0x20 in glyphs else font.width # unknown chars advance a space
def gw(ch):
g = glyphs.get(ord(ch))
return (g.width if g else font.width) + spax
return (g.width if g else missing) + spax
lines = []
for para in s.replace("\r\n", "\n").split("\n"):
para = "".join(map(chr, font.codes(para.encode("latin-1", "replace")))) # multi-byte text
cur = ""
for ch in para:
if wrap and cur and sum(gw(k) for k in cur + ch) - spax > w: