From f66c902443b8c9abade6f40297d2e20ac466a491 Mon Sep 17 00:00:00 2001 From: MangaValk Date: Mon, 16 Nov 2026 18:13:11 +0100 Subject: [PATCH] tft: page section format, pictures/fonts, first builder (nxhmi build-tft) Byte-identical to Nextion Editor 1.68.2 for NX4832T035 projects made of pages, hotspots and int variables with any pictures and fonts. Firmware files are read from the user's Editor folder at build time. FORMAT.md section 7 documents the header, resource area, code blob, attribute records, page table and objects. Co-Authored-By: Claude Fable 5.2 --- CLAUDE.md | 4 +- README.md | 2 +- docs/COVERAGE.md | 8 +- docs/FORMAT.md | 132 ++++++++++++++++++++++++ docs/NOTES.md | 25 +++++ src/nxhmi/cli.py | 32 ++++++ src/nxhmi/tft.py | 8 ++ src/nxhmi/tft_build.py | 223 ++++++++++++++++++++++++++++++++++++++++ src/nxhmi/tft_code.py | 35 +++++++ tests/test_tft_build.py | 125 ++++++++++++++++++++++ 10 files changed, 589 insertions(+), 5 deletions(-) create mode 100644 src/nxhmi/tft_build.py create mode 100644 src/nxhmi/tft_code.py create mode 100644 tests/test_tft_build.py diff --git a/CLAUDE.md b/CLAUDE.md index 2107fc8..2d998b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,9 @@ src/nxhmi/ pictures.py N.i / N.is fonts.py .zi v6 fonts, generation from TrueType render.py page renderer matching the Editor canvas - tft.py .tft research helpers (layout table, header checksums, trailer, seal) + tft.py .tft helpers (layout table, header checksums, trailer, seal, name hash) + tft_build.py .HMI -> .tft builder (subset; firmware files come from the user's Editor folder) + tft_code.py Nextion script -> tokenised chunks cli.py python -m nxhmi ... mcp_server.py MCP server (stdio JSON-RPC, no deps) -- tools for Claude incl. render_page images examples/make_demo.py builds demo/ (demo.HMI, decoded/, render/) diff --git a/README.md b/README.md index 9a311b1..1b8d30f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Every percentage is a count with a stated basis, so it can be checked (details i | Readable attribute names in JSON | **100 %** (31/31 types) | every value list the Editor documents, incl. the Intelligent-only types and load effects | | Rendering, Basic components | **100 %** (20/20 pixel-exact) | incl. gauges and QR codes (own encoder, no dependencies) | | Rendering, all components | **94 %** (29/31 pixel-exact) | Intelligent ComboBox and Switch are drawn, but their anti-aliased edges are approximate (~99 % / ~92 % of pixels exact) | -| `.tft` compilation | **50 %** (4/8 milestones) | layout mapped, layout table decoded, all four checksums solved (any length); page format not yet, so not usable | +| `.tft` compilation | **69 %** (5.5/8 milestones) | `nxhmi build-tft`: byte-identical to the Editor for NX4832T035 projects with pages, hotspots, int variables, pictures and fonts; other component types and most script not yet | ## Compatibility diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md index 0628de0..133d0b8 100644 --- a/docs/COVERAGE.md +++ b/docs/COVERAGE.md @@ -112,11 +112,13 @@ byte for byte, or a file written by nxhmi was opened in the Editor and showed th | Layout table | decoded (fixed XOR key) | | Trailer checksum | solved for any length (`tft.trailer`); one model measured | | 3 header checksums (0x44, 0xC4, 0x18C) | solved (`tft.header_checksums`, `tft.seal`) | -| Compiled page format, event-code tokens | first look only | -| Pictures and fonts inside the TFT | not analysed | +| Pictures and fonts inside the TFT | solved (FORMAT.md 7.2) | +| Compiled page format | structure solved (code blob, name hashes, RAM image, attribute records, page table, object records -- FORMAT.md 7.3-7.5); per-type tables done for page, hotspot, int variable; the other types need their slot tables and generated draw scripts | +| Event-code tokens | encoding understood (command / system-variable / attribute-reference / number tokens); table being filled | +| Builder (`nxhmi build-tft`) | byte-identical to the Editor for the supported subset (one model: NX4832T035); refuses anything else. **Never tested on a real display** | | Licensing | every `.tft` embeds Nextion's firmware files; the repo must never ship them -- a builder would read them from the user's own Editor installation | ## Suggested next steps -1. `.tft`: the compiled page/component format and the event-code token table (see NOTES.md). +1. `.tft`: script token table, then component types one at a time (see NOTES.md); a second display model. 2. The anti-aliasing routine of Intelligent rounded shapes (switch knob/track, ComboBox). 3. The premultiplied-alpha colour step for transparent JPEG pictures (a few levels off). diff --git a/docs/FORMAT.md b/docs/FORMAT.md index c792c93..5c15326 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -327,3 +327,135 @@ width of the space glyph. ## 6. Not yet covered See `docs/NOTES.md`. + +## 7. Compiled firmware (`.tft`) + +Measured on Editor 1.68.2.3035, model NX4832T035 (Basic series), landscape. Everything below is +reproduced byte for byte by `nxhmi.tft_build` for the supported subset unless marked **OPEN**. +All integers little-endian. "CRC" = the word-per-byte CRC of section 2 with seed 0xFFFFFFFF. + +### 7.1 File map + +| Range | Contents | +|---|---| +| 0x00000-0x0FFFF | header; unused bytes are FF | +| 0x10000-`base` | resource area, size = u32@0x40, a multiple of 0x10000, zero-padded | +| `base`-end | page section: code blob, attribute records, page table, object records, FF padding to a multiple of 4, trailer | + +Header: + +| Offset | Field | +|---|---| +| 0x00 | 12 constant bytes `00 01 44 4E 78 45 01 00 4C D3 00 00` (**OPEN**: meaning; probably format/Editor version) | +| 0x0C | u16 width, height, height, width | +| 0x14 | `01 00 00 02` (**OPEN**) | +| 0x18, 0x1E | u32 file offset of `cd0.dll` (twice; u16 0 between) | +| 0x22 | u32 size of `cd0.dll` | +| 0x26, 0x2A | u32 file offset, size of `asp0.dll` | +| 0x2E | u32 model id (as in main.HMI) | +| 0x32 | `21 00 00 00 01 00 0C 00 00 00` (**OPEN**) | +| 0x3C | u32 file size | +| 0x40 | u32 resource area size | +| 0x44 | CRC of the resource area | +| 0x48 | u32 0, 8 x FF, u32 0 | +| 0x58 | u32 size of code blob + attribute records | +| 0xC4 | CRC of `[0x00, 0xC4)` | +| 0xC8 | layout table: 19 u32, each XOR 0xEBAB2932 | +| 0x18C | CRC of `[0xC8, 0x18C)` as stored | + +Layout table words: 0, 1, 3 = code blob size; 4 = 0x10000; 5 = `base`; 6 = page table offset; +7 = object records offset; 8 = pictures offset; 12 = fonts offset; 13 = offset of the program +inside the blob; 14 = page count | component count << 16; 15 = picture count; 17 = font count; +18 = 3; the rest 0 (**OPEN**: probably audio/video/gmov resources). + +Trailer (last 4 bytes): CRC of everything before it, XOR `(file length & 0xFF) ^ 0x56`. + +### 7.2 Resource area + +A 0x90-byte directory of 12 entries `(u32 offset from 0x10000, u32 size, u32 0)`, then the data +in the same order: seven files copied verbatim from the Editor's install folder (`input.bin`, +`asp0.dll`, `cd0.dll`, `qr0.bin`, `syscom.bin`, `h0-0.dll`, `h1-0.dll`), all pictures, all fonts; +entries 9-11 are empty. + +- Pictures: the 24-byte headers of all `N.i` entries first, then their pixel data. In each + header, u32@4 = picture number and u32@8 = offset of its data from the start of the picture + block. (One picture: identical to the `.i` entry.) +- Fonts: the 44-byte headers of all `N.zi` entries first, then the rest of each font; u32@0x18 + of a header = offset of that font's data from the start of the font block. + +### 7.3 Code blob + +Offsets in this section are relative to `base`. + +``` +u32 0x24 + 6*pages, u32 3, u32 0x1C, u32 4, u32 0x20, u32 4, u32 0x24, u32 6*pages, u32 0, u32 0 +page names: (u32 name hash, u16 page number), sorted by hash +program +per page: code blocks of its components, name list, RAM image, init code +``` + +The first ten words describe three tables at 4 + (0x1C, 0x20, 0x24) with sizes 4, 4, 6*pages; +the first two are empty when the program declares no global variables (**OPEN**: their format). + +**Name hash** = CRC of the name NUL-padded to 14 bytes. Names are not stored anywhere else. + +Everything after the page names is a sequence of **chunks**: `u32 length` + bytes. A chunk is one +tokenised script line; an empty chunk ends a section. + +- Program: one chunk per `Program.s` statement (comments and blank lines dropped), then a chunk + `page 0` added by the Editor, then the empty chunk. +- Component block (the page itself is component 0): + - draw section -- lines, empty chunk. Page: `fill x,y,w,h,` (command `09 0D 04`, colour + as a reference to the page's `bco` record), then the Preinitialize event lines. + Types that draw themselves (progress bar, gauge, QR code, text ...) have their own script + here, generated from the attributes (**OPEN** per type). + - erase section (types with a rectangle only; empty for the page) -- `fill x,y,w,h,`, + empty chunk. + - events: a label chunk (`down`, `up`, `unload`, `timer`, `slide` -- plain text), the event's + lines, empty chunk. Page: down, up, unload. Hotspot: down, up. Variable: none. +- After the last component of a page: one chunk with the name list `(u32 hash, u16 id)` sorted + by hash; one chunk with the RAM image followed by 8 zero bytes; type-specific init lines + (timer, slider, touch cap -- **OPEN**); the chunk `09 21 08`; the page's Postinitialize lines; + empty chunk. + +RAM image: 8 x FF, then every RAM-stored attribute of every component in component order, +each aligned to its size, each component padded to 4. For a page: `sta` (u8), pad, `bco` (u16). + +### 7.4 Attribute records (24 bytes) + +Every component owns a fixed number of record slots (page 33, hotspot 27, variable 11, +timer 11, touch cap 10, progress bar 33, slider 40, gauge 41, QR code 33 ...); unused slots are +all zero. Record numbers are global (pages concatenated) in script references and page-relative +in the object records. + +| Offset | Field | +|---|---| +| 0 | u32 8 for a page, 16 for other components | +| 4 | i32 value (constant attributes) or 4 + offset into the RAM image | +| 8, 12 | i32 maximum, minimum | +| 16 | u8 page number, u8 component id of the component to redraw on change; FF FF = none | +| 18 | u16 change action (0 = none; timers use 1 for `tim`, 2 for `en`) | +| 20 | u8 kind: 0x01 constant; 0x0B/0x0C/0x0D/0x0E x/y/w/h; 0x08 endx/endy; 0x21, 0x22, 0x31, 0x32, 0x39, 0x3F RAM-stored | +| 21 | u8 size in bytes | +| 22 | u16 0 | + +Slots seen in all types: 0 `type`, 1 `id`, 3 `vscope`, 9-14 `x y w h endx endy`. + +### 7.5 Page table (16 bytes per page) and object records (232 bytes per component) + +Page: u16 first object number, u16 component count, u32 offset of the name-list chunk, u32 +offset of the page's first attribute record (from `base`), u32 RAM image size. + +Object: + +| Offset | Field | +|---|---| +| 0 | u8 type, u8 id, u8 0, u8 flags (0x37 visible types, 0x27 timer/touch cap, 0x07 variable) | +| 4 | u32 offset of the draw section, FFFFFFFF if empty | +| 8 | u32 offset of the erase section, FFFFFFFF if none/empty | +| 0x0C-0x1B | more code offsets (0x14: timer event), FF when unused | +| 0x1C | u32 8 / 16 as in the records | +| 0x20 | `00 00 7F 00`, u32 0 | +| 0x28 | u16 x, y, w, h, endx, endy (0,0,1,1,0,0 for invisible types) | +| 0x34 | u32 offset of the component's code block | +| 0x38 | 88 x u16: record number (page-relative) of well-known attributes, FFFF = absent. Positions: 1 vscope, 2 sta, 4 bco, 16 pco, 20 val, 42 x, 43 y, 44 endx, 45 endy, 46 w, 47 h, 49 id, 62 type (**OPEN**: the others) | diff --git a/docs/NOTES.md b/docs/NOTES.md index 09d6b6b..4f5693e 100644 --- a/docs/NOTES.md +++ b/docs/NOTES.md @@ -130,3 +130,28 @@ The file length is always a multiple of 4 (the page section is padded with zeros Other dead ends, for the record: seeds from header fields; header checksum fields zeroed/0xFF in every combination; the layout table de-obfuscated before checksumming; prefixes built from length/layout words; runs of 00/FF bytes before the file. + +### Page section solved, first builder (2026-11-16) + +Format: FORMAT.md section 7. `nxhmi build-tft` reproduces the Editor byte for byte for projects +made of pages (solid colour), hotspots and int variables, with any pictures and fonts, on +NX4832T035. Check with `NXHMI_TFT_SAMPLES= python -m unittest +discover -s tests -p test_tft_build.py` (samples stay outside the repo). + +How the samples are made: one tiny project per question (nxhmi builds them), compiled in the +Editor by a click script (open file, Compile, File > TFT file output > Output), ~45 s each. +A project whose script does not compile yields no TFT -- e.g. `sys0` is only known when +`Program.s` declares it (`int sys0=0,sys1=0,sys2=0`, as in the Editor's default program). + +Useful facts not obvious from the format description: + +- Text-like components need a font in the project or the compile fails. +- Every component type brings its own generated script (draw section, init lines); e.g. a + progress bar's draw section is three `fill`-like lines computed from x/y/w/h and `val`. + These must be collected per type -- the bulk of the remaining work. +- Record references in scripts (`01` + u32) are global record numbers, so adding a component + renumbers every reference after it. + +Not done: script tokens beyond `baud=115200` / `page N`; timers, sliders and every drawing +component; global variables declared in Program.s (blob header tables 0 and 1); other models, +portrait, other series; and nothing has been loaded onto a display. diff --git a/src/nxhmi/cli.py b/src/nxhmi/cli.py index 1b47c32..f470c1a 100644 --- a/src/nxhmi/cli.py +++ b/src/nxhmi/cli.py @@ -169,6 +169,32 @@ def cmd_dump(a): print(json.dumps(project.page_to_json(e.name, p, a.encoding, not a.raw), indent=1, ensure_ascii=False)) +def cmd_build_tft(a): + from . import tft_build + try: + out = tft_build.build_tft(HMIFile.load(a.hmi), a.editor_dir) + except tft_build.Unsupported as e: + print("error: cannot compile yet:", e, file=sys.stderr) + return 3 + except FileNotFoundError as e: + print("error:", e, file=sys.stderr) + return 2 + with open(a.tft, "wb") as fh: + fh.write(out) + print("wrote %s (%d bytes)" % (a.tft, len(out))) + + +def cmd_tft_info(a): + from . import tft + with open(a.tft, "rb") as fh: + data = fh.read() + info = tft.header_info(data) + info["header_checksums_ok"] = tft.verify_header(data) + info["trailer_ok"] = tft.verify_trailer(data) + print(json.dumps(info, indent=1)) + return 0 if info["header_checksums_ok"] and info["trailer_ok"] else 1 + + def main(argv=None): ap = argparse.ArgumentParser(prog="nxhmi", description="Read, verify and write Nextion Editor .HMI projects") sub = ap.add_subparsers(dest="cmd", required=True) @@ -210,6 +236,12 @@ def main(argv=None): s.add_argument("-o", "--out"); s.set_defaults(f=cmd_new_page) s = sub.add_parser("dump", help="print pages as JSON"); s.add_argument("hmi"); s.add_argument("--page") s.add_argument("--encoding", default="latin-1"); s.add_argument("--raw", action="store_true"); s.set_defaults(f=cmd_dump) + s = sub.add_parser("build-tft", help="compile to .tft (research stage: NX4832T035, few component types; " + "needs a Nextion Editor installation for the firmware files)") + s.add_argument("hmi"); s.add_argument("tft") + s.add_argument("--editor-dir", default=r"C:\Program Files (x86)\Nextion Editor"); s.set_defaults(f=cmd_build_tft) + s = sub.add_parser("tft-info", help="header fields, layout table and checksum status of a .tft") + s.add_argument("tft"); s.set_defaults(f=cmd_tft_info) a = ap.parse_args(argv) try: return a.f(a) or 0 diff --git a/src/nxhmi/tft.py b/src/nxhmi/tft.py index 1ada377..08e6855 100644 --- a/src/nxhmi/tft.py +++ b/src/nxhmi/tft.py @@ -34,6 +34,14 @@ def header_info(tft: bytes) -> dict: } +OBJ_SIZE, REC_SIZE = 232, 24 # object record / attribute record in the page section + + +def name_hash(name: str) -> int: + """Hash under which page and component names are stored: CRC of the name NUL-padded to 14.""" + return word_per_byte_crc(name.encode("latin-1")[:14].ljust(14, b"\0")) + + RES_START = 0x10000 # resource area: firmware files, pictures, fonts OFF_RES_SIZE, OFF_RES_CRC, OFF_HEAD1_CRC, OFF_HEAD2_CRC = 0x40, 0x44, 0xC4, 0x18C diff --git a/src/nxhmi/tft_build.py b/src/nxhmi/tft_build.py new file mode 100644 index 0000000..5668ab4 --- /dev/null +++ b/src/nxhmi/tft_build.py @@ -0,0 +1,223 @@ +"""Build a .tft from a .HMI project -- research stage. See docs/FORMAT.md section 7. + +What is reproduced byte for byte (Nextion Editor 1.68.2, model NX4832T035): header, resource +area (the firmware files are read from the user's own Editor installation; pictures, fonts), page +section for the component types in ``TYPES`` and the script subset in :mod:`nxhmi.tft_code`, +layout table and all checksums. Anything else raises :class:`Unsupported` -- never a guess. +""" +from __future__ import annotations + +import os +import re +import struct + +from . import tft, tft_code +from .container import HMIFile +from .page import Page +from .tft_code import Unsupported + +FIRMWARE_FILES = ("input.bin", "asp0.dll", "cd0.dll", "qr0.bin", "syscom.bin", "h0-0.dll", "h1-0.dll") +DEFAULT_EDITOR_DIR = r"C:\Program Files (x86)\Nextion Editor" +SUPPORTED_MODEL = 0xB23BC018 # NX4832T035_011, the only model measured +BLOCK = 0x10000 + +# attribute map positions in the object record (u16 table at +0x38) +MAP = {"vscope": 1, "sta": 2, "bco": 4, "val": 20, "x": 42, "y": 43, "endx": 44, "endy": 45, "w": 46, "h": 47, + "id": 49, "type": 62} +# attribute record templates: name -> (slot, kind, size, max, min) +_COMMON = {"type": (0, 0x01, 1, 255, 0), "id": (1, 0x01, 1, 255, 0), "vscope": (3, 0x01, 1, 1, 0)} +_RECT = {"x": (9, 0x0B, 2, 6000, -6000), "y": (10, 0x0C, 2, 6000, -6000), "w": (11, 0x0D, 2, 2000, -2000), + "h": (12, 0x0E, 2, 2000, -2000), "endx": (13, 0x08, 2, 32767, -32768), "endy": (14, 0x08, 2, 32767, -32768)} +# type id -> description. ``ram`` attributes live in the page's RAM image (in this order); +# ``owner``: whether records name the component to refresh when the attribute changes. +TYPES = { + 121: dict(name="page", slots=33, flag=0x37, group=8, const={**_COMMON, **_RECT}, + ram={"sta": (23, 0x21, 1, 2, 0), "bco": (24, 0x32, 2, 65535, 0)}, owner=True), + 109: dict(name="hotspot", slots=27, flag=0x37, group=16, const={**_COMMON, **_RECT}, ram={}, owner=True), + 52: dict(name="variable", slots=11, flag=0x07, group=16, + const={**_COMMON, "sta": (7, 0x01, 1, 1, 0)}, + ram={"val": (8, 0x39, 4, 2147483647, -2147483648)}, owner=False), +} + + +def _chunk(b: bytes) -> bytes: + return struct.pack(" int: + return int.from_bytes(attr.value, "little") + + +def _s(attr, signed) -> int: + v, bits = _u(attr), 8 * len(attr.value) + return v - (1 << bits) if signed and v >> (bits - 1) else v + + +def resource_area(hmi: HMIFile, editor_dir: str) -> tuple[bytes, dict]: + """Directory + firmware files + pictures + fonts, zero-padded to a 64 KB multiple.""" + def numbered(ext): + names = [n for n in hmi.names() if re.fullmatch(r"\d+\." + ext, n)] + return [hmi.read(n) for n in sorted(names, key=lambda s: int(s.split(".")[0]))] + + def packed(blobs, head, off_pos, idx_pos=None): + heads, datas, o = [], [], head * len(blobs) + for i, b in enumerate(blobs): + h = bytearray(b[:head]) + struct.pack_into(" (code blob, attribute records, page table, object records, program offset).""" + names = sorted((tft.name_hash(p.components[0].get("objname")), i) for i, p in enumerate(pages)) + n = len(pages) + blob = bytearray(struct.pack("<10I", 0x24 + 6 * n, 3, 0x1C, 4, 0x20, 4, 0x24, 6 * n, 0, 0)) + for h, i in names: + blob += struct.pack(" bytes: + main = hmi.read("main.HMI") + model, = struct.unpack_from(" the tokenised form stored in a .tft (research stage; see docs/FORMAT.md).""" +from __future__ import annotations + +import re +import struct + + +class Unsupported(Exception): + """The project uses something whose compiled form has not been worked out yet.""" + + +class Context: + pass + + +def group_lines(group) -> list[str]: + return [(r if isinstance(r, bytes) else r.value).decode("latin-1") for r in group.records] + + +def statements(lines) -> list[str]: + out = [] + for ln in lines: + ln = ln.split("//")[0].strip() + if ln: + out.append(ln) + return out + + +def compile_line(line: str, ctx: Context) -> bytes: + if line == "baud=115200": + return b"\x04\x04\x0e\x00\x00=\x03" + struct.pack(".HMI + .tft pairs made with Nextion Editor 1.68.2.""" +import glob +import os +import shutil +import struct +import tempfile +import unittest + +from helpers import HAVE_PIL + +from nxhmi import cli, components as C, project, tft, tft_build +from nxhmi.page import Page + +SAMPLES = os.environ.get("NXHMI_TFT_SAMPLES") + + +class Build(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + for i, n in enumerate(tft_build.FIRMWARE_FILES): + with open(os.path.join(self.tmp, n), "wb") as fh: + fh.write(bytes([i + 1]) * (100 + 7 * i)) + + def tearDown(self): + shutil.rmtree(self.tmp) + + def project(self): + h = project.new_project("NX4832T035_011") + p = Page.parse(h.read("0.pa")) + C.add(p, "hotspot", x=10, y=20, w=30, h=40) + C.add(p, "variable") + h.write("0.pa", p.pack()) + project.add_page(h, "second") + return h + + def test_structure(self): + out = tft_build.build_tft(self.project(), self.tmp) + self.assertEqual(len(out) % 4, 0) + self.assertTrue(tft.verify_header(out)) + self.assertTrue(tft.verify_trailer(out)) + self.assertEqual(struct.unpack_from("