diff --git a/CLAUDE.md b/CLAUDE.md index 2107fc8..2d998b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,9 @@ src/nxhmi/ pictures.py N.i / N.is fonts.py .zi v6 fonts, generation from TrueType render.py page renderer matching the Editor canvas - tft.py .tft research helpers (layout table, header checksums, trailer, seal) + tft.py .tft helpers (layout table, header checksums, trailer, seal, name hash) + tft_build.py .HMI -> .tft builder (subset; firmware files come from the user's Editor folder) + tft_code.py Nextion script -> tokenised chunks cli.py python -m nxhmi ... mcp_server.py MCP server (stdio JSON-RPC, no deps) -- tools for Claude incl. render_page images examples/make_demo.py builds demo/ (demo.HMI, decoded/, render/) diff --git a/README.md b/README.md index 9a311b1..1b8d30f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Every percentage is a count with a stated basis, so it can be checked (details i | Readable attribute names in JSON | **100 %** (31/31 types) | every value list the Editor documents, incl. the Intelligent-only types and load effects | | Rendering, Basic components | **100 %** (20/20 pixel-exact) | incl. gauges and QR codes (own encoder, no dependencies) | | Rendering, all components | **94 %** (29/31 pixel-exact) | Intelligent ComboBox and Switch are drawn, but their anti-aliased edges are approximate (~99 % / ~92 % of pixels exact) | -| `.tft` compilation | **50 %** (4/8 milestones) | layout mapped, layout table decoded, all four checksums solved (any length); page format not yet, so not usable | +| `.tft` compilation | **69 %** (5.5/8 milestones) | `nxhmi build-tft`: byte-identical to the Editor for NX4832T035 projects with pages, hotspots, int variables, pictures and fonts; other component types and most script not yet | ## Compatibility diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md index 0628de0..133d0b8 100644 --- a/docs/COVERAGE.md +++ b/docs/COVERAGE.md @@ -112,11 +112,13 @@ byte for byte, or a file written by nxhmi was opened in the Editor and showed th | Layout table | decoded (fixed XOR key) | | Trailer checksum | solved for any length (`tft.trailer`); one model measured | | 3 header checksums (0x44, 0xC4, 0x18C) | solved (`tft.header_checksums`, `tft.seal`) | -| Compiled page format, event-code tokens | first look only | -| Pictures and fonts inside the TFT | not analysed | +| Pictures and fonts inside the TFT | solved (FORMAT.md 7.2) | +| Compiled page format | structure solved (code blob, name hashes, RAM image, attribute records, page table, object records -- FORMAT.md 7.3-7.5); per-type tables done for page, hotspot, int variable; the other types need their slot tables and generated draw scripts | +| Event-code tokens | encoding understood (command / system-variable / attribute-reference / number tokens); table being filled | +| Builder (`nxhmi build-tft`) | byte-identical to the Editor for the supported subset (one model: NX4832T035); refuses anything else. **Never tested on a real display** | | Licensing | every `.tft` embeds Nextion's firmware files; the repo must never ship them -- a builder would read them from the user's own Editor installation | ## Suggested next steps -1. `.tft`: the compiled page/component format and the event-code token table (see NOTES.md). +1. `.tft`: script token table, then component types one at a time (see NOTES.md); a second display model. 2. The anti-aliasing routine of Intelligent rounded shapes (switch knob/track, ComboBox). 3. The premultiplied-alpha colour step for transparent JPEG pictures (a few levels off). diff --git a/docs/FORMAT.md b/docs/FORMAT.md index c792c93..5c15326 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -327,3 +327,135 @@ width of the space glyph. ## 6. Not yet covered See `docs/NOTES.md`. + +## 7. Compiled firmware (`.tft`) + +Measured on Editor 1.68.2.3035, model NX4832T035 (Basic series), landscape. Everything below is +reproduced byte for byte by `nxhmi.tft_build` for the supported subset unless marked **OPEN**. +All integers little-endian. "CRC" = the word-per-byte CRC of section 2 with seed 0xFFFFFFFF. + +### 7.1 File map + +| Range | Contents | +|---|---| +| 0x00000-0x0FFFF | header; unused bytes are FF | +| 0x10000-`base` | resource area, size = u32@0x40, a multiple of 0x10000, zero-padded | +| `base`-end | page section: code blob, attribute records, page table, object records, FF padding to a multiple of 4, trailer | + +Header: + +| Offset | Field | +|---|---| +| 0x00 | 12 constant bytes `00 01 44 4E 78 45 01 00 4C D3 00 00` (**OPEN**: meaning; probably format/Editor version) | +| 0x0C | u16 width, height, height, width | +| 0x14 | `01 00 00 02` (**OPEN**) | +| 0x18, 0x1E | u32 file offset of `cd0.dll` (twice; u16 0 between) | +| 0x22 | u32 size of `cd0.dll` | +| 0x26, 0x2A | u32 file offset, size of `asp0.dll` | +| 0x2E | u32 model id (as in main.HMI) | +| 0x32 | `21 00 00 00 01 00 0C 00 00 00` (**OPEN**) | +| 0x3C | u32 file size | +| 0x40 | u32 resource area size | +| 0x44 | CRC of the resource area | +| 0x48 | u32 0, 8 x FF, u32 0 | +| 0x58 | u32 size of code blob + attribute records | +| 0xC4 | CRC of `[0x00, 0xC4)` | +| 0xC8 | layout table: 19 u32, each XOR 0xEBAB2932 | +| 0x18C | CRC of `[0xC8, 0x18C)` as stored | + +Layout table words: 0, 1, 3 = code blob size; 4 = 0x10000; 5 = `base`; 6 = page table offset; +7 = object records offset; 8 = pictures offset; 12 = fonts offset; 13 = offset of the program +inside the blob; 14 = page count | component count << 16; 15 = picture count; 17 = font count; +18 = 3; the rest 0 (**OPEN**: probably audio/video/gmov resources). + +Trailer (last 4 bytes): CRC of everything before it, XOR `(file length & 0xFF) ^ 0x56`. + +### 7.2 Resource area + +A 0x90-byte directory of 12 entries `(u32 offset from 0x10000, u32 size, u32 0)`, then the data +in the same order: seven files copied verbatim from the Editor's install folder (`input.bin`, +`asp0.dll`, `cd0.dll`, `qr0.bin`, `syscom.bin`, `h0-0.dll`, `h1-0.dll`), all pictures, all fonts; +entries 9-11 are empty. + +- Pictures: the 24-byte headers of all `N.i` entries first, then their pixel data. In each + header, u32@4 = picture number and u32@8 = offset of its data from the start of the picture + block. (One picture: identical to the `.i` entry.) +- Fonts: the 44-byte headers of all `N.zi` entries first, then the rest of each font; u32@0x18 + of a header = offset of that font's data from the start of the font block. + +### 7.3 Code blob + +Offsets in this section are relative to `base`. + +``` +u32 0x24 + 6*pages, u32 3, u32 0x1C, u32 4, u32 0x20, u32 4, u32 0x24, u32 6*pages, u32 0, u32 0 +page names: (u32 name hash, u16 page number), sorted by hash +program +per page: code blocks of its components, name list, RAM image, init code +``` + +The first ten words describe three tables at 4 + (0x1C, 0x20, 0x24) with sizes 4, 4, 6*pages; +the first two are empty when the program declares no global variables (**OPEN**: their format). + +**Name hash** = CRC of the name NUL-padded to 14 bytes. Names are not stored anywhere else. + +Everything after the page names is a sequence of **chunks**: `u32 length` + bytes. A chunk is one +tokenised script line; an empty chunk ends a section. + +- Program: one chunk per `Program.s` statement (comments and blank lines dropped), then a chunk + `page 0` added by the Editor, then the empty chunk. +- Component block (the page itself is component 0): + - draw section -- lines, empty chunk. Page: `fill x,y,w,h,` (command `09 0D 04`, colour + as a reference to the page's `bco` record), then the Preinitialize event lines. + Types that draw themselves (progress bar, gauge, QR code, text ...) have their own script + here, generated from the attributes (**OPEN** per type). + - erase section (types with a rectangle only; empty for the page) -- `fill x,y,w,h,`, + empty chunk. + - events: a label chunk (`down`, `up`, `unload`, `timer`, `slide` -- plain text), the event's + lines, empty chunk. Page: down, up, unload. Hotspot: down, up. Variable: none. +- After the last component of a page: one chunk with the name list `(u32 hash, u16 id)` sorted + by hash; one chunk with the RAM image followed by 8 zero bytes; type-specific init lines + (timer, slider, touch cap -- **OPEN**); the chunk `09 21 08`; the page's Postinitialize lines; + empty chunk. + +RAM image: 8 x FF, then every RAM-stored attribute of every component in component order, +each aligned to its size, each component padded to 4. For a page: `sta` (u8), pad, `bco` (u16). + +### 7.4 Attribute records (24 bytes) + +Every component owns a fixed number of record slots (page 33, hotspot 27, variable 11, +timer 11, touch cap 10, progress bar 33, slider 40, gauge 41, QR code 33 ...); unused slots are +all zero. Record numbers are global (pages concatenated) in script references and page-relative +in the object records. + +| Offset | Field | +|---|---| +| 0 | u32 8 for a page, 16 for other components | +| 4 | i32 value (constant attributes) or 4 + offset into the RAM image | +| 8, 12 | i32 maximum, minimum | +| 16 | u8 page number, u8 component id of the component to redraw on change; FF FF = none | +| 18 | u16 change action (0 = none; timers use 1 for `tim`, 2 for `en`) | +| 20 | u8 kind: 0x01 constant; 0x0B/0x0C/0x0D/0x0E x/y/w/h; 0x08 endx/endy; 0x21, 0x22, 0x31, 0x32, 0x39, 0x3F RAM-stored | +| 21 | u8 size in bytes | +| 22 | u16 0 | + +Slots seen in all types: 0 `type`, 1 `id`, 3 `vscope`, 9-14 `x y w h endx endy`. + +### 7.5 Page table (16 bytes per page) and object records (232 bytes per component) + +Page: u16 first object number, u16 component count, u32 offset of the name-list chunk, u32 +offset of the page's first attribute record (from `base`), u32 RAM image size. + +Object: + +| Offset | Field | +|---|---| +| 0 | u8 type, u8 id, u8 0, u8 flags (0x37 visible types, 0x27 timer/touch cap, 0x07 variable) | +| 4 | u32 offset of the draw section, FFFFFFFF if empty | +| 8 | u32 offset of the erase section, FFFFFFFF if none/empty | +| 0x0C-0x1B | more code offsets (0x14: timer event), FF when unused | +| 0x1C | u32 8 / 16 as in the records | +| 0x20 | `00 00 7F 00`, u32 0 | +| 0x28 | u16 x, y, w, h, endx, endy (0,0,1,1,0,0 for invisible types) | +| 0x34 | u32 offset of the component's code block | +| 0x38 | 88 x u16: record number (page-relative) of well-known attributes, FFFF = absent. Positions: 1 vscope, 2 sta, 4 bco, 16 pco, 20 val, 42 x, 43 y, 44 endx, 45 endy, 46 w, 47 h, 49 id, 62 type (**OPEN**: the others) | diff --git a/docs/NOTES.md b/docs/NOTES.md index 09d6b6b..4f5693e 100644 --- a/docs/NOTES.md +++ b/docs/NOTES.md @@ -130,3 +130,28 @@ The file length is always a multiple of 4 (the page section is padded with zeros Other dead ends, for the record: seeds from header fields; header checksum fields zeroed/0xFF in every combination; the layout table de-obfuscated before checksumming; prefixes built from length/layout words; runs of 00/FF bytes before the file. + +### Page section solved, first builder (2026-11-16) + +Format: FORMAT.md section 7. `nxhmi build-tft` reproduces the Editor byte for byte for projects +made of pages (solid colour), hotspots and int variables, with any pictures and fonts, on +NX4832T035. Check with `NXHMI_TFT_SAMPLES= python -m unittest +discover -s tests -p test_tft_build.py` (samples stay outside the repo). + +How the samples are made: one tiny project per question (nxhmi builds them), compiled in the +Editor by a click script (open file, Compile, File > TFT file output > Output), ~45 s each. +A project whose script does not compile yields no TFT -- e.g. `sys0` is only known when +`Program.s` declares it (`int sys0=0,sys1=0,sys2=0`, as in the Editor's default program). + +Useful facts not obvious from the format description: + +- Text-like components need a font in the project or the compile fails. +- Every component type brings its own generated script (draw section, init lines); e.g. a + progress bar's draw section is three `fill`-like lines computed from x/y/w/h and `val`. + These must be collected per type -- the bulk of the remaining work. +- Record references in scripts (`01` + u32) are global record numbers, so adding a component + renumbers every reference after it. + +Not done: script tokens beyond `baud=115200` / `page N`; timers, sliders and every drawing +component; global variables declared in Program.s (blob header tables 0 and 1); other models, +portrait, other series; and nothing has been loaded onto a display. diff --git a/src/nxhmi/cli.py b/src/nxhmi/cli.py index 1b47c32..f470c1a 100644 --- a/src/nxhmi/cli.py +++ b/src/nxhmi/cli.py @@ -169,6 +169,32 @@ def cmd_dump(a): print(json.dumps(project.page_to_json(e.name, p, a.encoding, not a.raw), indent=1, ensure_ascii=False)) +def cmd_build_tft(a): + from . import tft_build + try: + out = tft_build.build_tft(HMIFile.load(a.hmi), a.editor_dir) + except tft_build.Unsupported as e: + print("error: cannot compile yet:", e, file=sys.stderr) + return 3 + except FileNotFoundError as e: + print("error:", e, file=sys.stderr) + return 2 + with open(a.tft, "wb") as fh: + fh.write(out) + print("wrote %s (%d bytes)" % (a.tft, len(out))) + + +def cmd_tft_info(a): + from . import tft + with open(a.tft, "rb") as fh: + data = fh.read() + info = tft.header_info(data) + info["header_checksums_ok"] = tft.verify_header(data) + info["trailer_ok"] = tft.verify_trailer(data) + print(json.dumps(info, indent=1)) + return 0 if info["header_checksums_ok"] and info["trailer_ok"] else 1 + + def main(argv=None): ap = argparse.ArgumentParser(prog="nxhmi", description="Read, verify and write Nextion Editor .HMI projects") sub = ap.add_subparsers(dest="cmd", required=True) @@ -210,6 +236,12 @@ def main(argv=None): s.add_argument("-o", "--out"); s.set_defaults(f=cmd_new_page) s = sub.add_parser("dump", help="print pages as JSON"); s.add_argument("hmi"); s.add_argument("--page") s.add_argument("--encoding", default="latin-1"); s.add_argument("--raw", action="store_true"); s.set_defaults(f=cmd_dump) + s = sub.add_parser("build-tft", help="compile to .tft (research stage: NX4832T035, few component types; " + "needs a Nextion Editor installation for the firmware files)") + s.add_argument("hmi"); s.add_argument("tft") + s.add_argument("--editor-dir", default=r"C:\Program Files (x86)\Nextion Editor"); s.set_defaults(f=cmd_build_tft) + s = sub.add_parser("tft-info", help="header fields, layout table and checksum status of a .tft") + s.add_argument("tft"); s.set_defaults(f=cmd_tft_info) a = ap.parse_args(argv) try: return a.f(a) or 0 diff --git a/src/nxhmi/tft.py b/src/nxhmi/tft.py index 1ada377..08e6855 100644 --- a/src/nxhmi/tft.py +++ b/src/nxhmi/tft.py @@ -34,6 +34,14 @@ def header_info(tft: bytes) -> dict: } +OBJ_SIZE, REC_SIZE = 232, 24 # object record / attribute record in the page section + + +def name_hash(name: str) -> int: + """Hash under which page and component names are stored: CRC of the name NUL-padded to 14.""" + return word_per_byte_crc(name.encode("latin-1")[:14].ljust(14, b"\0")) + + RES_START = 0x10000 # resource area: firmware files, pictures, fonts OFF_RES_SIZE, OFF_RES_CRC, OFF_HEAD1_CRC, OFF_HEAD2_CRC = 0x40, 0x44, 0xC4, 0x18C diff --git a/src/nxhmi/tft_build.py b/src/nxhmi/tft_build.py new file mode 100644 index 0000000..5668ab4 --- /dev/null +++ b/src/nxhmi/tft_build.py @@ -0,0 +1,223 @@ +"""Build a .tft from a .HMI project -- research stage. See docs/FORMAT.md section 7. + +What is reproduced byte for byte (Nextion Editor 1.68.2, model NX4832T035): header, resource +area (the firmware files are read from the user's own Editor installation; pictures, fonts), page +section for the component types in ``TYPES`` and the script subset in :mod:`nxhmi.tft_code`, +layout table and all checksums. Anything else raises :class:`Unsupported` -- never a guess. +""" +from __future__ import annotations + +import os +import re +import struct + +from . import tft, tft_code +from .container import HMIFile +from .page import Page +from .tft_code import Unsupported + +FIRMWARE_FILES = ("input.bin", "asp0.dll", "cd0.dll", "qr0.bin", "syscom.bin", "h0-0.dll", "h1-0.dll") +DEFAULT_EDITOR_DIR = r"C:\Program Files (x86)\Nextion Editor" +SUPPORTED_MODEL = 0xB23BC018 # NX4832T035_011, the only model measured +BLOCK = 0x10000 + +# attribute map positions in the object record (u16 table at +0x38) +MAP = {"vscope": 1, "sta": 2, "bco": 4, "val": 20, "x": 42, "y": 43, "endx": 44, "endy": 45, "w": 46, "h": 47, + "id": 49, "type": 62} +# attribute record templates: name -> (slot, kind, size, max, min) +_COMMON = {"type": (0, 0x01, 1, 255, 0), "id": (1, 0x01, 1, 255, 0), "vscope": (3, 0x01, 1, 1, 0)} +_RECT = {"x": (9, 0x0B, 2, 6000, -6000), "y": (10, 0x0C, 2, 6000, -6000), "w": (11, 0x0D, 2, 2000, -2000), + "h": (12, 0x0E, 2, 2000, -2000), "endx": (13, 0x08, 2, 32767, -32768), "endy": (14, 0x08, 2, 32767, -32768)} +# type id -> description. ``ram`` attributes live in the page's RAM image (in this order); +# ``owner``: whether records name the component to refresh when the attribute changes. +TYPES = { + 121: dict(name="page", slots=33, flag=0x37, group=8, const={**_COMMON, **_RECT}, + ram={"sta": (23, 0x21, 1, 2, 0), "bco": (24, 0x32, 2, 65535, 0)}, owner=True), + 109: dict(name="hotspot", slots=27, flag=0x37, group=16, const={**_COMMON, **_RECT}, ram={}, owner=True), + 52: dict(name="variable", slots=11, flag=0x07, group=16, + const={**_COMMON, "sta": (7, 0x01, 1, 1, 0)}, + ram={"val": (8, 0x39, 4, 2147483647, -2147483648)}, owner=False), +} + + +def _chunk(b: bytes) -> bytes: + return struct.pack(" int: + return int.from_bytes(attr.value, "little") + + +def _s(attr, signed) -> int: + v, bits = _u(attr), 8 * len(attr.value) + return v - (1 << bits) if signed and v >> (bits - 1) else v + + +def resource_area(hmi: HMIFile, editor_dir: str) -> tuple[bytes, dict]: + """Directory + firmware files + pictures + fonts, zero-padded to a 64 KB multiple.""" + def numbered(ext): + names = [n for n in hmi.names() if re.fullmatch(r"\d+\." + ext, n)] + return [hmi.read(n) for n in sorted(names, key=lambda s: int(s.split(".")[0]))] + + def packed(blobs, head, off_pos, idx_pos=None): + heads, datas, o = [], [], head * len(blobs) + for i, b in enumerate(blobs): + h = bytearray(b[:head]) + struct.pack_into(" (code blob, attribute records, page table, object records, program offset).""" + names = sorted((tft.name_hash(p.components[0].get("objname")), i) for i, p in enumerate(pages)) + n = len(pages) + blob = bytearray(struct.pack("<10I", 0x24 + 6 * n, 3, 0x1C, 4, 0x20, 4, 0x24, 6 * n, 0, 0)) + for h, i in names: + blob += struct.pack(" bytes: + main = hmi.read("main.HMI") + model, = struct.unpack_from(" the tokenised form stored in a .tft (research stage; see docs/FORMAT.md).""" +from __future__ import annotations + +import re +import struct + + +class Unsupported(Exception): + """The project uses something whose compiled form has not been worked out yet.""" + + +class Context: + pass + + +def group_lines(group) -> list[str]: + return [(r if isinstance(r, bytes) else r.value).decode("latin-1") for r in group.records] + + +def statements(lines) -> list[str]: + out = [] + for ln in lines: + ln = ln.split("//")[0].strip() + if ln: + out.append(ln) + return out + + +def compile_line(line: str, ctx: Context) -> bytes: + if line == "baud=115200": + return b"\x04\x04\x0e\x00\x00=\x03" + struct.pack(".HMI + .tft pairs made with Nextion Editor 1.68.2.""" +import glob +import os +import shutil +import struct +import tempfile +import unittest + +from helpers import HAVE_PIL + +from nxhmi import cli, components as C, project, tft, tft_build +from nxhmi.page import Page + +SAMPLES = os.environ.get("NXHMI_TFT_SAMPLES") + + +class Build(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + for i, n in enumerate(tft_build.FIRMWARE_FILES): + with open(os.path.join(self.tmp, n), "wb") as fh: + fh.write(bytes([i + 1]) * (100 + 7 * i)) + + def tearDown(self): + shutil.rmtree(self.tmp) + + def project(self): + h = project.new_project("NX4832T035_011") + p = Page.parse(h.read("0.pa")) + C.add(p, "hotspot", x=10, y=20, w=30, h=40) + C.add(p, "variable") + h.write("0.pa", p.pack()) + project.add_page(h, "second") + return h + + def test_structure(self): + out = tft_build.build_tft(self.project(), self.tmp) + self.assertEqual(len(out) % 4, 0) + self.assertTrue(tft.verify_header(out)) + self.assertTrue(tft.verify_trailer(out)) + self.assertEqual(struct.unpack_from("