From 193bcd5d9736ac919987f099469ffc14a29f2812 Mon Sep 17 00:00:00 2001 From: MangaValk Date: Sun, 15 Nov 2026 16:49:01 +0100 Subject: [PATCH] Intelligent pictures (format 4) and direction-aware picture order Format 4 decoded from Editor output: control words of eight 4-bit ops (runs up to 13, 12-bit long runs, literal blocks) over 8 colour slots, plus a run-length 7-bit alpha section. The encoder reproduces all 28 Editor samples byte for byte and the Editor keeps our pictures as is. Pictures are stored rotated by the display direction, byte 0 of N.i is 0x0A + direction; this applies to format 0 too. add_picture now writes the right format and order for the project; the renderer decodes format 4 and blends alpha. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/nextion-hmi/SKILL.md | 4 +- README.md | 2 +- docs/COVERAGE.md | 16 +- docs/FORMAT.md | 39 ++++- src/nxhmi/pictures.py | 224 +++++++++++++++++++++++----- src/nxhmi/project.py | 7 +- src/nxhmi/render.py | 5 +- tests/test_media.py | 60 ++++++++ 8 files changed, 301 insertions(+), 56 deletions(-) diff --git a/.claude/skills/nextion-hmi/SKILL.md b/.claude/skills/nextion-hmi/SKILL.md index 839b13d..c5f5f66 100644 --- a/.claude/skills/nextion-hmi/SKILL.md +++ b/.claude/skills/nextion-hmi/SKILL.md @@ -33,8 +33,8 @@ Back up the user's file first (copy it) unless they said not to; edits write in `direction` and `encoding`. - Text needs a font: `font` is a font id. A new project has **no fonts** -- call `add_font` first (height in px; no TrueType path = bundled CC0 font). `txt_maxl` must be >= the text length. -- Pictures: `add_picture` returns an id for `pic` / `picc` (crop) / `pic2` etc. Basic-series - displays only (Intelligent compressed pictures are not supported yet). +- Pictures: `add_picture` returns an id for `pic` / `picc` (crop) / `pic2` etc. Works for Basic + and Intelligent displays (Intelligent keeps PNG transparency). - Event code is Nextion script, one line per list item, e.g. `["page settings"]`, `["n0.val++", "t0.txt=\"hi\""]`. Events: `codesdown` (press), `codesup` (release), `codesload` / `codesloadend` (page pre/post-initialize), `codesunload`, `codestimer`, `codesslide`. diff --git a/README.md b/README.md index 1bbca9f..bf13adc 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Every percentage is a count with a stated basis, so it can be checked (details i | Checksums (`.HMI`) | **100 %** (3/3) | page, project header, file table | | Component types (read, write, create) | **100 %** (31/31) | all types in the Editor's toolbox, incl. 11 Intelligent-only | | Display series | **40 %** (2/5) | Basic (T) and Intelligent (P) confirmed; Discovery, Enhanced, Edge not sampled | -| Picture formats | **50 %** (1/2) | raw RGB565 (Basic) done; compressed format (Intelligent) not yet | +| Picture formats | **100 %** (2/2) | raw RGB565 (Basic) and run-length + alpha (Intelligent), byte-identical to the Editor, every display direction | | Font modes | **50 %** (1/2) | black/white done (incl. TrueType import); anti-aliased not yet | | Readable attribute names in JSON | **65 %** (20/31 types) | value lists taken from the Editor for all Basic types; Intelligent-only types still raw numbers | | Rendering, Basic components | **100 %** (20/20 pixel-exact) | incl. gauges and QR codes (own encoder, no dependencies) | diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md index 31d1b8c..143f6a7 100644 --- a/docs/COVERAGE.md +++ b/docs/COVERAGE.md @@ -13,9 +13,9 @@ byte for byte, or a file written by nxhmi was opened in the Editor and showed th | Checksums (page, project header, file table) | **complete**, verified | | Components | **31 / 31 types** (20 Basic + 11 Intelligent-only) read/write/create | | Readable JSON (enum names, colours) | done for every value list the Editor documents | -| Pictures | Basic series **complete**; Intelligent compressed format **missing** | +| Pictures | **complete**: format 0 (Basic) and format 4 (Intelligent, with alpha), all display directions | | Fonts | black/white **complete** incl. generation from TrueType; anti-aliased **missing** | -| Rendering pages like the Editor | **pixel-identical** for most components for every Basic component incl. picture skins, radios, gauges and QR codes | +| Rendering pages like the Editor | **pixel-identical** for every Basic component incl. picture skins, radios, gauges and QR codes | | New projects from scratch | `project.new_project(model)` for 12 Basic/Intelligent models; the generated demo is byte-identical to the Editor's re-save | | Compiling `.tft` firmware | **research only** -- not possible yet | @@ -60,9 +60,11 @@ byte for byte, or a file written by nxhmi was opened in the Editor and showed th `format`, `dir`, `en`, `mode`, `psta`, `dez`, variable type, QR logo -- taken from the Editor's own attribute notes. Colours as `#rrggbb`. Still round-trips byte for byte. -### Pictures (Basic series) -- `N.i` (RGB565, panel-native rotated order) and `N.is` (original file) -- encoder output is - byte-identical to the Editor's; adding pictures verified in the Editor. +### Pictures +- `N.i` format 0 (Basic, raw RGB565) and format 4 (Intelligent, run-length + 7-bit alpha), stored + rotated for the display direction, and `N.is` (original file) -- encoder output is + byte-identical to the Editor's (format 4: 28 samples, all four directions); the Editor keeps + our format-4 pictures unchanged; adding pictures verified in the Editor. ### Fonts - `.zi` version 6, black/white: decode/encode byte-identical to the Editor (224-glyph font). @@ -86,10 +88,10 @@ byte for byte, or a file written by nxhmi was opened in the Editor and showed th ### `.HMI` side | Item | Notes | |---|---| -| Intelligent picture format 4 | block codec with 8-bit alpha; first findings in FORMAT.md | +| Picture quality below 100 (Editor picture list "Q:") | not studied | | Anti-aliased fonts, multi-byte encodings (GB2312, Big5, ...) | only B/W single-byte fonts seen | | Other display series | Discovery / Enhanced / Edge not sampled; only Basic + Intelligent templates | -| Display direction | byte 12 decoded and written (`new_project(portrait=...)`); picture/glyph order verified for one direction (90° Basic) only | +| Display direction | byte 12 decoded and written (`new_project(portrait=...)`); picture order verified for all four (format 4); font glyph order for 90° only | | Some header fields | main.HMI bytes 8, 13, 36, 0x41; page header 21/23; entry bytes 25-27 | | Font name shown in the Editor's Fonts panel | cosmetic, generated fonts show an odd label | | Media resources | video / audio / Gmov files themselves (not just the components) | diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 23d6a2a..0ff22a3 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -204,15 +204,38 @@ PNG that was imported): Format 0: RGB565 little-endian, converted by plain truncation (`r>>3, g>>2, b>>3`), stored in the panel's native scan order -- column by column, each column bottom-to-top: `offset = 2 * (x * height + (height - 1 - y))`. Verified pixel-exact (480×320 and 64×64, Editor -output byte-identical to ours). **OPEN**: whether the order changes with the project's display -direction (all samples: main.HMI byte 12 = 1, i.e. 90° on a Basic panel). +output byte-identical to ours). The order follows the display direction, see format 4. -Format 4 (Intelligent series, **OPEN**): header field at 20 is non-zero for small pictures -(e.g. 2640 for a 3307-byte payload); payload starts with u32 1, u32 height, u32 width (note: -swapped vs. the header, i.e. panel orientation), ... then a block codec: runs of RGB565 colours -followed by small index bytes (e.g. `18c6 38c6 18c6 38c6 2131 1111`), with a variable number of -bytes per 4x4 block, and full 8-bit alpha (a PNG with 256 alpha levels was imported). A 480x320 -photo compresses to 215,052 bytes at "Q:100". +Format 4 (Intelligent series) -- decoded and re-encoded byte-identically (28 Editor samples, all +four directions, with and without alpha): + +- Header as above; byte 0 = 0x0A + display direction (also for format 0: a landscape Basic + project, direction 1, writes 0x0B); +16 data length; +20 offset of the alpha section within the + data (0 = opaque picture). +- Pixel order: the picture rotated by the display direction (`pictures.stored_order`): 0 = row + by row, 1 = column by column bottom to top (same as format 0 on Basic), 2 = 180°, 3 = columns + from the right, top down. +- Data: u32 1, u32 stored width, u32 stored height (swapped for directions 1/3), u32 0, u32 0, + then blocks. A block is a u32 control word of eight 4-bit ops (low nibble first): + - `1`..`13`: run of that many pixels in the colour of the op's slot (slot = nibble index); + - `14`: run of N pixels, N in the next three nibbles (12 bits), colour of the op's slot; + - `15`: N literal pixels (N in the next three nibbles); their colours follow the control + word and end the block; + - `0`: nothing. + + Run blocks carry 8 colour slots (RGB565 LE) after the control word; the last block is padded + with zero slots to 8. +- How the Editor encodes: split the pixels into runs of one colour of at most 13; pack 8 runs + per block; a block of eight 13-runs of one colour becomes op 14 (consecutive such blocks merge, + N = 104 × blocks, at most 39); a block of eight single pixels becomes a literal block + (consecutive ones merge, N = 8 × blocks). +- Alpha section: 7-bit alpha (`a * 127 // 255`) per pixel in storage order, run-length coded -- + a byte with bit 7 set is one pixel of alpha `b & 0x7F`, otherwise `b` is the alpha and the next + byte the run length (≤ 255). Fully transparent pixels store colour 0. +- Pictures of 1-2 pixels stay format 0 (seen: 1x1, 2x1, 1x2; 8x1 is converted); nxhmi does the + same. The Editor shows a message box per picture while converting format-0 pictures it finds + in an Intelligent project. +- **OPEN**: the "Q:100" quality shown in the Editor's picture list -- only quality 100 studied. Adding a picture = `N.is` + `N.i` + a `i`/`N.i` resource-list entry (kept grouped with the other `i` entries) -- verified in the Editor. diff --git a/src/nxhmi/pictures.py b/src/nxhmi/pictures.py index 75e5909..0ff9b5d 100644 --- a/src/nxhmi/pictures.py +++ b/src/nxhmi/pictures.py @@ -13,18 +13,34 @@ ``N.i`` header (24 bytes) + pixels:: - 0 4 0B 64 01 format 0 = raw RGB565 (Basic series), 4 = compressed (Intelligent; OPEN) + 0 4 <0A + direction> 64 01 format 0 = raw RGB565 (Basic), 4 = run-length (Intelligent) 4 4 0 8 4 24 (data offset) 12 2 width 14 2 height 16 4 data length (format 0: width * height * 2) - 20 4 0 + 20 4 0; format 4: offset of the alpha section inside the data, 0 = no alpha -Format 0 pixels are RGB565 little-endian, converted by plain truncation (r>>3, g>>2, b>>3) and -stored in the panel's native order: rotated 90 degrees, i.e. for each source column x, the rows -from bottom to top -- byte offset 2 * (x * height + (height - 1 - y)). Verified pixel-exact against -Editor output for a 480x320 and a 64x64 picture on a project with display direction 270 (OPEN: -other directions, main.HMI byte 13). +Pixels are RGB565 little-endian, converted by plain truncation (r>>3, g>>2, b>>3), stored in the +panel's order: the picture rotated by the project's display direction (main.HMI byte 12), see +``stored_order`` -- a landscape Basic project (direction 1) stores column by column, bottom to top. +Verified pixel-exact against Editor output for both formats and all four directions (format 4) +or direction 1 (format 0). Format 0 data is just the pixels; format 4 is below. + +Format 4 data (Intelligent series), decoded from Editor output (see docs/FORMAT.md): +u32 1, u32 stored width, u32 stored height (swapped for directions 1 and 3), u32 0, u32 0, +then blocks. Each block is a u32 control word of +eight 4-bit ops (low nibble first) followed by its colours (RGB565 LE, pixels in storage order): + +- op 1..13: a run of that many pixels in the colour of the op's slot (slot = nibble index); +- op 14: a run whose 12-bit length is in the next three nibbles (colour of the op's slot); +- op 15: literal pixels, 12-bit count in the next three nibbles; the colours follow the + control word directly and end the block; +- op 0: nothing. + +A run block is followed by 8 colour slots (the last block only by the slots it uses). +Alpha section: one byte per pixel run -- with bit 7 set, a single pixel of alpha `b & 0x7F`; +otherwise `b` is the alpha and the next byte the run length. Alpha is 7-bit (`a * 127 // 255`); +fully transparent pixels store colour 0. Pillow is only needed for converting to/from image files. """ @@ -34,7 +50,6 @@ import io import struct IS_MAGIC = b"\x0a\x64\x01\x01" -I_MAGIC = b"\x0b\x64\x01" def source_info(blob: bytes) -> dict: @@ -59,44 +74,183 @@ def info(blob: bytes) -> dict: return {"format": blob[3], "width": w, "height": h, "length": n} -def to_rgb565(pixels_rgb, width: int, height: int) -> bytes: - """pixels_rgb: callable (x, y) -> (r, g, b).""" - out = bytearray(width * height * 2) - for x in range(width): - for y in range(height): - r, g, b = pixels_rgb(x, y)[:3] - v = (r >> 3) << 11 | (g >> 2) << 5 | (b >> 3) - struct.pack_into(" bytes: - return I_MAGIC + b"\x00" + struct.pack(" int: + return (c[0] >> 3) << 11 | (c[1] >> 2) << 5 | (c[2] >> 3) -def from_image_file(data: bytes): - """Image file bytes (png/jpg/bmp) -> (N.i blob, N.is blob). Needs Pillow.""" +def to_rgb565(pixels_rgb, width: int, height: int, direction: int = 1) -> bytes: + """Format-0 pixel data. pixels_rgb: callable (x, y) -> (r, g, b); truncated to RGB565.""" + return struct.pack("<%dH" % (width * height), *[_565(pixels_rgb(x, y)) for x, y in stored_order(width, height, direction)]) + + +def make_picture(rgb565: bytes, width: int, height: int, direction: int = 1) -> bytes: + """Format-0 ``N.i``. Byte 0 is 0x0A + the project's display direction.""" + return bytes([0x0A + direction % 4, 0x64, 0x01, 0x00]) + struct.pack(" bytes: + """Format-4 ``N.i`` (Intelligent series), byte-identical to Nextion Editor 1.68's output. + pixels_rgba: callable (x, y) -> (r, g, b[, a]). Colours run in groups of up to 13 pixels; + eight ops make a block; blocks of eight 13-runs of one colour merge into a long run (op 14), + blocks of eight single pixels into a literal block (op 15).""" + order = stored_order(width, height, direction) + rgba = [tuple(pixels_rgba(x, y)) + (255,) for x, y in order] + px = [_565(c) if c[3] else 0 for c in rgba] + alpha = [c[3] * 127 // 255 for c in rgba] + has_alpha = any(a != 127 for a in alpha) + n = len(px) + ops, i = [], 0 + while i < n: + j = i + while j < n and px[j] == px[i] and j - i < 13: + j += 1 + ops.append((px[i], j - i)) + i = j + blocks = [ops[k:k + 8] for k in range(0, len(ops), 8)] + pw, ph = (height, width) if direction % 2 else (width, height) + out = bytearray(struct.pack(" (N.i blob, N.is blob). Needs Pillow. + series 0 (Basic) writes format 0, series 3 (Intelligent) format 4 with alpha.""" from PIL import Image img = Image.open(io.BytesIO(data)) ftype = (img.format or "png").lower()[:3] - rgb = img.convert("RGB") - px = rgb.load() - w, h = rgb.size - return make_picture(to_rgb565(lambda x, y: px[x, y], w, h), w, h), make_source(data, w, h, ftype) + w, h = img.size + if series == 3 and w * h > 2: # the Editor keeps 1-2 pixel pictures in format 0 + rgba = img.convert("RGBA").load() + pic = encode_format4(lambda x, y: rgba[x, y], w, h, direction) + else: + rgb = img.convert("RGB").load() + pic = make_picture(to_rgb565(lambda x, y: rgb[x, y], w, h, direction), w, h, direction) + return pic, make_source(data, w, h, ftype) + + +def decode_format4(blob: bytes): + """Format-4 ``N.i`` -> (width, height, [rgb565 per pixel], [alpha 0..127] or None), in storage order.""" + w, h, plen, aoff = struct.unpack_from("> (4 * i)) & 15 for i in range(8)] + [0, 0, 0] + i, runs, literal = 0, [], 0 + while i < 8: + op = nib[i] + if op == 15: + literal = nib[i + 1] | nib[i + 2] << 4 | nib[i + 3] << 8 + break + if op == 14: + runs.append((i, nib[i + 1] | nib[i + 2] << 4 | nib[i + 3] << 8)) + i += 4 + continue + if op: + runs.append((i, op)) + i += 1 + if literal: + px.extend(struct.unpack_from("<%dH" % literal, p, o)) + o += 2 * literal + continue + used = 0 + for slot, length in runs: + colour, = struct.unpack_from("> 11, (v >> 5) & 63, v & 31 + return (r << 3 | r >> 2, g << 2 | g >> 4, b << 3 | b >> 2) def to_image(blob: bytes): - """Decode a format-0 ``N.i`` to a Pillow image.""" + """Decode an ``N.i`` (format 0 or 4) to a Pillow image; RGBA when the picture has alpha.""" from PIL import Image d = info(blob) - if d["format"] != 0: - raise NotImplementedError("picture format %d (Intelligent compressed) not supported yet" % d["format"]) w, h = d["width"], d["height"] - img = Image.new("RGB", (w, h)) - px = img.load() - for x in range(w): - for y in range(h): - v, = struct.unpack_from("> 11, (v >> 5) & 63, v & 31 - px[x, y] = (r << 3 | r >> 2, g << 2 | g >> 4, b << 3 | b >> 2) + direction = (blob[0] - 0x0A) % 4 + order = stored_order(w, h, direction) + if d["format"] == 4: + _, _, px, alpha = decode_format4(blob) + elif d["format"] == 0: + px, alpha = struct.unpack_from("<%dH" % (w * h), blob, 24), None + else: + raise NotImplementedError("picture format %d not supported" % d["format"]) + img = Image.new("RGBA" if alpha else "RGB", (w, h)) + out = img.load() + for i, (x, y) in enumerate(order): + out[x, y] = _rgb(px[i]) + (((alpha[i] * 255 + 63) // 127,) if alpha else ()) return img diff --git a/src/nxhmi/project.py b/src/nxhmi/project.py index 0faed7a..cff2720 100644 --- a/src/nxhmi/project.py +++ b/src/nxhmi/project.py @@ -215,12 +215,15 @@ def main_insert_resource(main: bytes, rtype: str, name: str) -> bytes: def add_picture(h: HMIFile, image_file: bytes) -> int: - """Add a picture (png/jpg/bmp bytes) to the project; returns its picture id. Basic series only.""" + """Add a picture (png/jpg/bmp bytes) to the project; returns its picture id. Written the way + the Editor would: format 0 on Basic, format 4 (with alpha) on Intelligent displays, rotated + for the project's display direction.""" from . import pictures n = 0 while h.find("%d.i" % n): n += 1 - pic, src = pictures.from_image_file(image_file) + main = h.read("main.HMI") + pic, src = pictures.from_image_file(image_file, direction=main[12] % 4, series=main[11]) h.write("%d.is" % n, src) h.write("%d.i" % n, pic) h.write("main.HMI", main_insert_resource(h.read("main.HMI"), "i", "%d.i" % n)) diff --git a/src/nxhmi/render.py b/src/nxhmi/render.py index d6639c8..cc28075 100644 --- a/src/nxhmi/render.py +++ b/src/nxhmi/render.py @@ -185,7 +185,10 @@ class Renderer: w = min(w, src.width - sx) h = min(h, src.height - sy) if w > 0 and h > 0: - img.paste(src.crop((sx, sy, sx + w, sy + h)), (dx, dy)) + part = src.crop((sx, sy, sx + w, sy + h)) + # Intelligent pictures can carry alpha: blended over what is below (blend not yet + # compared with the Editor pixel by pixel) + img.paste(part, (dx, dy), part if part.mode == "RGBA" else None) def _box(self, c): x, y, w, h = c.get("x"), c.get("y"), c.get("w"), c.get("h") diff --git a/tests/test_media.py b/tests/test_media.py index 44a4017..9247f37 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -27,6 +27,66 @@ class Pictures(unittest.TestCase): self.assertEqual(pictures.source_file(s), b"PNGDATA") self.assertEqual(pictures.source_info(s)["type"], "png") + def test_stored_order_follows_direction(self): + self.assertEqual(pictures.stored_order(2, 2, 0), [(0, 0), (1, 0), (0, 1), (1, 1)]) + self.assertEqual(pictures.stored_order(2, 2, 1), [(0, 1), (0, 0), (1, 1), (1, 0)]) + self.assertEqual(pictures.stored_order(2, 2, 2), [(1, 1), (0, 1), (1, 0), (0, 0)]) + self.assertEqual(pictures.stored_order(2, 2, 3), [(1, 0), (1, 1), (0, 0), (0, 1)]) + self.assertEqual(pictures.make_picture(b"", 0, 0, 1)[0], 0x0B) # 0x0A + direction + + +def _distinct(i): + return (((i * 37) % 32) << 3, ((i * 23 + 5) % 64) << 2, ((i * 11 + 3) % 32) << 3) + + +class Format4(unittest.TestCase): + """Intelligent pictures: Nextion Editor 1.68 output for small test images (see FORMAT.md).""" + EDITOR = { # name: (direction, pixel function, w, h, Editor bytes) + "solid 16x16": (0, lambda x, y: (255, 0, 0), 16, 16, + "0a6401040000000018000000100010003c0000000000000001000000100000001000000000000000000000000e0d0000" + "00f800f800f800f800f800f800f800f8dd9d000000f800f800f800f80000000000000000"), + "distinct 3x3": (0, lambda x, y: _distinct(y * 3 + x + 1), 3, 3, + "0a6401040000000018000000030003003c0000000000000001000000030000000300000000000000000000008f000000" + "8e2b795644792fa41acfe5f1d01cbb4701000000866a0000000000000000000000000000"), + "alpha 4x4": (0, lambda x, y: (255, 0, 0, (y * 4 + x) * 17), 4, 4, + "0a64010400000000180000000400040038000000280000000100000004000000040000000000000000000000d1020000" + "000000f800f80000000000000000000080889099a1aab2bbc3ccd4dde5eef6ff"), + "alpha 50% 4x4": (0, lambda x, y: (0, 0, 255, 128), 4, 4, + "0a6401040000000018000000040004002a0000002800000001000000040000000400000000000000000000003d000000" + "1f001f000000000000000000000000003f10"), + "distinct 5x7, 90 degrees": (1, lambda x, y: _distinct(y * 5 + x + 1), 5, 7, + "0b6401040000000018000000050007006c0000000000000001000000070000000500000000000000000000000f020000" + "d8d961130a4db3865cb8e5f18e2ba3044c3ef5779ea927e3d01c79568e2f3761c09a69d4120ebb4744797952028cab" + "c554fffd30866a2fa4447dedb696e83f2211010000c85b71951acf00000000000000000000"), + } + + def test_encode_matches_editor(self): + for name, (d, f, w, h, hexed) in self.EDITOR.items(): + self.assertEqual(pictures.encode_format4(f, w, h, d).hex(), hexed, name) + + def test_long_run_and_literals(self): + blob = pictures.encode_format4(lambda x, y: (255, 0, 0), 40, 40, 0) + self.assertEqual(struct.unpack_from("> 3 for v in img.getpixel((x, y))[:3]), (r >> 3, g >> 3, b >> 3)) + self.assertLess(img.getpixel((0, 3))[3], 255) + @unittest.skipUnless(HAVE_PIL, "needs Pillow") def test_image_roundtrip(self): from PIL import Image