diff --git a/.dockerignore b/.dockerignore index 5234a53..27e1df4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,7 +42,12 @@ vendor/mock_babylin_wrapper.py vendor/*.sdf vendor/Example.sdf -# Other artifacts you don't want round-tripping into the image +# Other artifacts you don't want round-tripping into the image. +# `melexis-bundle/` is the dedicated subdir holding melexis-pkgs.tar.gz; +# the hw build reaches it via `--build-context melexis-bundle=./melexis-bundle` +# (a named context — unaffected by THIS .dockerignore, since named contexts +# only respect a .dockerignore at their own root). +melexis-bundle/ melexis-pkgs.tar.gz # Docker itself doesn't need to copy its own files into the image diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f640db..6ee3873 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,11 +11,14 @@ # DOCKER_BUILDKIT=1 docker build \ # -f docker/Dockerfile -t ecu-tests:hw \ # --build-arg INCLUDE_MELEXIS=1 \ -# --secret id=melexis_tarball,src=./melexis-pkgs.tar.gz \ +# --build-context melexis-bundle=./melexis-bundle \ # . -# → "hw" flavour: also bundles pylin / pymumclient / pylinframe so -# hardware tests can drive a real MUM. The Melexis tarball is -# passed via BuildKit secret — see docs/20_docker_image.md §5. +# → "hw" flavour: also bundles the full Melexis set (mlx, pylin, +# pylinframe, pymumclient, pymlxabc, pymlxchip, pymlxexceptions, +# pymlxgdb, pymlxhex, pymlxloader) so hardware tests can drive a +# real MUM. The tarball is passed via a named build context +# (`--build-context`) bind-mounted at /melexis-bundle for one +# RUN step — see docs/20_docker_image.md §5. # # A matching ../.dockerignore at the repo root excludes .venv/, reports/*, # the deprecated BabyLIN SDK, Python caches, etc. so the build context @@ -32,6 +35,24 @@ ARG PYTHON_VERSION=3.11 +# ╔══════════════════════════════════════════════════════════════════════╗ +# ║ Stub stage — "melexis-bundle" ║ +# ║ ║ +# ║ A no-op `scratch` stage that the builder bind-mounts from when ║ +# ║ extracting the Melexis tarball. For hw builds the caller overrides ║ +# ║ this stage with `--build-context melexis-bundle=` so the dir ║ +# ║ that contains `melexis-pkgs.tar.gz` shows up under /melexis-bundle. ║ +# ║ ║ +# ║ Why this dance: BuildKit's `--mount=type=secret` is capped at 500 ║ +# ║ KiB (secrets are meant for keys, not blobs). `--mount=type=bind` ║ +# ║ has no size limit and never lands in an image layer either, but it ║ +# ║ needs a source to mount from. A named build context overriding a ║ +# ║ stub stage gives us "optional, file-of-any-size, never-in-image" ║ +# ║ semantics without polluting the default build context. ║ +# ╚══════════════════════════════════════════════════════════════════════╝ +FROM scratch AS melexis-bundle + + # ╔══════════════════════════════════════════════════════════════════════╗ # ║ Stage 1 — "builder" ║ # ║ ║ @@ -110,28 +131,36 @@ RUN pip install --upgrade pip wheel \ # Melexis packages step — only runs when INCLUDE_MELEXIS=1. # -# `RUN --mount=type=secret,id=melexis_tarball,required=false` makes the -# secret file available at /run/secrets/melexis_tarball for the duration -# of this RUN only. The content is NEVER baked into any image layer, -# even if you `docker history` later. `required=false` means the secret -# is optional — the mock build doesn't pass one and shouldn't fail. -RUN --mount=type=secret,id=melexis_tarball,required=false \ +# `RUN --mount=type=bind,from=melexis-bundle,…` mounts the named context +# (or its scratch stub, for mock builds) read-only at /melexis-bundle for +# the duration of this RUN only. No image layer ever contains the +# tarball — the bind mount is torn down before the layer is committed. +# +# Hw build supplies the real bundle: +# --build-context melexis-bundle= +# Mock build omits it and the stub `scratch` stage applies, yielding an +# empty /melexis-bundle that the `if` below never reads. +RUN --mount=type=bind,from=melexis-bundle,target=/melexis-bundle,readonly \ if [ "$INCLUDE_MELEXIS" = "1" ]; then \ set -e; \ - # Sanity-check: hw build was requested but the secret wasn't bound. + # Sanity-check: hw build was requested but the bundle wasn't bound. # Fail loudly here rather than producing a "looks-fine" image that # then crashes on `import pylin` at runtime. - test -s /run/secrets/melexis_tarball \ - || { echo 'INCLUDE_MELEXIS=1 but no melexis_tarball secret bound'; exit 2; }; \ + test -s /melexis-bundle/melexis-pkgs.tar.gz \ + || { echo 'INCLUDE_MELEXIS=1 but melexis-pkgs.tar.gz missing — pass --build-context melexis-bundle='; exit 2; }; \ # Discover the venv's site-packages dir (path varies per Python # version) and extract the tarball directly into it. The tarball - # contains three top-level directories: pylin/, pymumclient/, - # pylinframe/ — they slot in as proper packages. + # contains the full Melexis set (mlx, pylin, pylinframe, + # pymumclient, pymlxabc, pymlxchip, pymlxexceptions, pymlxgdb, + # pymlxhex, pymlxloader, pyldfparser, pymbdfparser, pymelibu, + # pymelibuframe) — they slot in as proper packages. SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])"); \ - tar -xzf /run/secrets/melexis_tarball -C "$SITE_PACKAGES"; \ - # Smoke-test the import inside the builder so a corrupt tarball - # fails the build instead of producing a broken runtime image. - python -c "import pylin, pymumclient; print('melexis pkgs OK')"; \ + tar -xzf /melexis-bundle/melexis-pkgs.tar.gz -C "$SITE_PACKAGES"; \ + # Smoke-test the imports inside the builder so a corrupt or + # incomplete tarball fails the build instead of producing a + # broken runtime image. `import pylin` transitively pulls in + # pymlxabc, so checking it here catches missing transitive deps. + python -c "import pylin, pymumclient, pymlxabc; print('melexis pkgs OK')"; \ fi diff --git a/docker/README.md b/docker/README.md index 89c9444..ba047a8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,7 +5,7 @@ This file is just the copy-paste commands. | File | What it is | |---|---| -| `Dockerfile` | Multi-stage image. Mock-only by default; hardware variant via `--build-arg INCLUDE_MELEXIS=1` + a BuildKit secret carrying the Melexis Python packages. | +| `Dockerfile` | Multi-stage image. Mock-only by default; hardware variant via `--build-arg INCLUDE_MELEXIS=1` + a named BuildKit context carrying the Melexis Python packages. | | `compose.hw.yml` | docker-compose service for the hardware variant — host networking, USB device passthrough, reports volume, bench-config bind mount. | | `../.dockerignore` | Excludes `.venv/`, `reports/*`, the deprecated BabyLIN SDK, generated caches, etc. | @@ -238,22 +238,51 @@ changes show up immediately. ### One-time setup — Melexis packages `pylin` / `pymumclient` / `pylinframe` ship inside the Melexis IDE, -not on PyPI. Bundle them into a tarball that you'll pass as a -BuildKit secret: +not on PyPI. They also pull in a handful of transitive Melexis +packages that `pylin` and friends import at module load — if any +are missing the build fails partway through with +`ModuleNotFoundError`. Bundle the **full set** into a dedicated +`melexis-bundle/` subdir at the repo root: ```bash # Adjust the path to where Melexis IDE is installed MELEXIS_SITE="/mnt/c/Program Files/Melexis/Melexis IDE/plugins/com.melexis.mlxide.python_1.2.0.202408130945/python/Lib/site-packages" -tar -czf melexis-pkgs.tar.gz \ +mkdir -p melexis-bundle +tar -czf melexis-bundle/melexis-pkgs.tar.gz \ -C "$MELEXIS_SITE" \ - pylin pymumclient pylinframe + mlx \ + pylin pylinframe pymumclient \ + pymlxabc pymlxchip pymlxexceptions \ + pymlxgdb pymlxhex pymlxloader \ + pyldfparser pymbdfparser pymelibu pymelibuframe ``` -The tarball is gitignored (see `.dockerignore`) and never enters -any image layer — BuildKit's `--mount=type=secret` only exposes it -to the single `RUN` step that copies the packages into -`/opt/venv/lib/python3.x/site-packages/`. +Reason for the long list: `pylin/__init__.py` transitively imports +`pymlxabc` and `pyldfparser`; `pylinframe` pulls in `pymbdfparser` +and the `pymelibu*` pair; the `pymlx*` family pulls in the rest. +Shipping a partial set fails *during* the docker build, not at +runtime — the builder runs a smoke import as the final extraction +step (see Dockerfile §7). + +If you already have a working install in a local venv (e.g. +`~/ecu-tests/.venv/lib/python3.10/site-packages/`), you can tar from +there instead — `mlx*` / `py*` are pure-Python, so a 3.10-sourced +tarball extracts cleanly into the image's Python 3.11 site-packages. +Include the matching `*.dist-info/` directories if you want `pip +list` and metadata-aware tools to work inside the container. + +`melexis-bundle/` is excluded by the root `.dockerignore`, so the +tarball never enters the default build context (no leak into +`/workspace`). The hw build reaches it via a **named build context** +(`--build-context melexis-bundle=./melexis-bundle`) that overrides +a stub `scratch` stage in the Dockerfile; named contexts apply only +the `.dockerignore` at their own root (none here), so the file is +visible there. The single `RUN` that extracts it uses +`--mount=type=bind` from that named context — no size limit +(`--mount=type=secret` is capped at 500 KiB, which the full tarball +exceeds), and like secrets the mount exists only for the duration +of one `RUN`. > **License**: the resulting image contains proprietary Melexis > code. Treat it like the Melexis IDE itself — keep it on a private @@ -265,17 +294,31 @@ to the single `RUN` step that copies the packages into DOCKER_BUILDKIT=1 docker build \ -f docker/Dockerfile -t ecu-tests:hw \ --build-arg INCLUDE_MELEXIS=1 \ - --secret id=melexis_tarball,src=./melexis-pkgs.tar.gz \ + --build-context melexis-bundle=./melexis-bundle \ . ``` +`--build-context melexis-bundle=./melexis-bundle` points the named +context at the `melexis-bundle/` subdir (where the tarball lives). +If you keep the tarball elsewhere, pass that directory instead — any +directory containing a file called `melexis-pkgs.tar.gz` works, e.g. +`--build-context melexis-bundle=/path/to/melexis/bundle/dir`. Don't +point it at the repo root: the root `.dockerignore` filters +`melexis-pkgs.tar.gz` out of every context anchored there, named +or otherwise. + Verify the Melexis packages landed inside the image: ```bash docker run --rm ecu-tests:hw \ - python -c "import pylin, pymumclient, pylinframe; print('OK')" + python -c "import pylin, pymumclient, pylinframe, pymlxabc; print('OK')" ``` +The Dockerfile already runs a smoke import during the secret-mount +step, but it only checks the top-level packages — `import pylin` +transitively imports `pymlxabc`, so if `pymlxabc` is missing the +build fails at that step rather than at runtime. + ### Run the hardware suite — direct `docker run` ```bash @@ -486,7 +529,12 @@ ls reports/ # outputs landed where? | Symptom | Likely cause | Fix | |---|---|---| +| `'docker buildx build' requires 1 argument` | Missing build-context path on the command line | Append `.` (or the repo root) as the final argument to `docker build …` | +| `failed to build: resolve : lstat docker: no such file or directory` | Running `docker build -f docker/Dockerfile …` from somewhere other than the repo root | `cd` into the repo root first (the directory that contains `docker/Dockerfile`) | | `ModuleNotFoundError: No module named 'pylin'` | Image built without `INCLUDE_MELEXIS=1` | Rebuild with the build-arg + secret | +| `ModuleNotFoundError: No module named 'pymlxabc'` (or `pymlxchip`, `pymlxhex`, …) during the build | Melexis tarball is missing a transitive package | Rebuild the tarball with the full package list above | +| `secret melexis_tarball too big. max size 500KiB` | Old `--secret id=melexis_tarball,src=…` flag with the full Melexis bundle | Switch to `--build-context melexis-bundle=` (see Build command above) — BuildKit caps secrets at 500 KiB | +| `INCLUDE_MELEXIS=1 but melexis-pkgs.tar.gz missing` | `--build-context melexis-bundle=` not passed, points at the wrong dir, or points at a dir whose `.dockerignore` filters the tarball (typical foot-gun: passing `=.` from the repo root — the root `.dockerignore` excludes `melexis-pkgs.tar.gz`) | Place the tarball at `melexis-bundle/melexis-pkgs.tar.gz` and pass `--build-context melexis-bundle=./melexis-bundle` | | `Permission denied: '/dev/ttyUSB0'` | Missing `--group-add dialout` | Add it (or the group that owns the device on the host) | | MUM unreachable at 192.168.7.2 | Bridge network instead of host network | Add `--network host` (Linux); on macOS see §4.3 | | Empty `reports/` after run | `/reports` not bind-mounted | Add `-v "$PWD/reports:/reports"` | diff --git a/requirements.txt b/requirements.txt index d1bdb96..456b42b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,17 @@ ldfparser>=0.26,<1 # Pure-Python LDF 1.x/2.x parser; pulls in lark + b configparser>=6,<7 # Optional INI-based config support if you add .ini configs later colorlog>=6,<7 # Colored logging output for readable test logs typing-extensions>=4.12,<5 # Typing backports for older Python versions + +# Transitive PyPI deps of the Melexis stack (pylin / pymumclient / …). +# Installed unconditionally so mock and hw images share one venv layout; +# the size delta in the mock image is a few MB. Version pins come from +# the Requires-Dist metadata of the Melexis packages bundled into +# melexis-bundle/melexis-pkgs.tar.gz — keep them in sync if you upgrade +# the Melexis IDE. +pyparsing>=3.0.9,<3.1 # LDF + MBDF grammar (pyldfparser, pymbdfparser) +natsort>=7.1.0 # Natural-order signal sorting (pymbdfparser) +intelhex>=2.1 # Intel HEX I/O (pymlxchip, pymlxhex) +pygdbmi>=0.9,<0.10 # GDB Machine Interface (pymlxgdb) +crcmod>=1.7 # CRC for MUM framing (pymumclient) +packaging>=20.3 # Version parsing (pymumclient) +zeroconf>=0.37.0 # mDNS discovery of MUM on the bench (pymumclient)