From de01b1596d504b70e81bb236a5be28fe3fda252f Mon Sep 17 00:00:00 2001 From: Guy Sheffer Date: Wed, 11 Mar 2026 01:16:37 +0200 Subject: [PATCH] Add shared distro e2e testing framework and update QEMU boot scripts Introduce src/distro_testing/ -- a reusable QEMU-based end-to-end testing framework for CustomPiOS distros. Includes image preparation (qcow2 conversion, fstab/udev patching, service masking for QEMU compatibility), boot orchestration, SSH wait logic, and a hook system for distro-specific image patches, post-boot setup, screenshot capture, and custom tests. Also update qemu_boot.sh (kernel to 5.10 bullseye, add HTTP/HTTPS/5000 port forwards) and qemu_boot64.sh (Pi 4B machine type, DTB v4 glob, 2G RAM). --- src/distro_testing/Dockerfile.base | 7 + src/distro_testing/README.md | 301 ++++++++++++++++++++ src/distro_testing/scripts/boot-qemu.sh | 39 +++ src/distro_testing/scripts/entrypoint.sh | 155 ++++++++++ src/distro_testing/scripts/prepare-image.sh | 103 +++++++ src/distro_testing/scripts/wait-for-ssh.sh | 44 +++ src/distro_testing/tests/test_boot.sh | 24 ++ src/qemu_boot.sh | 4 +- src/qemu_boot64.sh | 6 +- 9 files changed, 678 insertions(+), 5 deletions(-) create mode 100644 src/distro_testing/Dockerfile.base create mode 100644 src/distro_testing/README.md create mode 100755 src/distro_testing/scripts/boot-qemu.sh create mode 100755 src/distro_testing/scripts/entrypoint.sh create mode 100755 src/distro_testing/scripts/prepare-image.sh create mode 100755 src/distro_testing/scripts/wait-for-ssh.sh create mode 100755 src/distro_testing/tests/test_boot.sh diff --git a/src/distro_testing/Dockerfile.base b/src/distro_testing/Dockerfile.base new file mode 100644 index 00000000..af0f2435 --- /dev/null +++ b/src/distro_testing/Dockerfile.base @@ -0,0 +1,7 @@ +FROM ptrsr/pi-ci:latest + +ENV LIBGUESTFS_BACKEND=direct + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sshpass openssh-client curl socat imagemagick \ + && rm -rf /var/lib/apt/lists/* diff --git a/src/distro_testing/README.md b/src/distro_testing/README.md new file mode 100644 index 00000000..f0d38858 --- /dev/null +++ b/src/distro_testing/README.md @@ -0,0 +1,301 @@ +# CustomPiOS Distro Testing Framework + +A shared e2e testing framework for distros built with CustomPiOS. It boots a built image in QEMU inside a Docker container, waits for SSH, runs test scripts, and captures a QEMU screenshot. + +This directory (`src/distro_testing/`) provides the **generic** infrastructure. Each distro adds its own `testing/` directory with distro-specific tests and hooks. + +## How It Works + +``` +┌─────────────────────────────────────────────────┐ +│ Docker container (ptrsr/pi-ci + test tools) │ +│ │ +│ 1. prepare-image.sh → convert & patch image │ +│ 2. boot-qemu.sh → start QEMU -M virt │ +│ 3. wait-for-ssh.sh → poll until SSH ready │ +│ 4. test_*.sh → run all tests via SSH │ +│ 5. screendump → QEMU monitor capture │ +└─────────────────────────────────────────────────┘ +``` + +## Directory Structure + +### In CustomPiOS (`src/distro_testing/`) + +``` +src/distro_testing/ +├── Dockerfile.base # Reference base image (ptrsr/pi-ci + tools) +├── README.md # This file +├── scripts/ +│ ├── prepare-image.sh # Generic image prep (qcow2, fstab, SSH, etc.) +│ ├── boot-qemu.sh # QEMU boot with configurable ports +│ ├── wait-for-ssh.sh # SSH readiness poller +│ └── entrypoint.sh # Test orchestrator +└── tests/ + └── test_boot.sh # Generic SSH boot test +``` + +### In Your Distro (`testing/`) + +``` +testing/ +├── Dockerfile # Extends base, copies both shared + distro files +├── tests/ +│ └── test_myservice.sh # Distro-specific tests +└── hooks/ + └── prepare-image.sh # (optional) Distro-specific image patches +``` + +## Adding E2E Tests to Your Distro + +### 1. Create the Dockerfile + +Your distro's `testing/Dockerfile` copies the shared framework (placed in `custompios/` by CI) and your distro-specific tests: + +```dockerfile +FROM ptrsr/pi-ci:latest + +ENV LIBGUESTFS_BACKEND=direct + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sshpass openssh-client curl socat imagemagick \ + && rm -rf /var/lib/apt/lists/* + +# Shared framework from CustomPiOS (copied into build context by CI) +COPY custompios/scripts/ /test/scripts/ +COPY custompios/tests/ /test/tests/ + +# Distro-specific tests and hooks +COPY tests/ /test/tests/ +COPY hooks/ /test/hooks/ + +RUN chmod +x /test/scripts/*.sh /test/tests/*.sh; \ + chmod +x /test/hooks/*.sh 2>/dev/null || true + +ENTRYPOINT ["/test/scripts/entrypoint.sh"] +``` + +### 2. Write Test Scripts + +Test scripts live in `testing/tests/` and follow this convention: + +```bash +#!/bin/bash +set -e + +HOST="${1:-localhost}" +PORT="${2:-2222}" +ARTIFACTS_DIR="${3:-}" +USER="pi" +PASS="raspberry" + +SSH_CMD="sshpass -p $PASS ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o PreferredAuthentications=password \ + -o PubkeyAuthentication=no \ + -o LogLevel=ERROR \ + -p $PORT ${USER}@${HOST}" + +# Your test logic here -- use $SSH_CMD to run commands on the guest +OUTPUT=$($SSH_CMD 'systemctl is-active myservice' 2>/dev/null) + +if [ "$OUTPUT" = "active" ]; then + echo " PASS: myservice is running" + exit 0 +else + echo " FAIL: myservice is not running (status: $OUTPUT)" + exit 1 +fi +``` + +**Conventions:** +- Script name must start with `test_` (e.g. `test_myservice.sh`) +- Arguments: `$1` = host, `$2` = SSH port, `$3` = artifacts directory (optional) +- Exit 0 for pass, non-zero for fail +- Use the `SSH_CMD` pattern shown above for guest commands + +### 3. Write a Prepare-Image Hook (optional) + +If your distro needs image patches beyond the generic ones (e.g. fixing configs for QEMU), create `testing/hooks/prepare-image.sh`: + +```bash +#!/bin/bash +set -e +IMAGE_FILE="${1:?Usage: $0 }" + +export LIBGUESTFS_BACKEND=direct + +# Example: patch a config file inside the image +guestfish -a "$IMAGE_FILE" <&1 | tail -80 + exit 1 + fi + echo "Tests finished with exit code: $(cat artifacts/exit-code)" + cat artifacts/test-results.txt 2>/dev/null || true + + - name: Collect logs + if: always() + run: | + docker logs e2e-test > artifacts/container.log 2>&1 || true + docker stop e2e-test 2>/dev/null || true + + - name: Check test result + run: exit "$(cat artifacts/exit-code 2>/dev/null || echo 1)" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-test-results + path: artifacts/ +``` + +## QEMU Screenshots + +The entrypoint automatically attempts a QEMU monitor screendump after tests complete. For distros with a GUI (e.g. FullPageOS), add a virtual GPU: + +```yaml + -e QEMU_EXTRA_ARGS="-device virtio-gpu-pci" +``` + +The screenshot is captured via: +``` +echo "screendump /tmp/screenshot.ppm" | socat - unix-connect:/tmp/qemu-monitor.sock +``` + +This is purely QEMU-internal -- no guest-side VNC or screenshot tools are needed. The resulting image is saved to `$ARTIFACTS_DIR/screenshot.png`. + +**Note:** The `-nographic` flag is always set for serial console output. The screendump captures the virtual GPU framebuffer, which is separate from the serial console. If no GPU device is added, the screendump will be empty or unavailable. + +## Local Testing + +### Run against a pre-built image + +```bash +# From your distro's repo root +cd testing + +# Copy the shared framework +mkdir -p custompios +cp -r /path/to/CustomPiOS/src/distro_testing/scripts custompios/scripts +cp -r /path/to/CustomPiOS/src/distro_testing/tests custompios/tests + +# Build the Docker image +DOCKER_BUILDKIT=0 docker build -t my-distro-e2e . + +# Run tests +mkdir -p artifacts +docker run --rm \ + -v "$PWD/artifacts:/output" \ + -v "/path/to/my-distro.img:/input/image.img:ro" \ + -e ARTIFACTS_DIR=/output \ + -e DISTRO_NAME="My Distro" \ + my-distro-e2e +``` + +### Debug a failing test + +Add `KEEP_ALIVE=true` to keep the container running after tests: + +```bash +docker run -d --name debug-test \ + -v "$PWD/artifacts:/output" \ + -v "/path/to/image.img:/input/image.img:ro" \ + -e ARTIFACTS_DIR=/output \ + -e KEEP_ALIVE=true \ + my-distro-e2e + +# Watch logs +docker logs -f debug-test + +# SSH into the running guest (from inside the container) +docker exec -it debug-test sshpass -p raspberry ssh \ + -o StrictHostKeyChecking=no -p 2222 pi@localhost + +# Check QEMU serial log +docker exec -it debug-test cat /tmp/qemu-serial.log +``` diff --git a/src/distro_testing/scripts/boot-qemu.sh b/src/distro_testing/scripts/boot-qemu.sh new file mode 100755 index 00000000..93140449 --- /dev/null +++ b/src/distro_testing/scripts/boot-qemu.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +IMAGE_FILE="${1:?Usage: $0 }" +KERNEL="${2:-/base/kernel.img}" +SSH_PORT="${QEMU_SSH_PORT:-2222}" +LOG_FILE="${3:-/tmp/qemu-serial.log}" +HTTP_PORT="${QEMU_HTTP_PORT:-8080}" +MONITOR_SOCK="${QEMU_MONITOR_SOCK:-/tmp/qemu-monitor.sock}" + +echo "=== Starting QEMU (aarch64, -M virt) ===" +echo " Image: $IMAGE_FILE" +echo " Kernel: $KERNEL" +echo " SSH: port $SSH_PORT -> guest:22" +echo " HTTP: port $HTTP_PORT -> guest:80" +echo " Monitor: $MONITOR_SOCK" + +HOSTFWD="hostfwd=tcp::${SSH_PORT}-:22,hostfwd=tcp::${HTTP_PORT}-:80" +if [ -n "$QEMU_EXTRA_PORTS" ]; then + HOSTFWD="${HOSTFWD},${QEMU_EXTRA_PORTS}" + echo " Extra: $QEMU_EXTRA_PORTS" +fi + +qemu-system-aarch64 \ + -machine virt \ + -cpu cortex-a72 \ + -m 2G \ + -smp 4 \ + -kernel "$KERNEL" \ + -append "rw console=ttyAMA0 root=/dev/vda2 rootfstype=ext4 rootdelay=1 loglevel=2 systemd.firstboot=off systemd.condition-first-boot=false" \ + -drive "file=$IMAGE_FILE,format=qcow2,id=hd0,if=none,cache=writeback" \ + -device virtio-blk,drive=hd0,bootindex=0 \ + -netdev "user,id=mynet,${HOSTFWD}" \ + -device virtio-net-pci,netdev=mynet \ + -monitor "unix:${MONITOR_SOCK},server,nowait" \ + -nographic \ + -no-reboot \ + ${QEMU_EXTRA_ARGS} \ + 2>&1 | tee "$LOG_FILE" diff --git a/src/distro_testing/scripts/entrypoint.sh b/src/distro_testing/scripts/entrypoint.sh new file mode 100755 index 00000000..2ce049cc --- /dev/null +++ b/src/distro_testing/scripts/entrypoint.sh @@ -0,0 +1,155 @@ +#!/bin/bash +set -e + +INPUT_IMAGE="/input/image.img" +WORK_DIR="/work" +IMAGE_FILE="${WORK_DIR}/distro.qcow2" +KERNEL="/base/kernel.img" +SSH_PORT="${QEMU_SSH_PORT:-2222}" +SSH_TIMEOUT="${SSH_TIMEOUT:-600}" +LOG_FILE="/tmp/qemu-serial.log" +DISTRO_NAME="${DISTRO_NAME:-CustomPiOS Distro}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TEST_DIR="$(dirname "$SCRIPT_DIR")/tests" + +echo "============================================" +echo " ${DISTRO_NAME} E2E Test" +echo "============================================" + +if [ ! -f "$INPUT_IMAGE" ]; then + echo "ERROR: No image found at $INPUT_IMAGE" + echo "Mount an .img file with: -v /path/to/image.img:/input/image.img:ro" + exit 1 +fi + +if [ ! -f "$KERNEL" ]; then + echo "ERROR: No kernel found at $KERNEL" + exit 1 +fi + +cleanup() { + if [ -n "$QEMU_PID" ] && kill -0 "$QEMU_PID" 2>/dev/null; then + echo "Stopping QEMU (pid $QEMU_PID)..." + kill "$QEMU_PID" 2>/dev/null || true + wait "$QEMU_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "" +echo "--- Step 1: Prepare image ---" +"$SCRIPT_DIR/prepare-image.sh" "$INPUT_IMAGE" "$IMAGE_FILE" + +echo "" +echo "--- Step 2: Boot QEMU ---" +"$SCRIPT_DIR/boot-qemu.sh" "$IMAGE_FILE" "$KERNEL" "$LOG_FILE" & +QEMU_PID=$! +echo "QEMU started (pid $QEMU_PID)" + +echo "" +echo "--- Step 3: Wait for SSH ---" +set +e +"$SCRIPT_DIR/wait-for-ssh.sh" localhost "$SSH_PORT" "$SSH_TIMEOUT" +SSH_WAIT_RC=$? +set -e +if [ "$SSH_WAIT_RC" -ne 0 ]; then + echo "SSH wait failed. QEMU log tail:" + tail -50 "$LOG_FILE" 2>/dev/null || true + if [ -n "$ARTIFACTS_DIR" ]; then + cp "$LOG_FILE" "$ARTIFACTS_DIR/qemu-boot.log" 2>/dev/null || true + echo "1" > "$ARTIFACTS_DIR/exit-code" + fi + exit 1 +fi + +# Run distro-specific post-boot setup (e.g. install packages, restart services) +if [ -x /test/hooks/post-boot.sh ]; then + echo "" + echo "--- Step 3b: Post-boot setup ---" + /test/hooks/post-boot.sh localhost "$SSH_PORT" || echo "WARNING: post-boot hook failed" +fi + +echo "" +echo "--- Step 4: Run tests ---" +TEST_RESULT=0 +for test_script in "$TEST_DIR"/test_*.sh; do + if [ -x "$test_script" ]; then + echo "Running $(basename "$test_script")..." + if [ -n "$ARTIFACTS_DIR" ]; then + if "$test_script" localhost "$SSH_PORT" "$ARTIFACTS_DIR"; then + echo " -> PASSED" + else + echo " -> FAILED" + TEST_RESULT=1 + fi + else + if "$test_script" localhost "$SSH_PORT"; then + echo " -> PASSED" + else + echo " -> FAILED" + TEST_RESULT=1 + fi + fi + fi +done + +echo "" +echo "--- Step 5: Capture screenshot ---" +SCREENSHOT_TAKEN=false + +# Method 1: QEMU monitor screendump (works when QEMU has a display device + driver) +MONITOR_SOCK="${QEMU_MONITOR_SOCK:-/tmp/qemu-monitor.sock}" +if [ -S "$MONITOR_SOCK" ]; then + echo "Trying QEMU screendump..." + echo "screendump /tmp/screenshot.ppm" | socat - "unix-connect:${MONITOR_SOCK}" 2>/dev/null || true + sleep 2 + if [ -f /tmp/screenshot.ppm ]; then + if command -v convert &>/dev/null; then + convert /tmp/screenshot.ppm /tmp/screenshot.png 2>/dev/null && \ + echo "QEMU screenshot saved as PNG" || \ + echo "PNG conversion failed, keeping PPM" + fi + if [ -n "$ARTIFACTS_DIR" ]; then + cp /tmp/screenshot.png "$ARTIFACTS_DIR/qemu-screenshot.png" 2>/dev/null || \ + cp /tmp/screenshot.ppm "$ARTIFACTS_DIR/qemu-screenshot.ppm" 2>/dev/null || true + fi + SCREENSHOT_TAKEN=true + fi +fi + +# Method 2: Distro-specific screenshot hook (e.g. headless chromium, import, etc.) +if [ -x /test/hooks/screenshot.sh ]; then + echo "Running distro-specific screenshot hook..." + /test/hooks/screenshot.sh localhost "$SSH_PORT" "$ARTIFACTS_DIR" && \ + SCREENSHOT_TAKEN=true || \ + echo "Distro screenshot hook failed" +fi + +if [ "$SCREENSHOT_TAKEN" = false ]; then + echo "No screenshot captured" +fi + +echo "" +echo "============================================" +if [ "$TEST_RESULT" -eq 0 ]; then + echo " ALL TESTS PASSED" +else + echo " SOME TESTS FAILED" +fi +echo "============================================" + +if [ -n "$ARTIFACTS_DIR" ]; then + echo "Collecting artifacts to $ARTIFACTS_DIR..." + cp "$LOG_FILE" "$ARTIFACTS_DIR/qemu-boot.log" 2>/dev/null || true + echo "$TEST_RESULT" > "$ARTIFACTS_DIR/exit-code" + echo "TEST_RESULT=$TEST_RESULT" > "$ARTIFACTS_DIR/test-results.txt" +fi + +if [ -n "$KEEP_ALIVE" ]; then + echo "Keeping container alive (KEEP_ALIVE set)..." + trap - EXIT + sleep infinity +else + exit "$TEST_RESULT" +fi diff --git a/src/distro_testing/scripts/prepare-image.sh b/src/distro_testing/scripts/prepare-image.sh new file mode 100755 index 00000000..d78de60c --- /dev/null +++ b/src/distro_testing/scripts/prepare-image.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -e +INPUT_IMAGE="${1:?Usage: $0 }" +OUTPUT_IMAGE="${2:?Usage: $0 }" +PIPASS=$(openssl passwd -6 raspberry) + +echo '=== Preparing image ===' +mkdir -p /work +echo 'Converting to qcow2...' +qemu-img convert -f raw -O qcow2 "$INPUT_IMAGE" "$OUTPUT_IMAGE" +echo 'Patching image (rootfs)...' +export LIBGUESTFS_BACKEND=direct +export LIBGUESTFS_DEBUG=0 +export LIBGUESTFS_TRACE=0 +guestfish -a "$OUTPUT_IMAGE" < /dev/tcp/"$HOST"/"$PORT") 2>/dev/null; then + RESULT=$(sshpass -p "$PASS" ssh $SSH_OPTS -p "$PORT" "${USER}@${HOST}" true 2>&1) + RC=$? + if [ "$RC" -eq 0 ]; then + echo "" + echo "SSH is ready (took ${ELAPSED}s)" + exit 0 + fi + if [ $(( ATTEMPT % 6 )) -eq 0 ]; then + echo "" + echo "[${ELAPSED}s] Port open, sshpass rc=$RC output: $RESULT" + echo "[${ELAPSED}s] Trying verbose SSH..." + sshpass -p "$PASS" ssh -v $SSH_OPTS -p "$PORT" "${USER}@${HOST}" true 2>&1 | tail -20 + else + printf "x" + fi + else + printf "." + fi + sleep 5 +done diff --git a/src/distro_testing/tests/test_boot.sh b/src/distro_testing/tests/test_boot.sh new file mode 100755 index 00000000..ad4e624f --- /dev/null +++ b/src/distro_testing/tests/test_boot.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +HOST="${1:-localhost}" +PORT="${2:-2222}" +USER="pi" +PASS="raspberry" + +SSH_CMD="sshpass -p $PASS ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=password -o PubkeyAuthentication=no -o LogLevel=ERROR -p $PORT ${USER}@${HOST}" + +echo "Test: SSH login and run 'echo hello world'" + +OUTPUT=$($SSH_CMD 'echo hello world' 2>/dev/null) + +if [ "$OUTPUT" = "hello world" ]; then + echo " Output: '$OUTPUT'" + echo " PASS: Got expected output" + exit 0 +else + echo " Expected: 'hello world'" + echo " Got: '$OUTPUT'" + echo " FAIL: Unexpected output" + exit 1 +fi diff --git a/src/qemu_boot.sh b/src/qemu_boot.sh index 6580cfd1..5a5f3a22 100755 --- a/src/qemu_boot.sh +++ b/src/qemu_boot.sh @@ -35,7 +35,7 @@ if [ ! -f "${BASE_IMG_PATH}" ]; then sudo bash -c "$(declare -f unmount_image); unmount_image $BASE_MOUNT_PATH force" fi -KERNEL_VERSION=kernel-qemu-4.19.50-buster +KERNEL_VERSION=kernel-qemu-5.10.63-bullseye DTB_VERSION=versatile-pb.dtb KERNEL_PATH=${DEST}/${KERNEL_VERSION} @@ -51,7 +51,7 @@ fi -/usr/bin/qemu-system-arm -kernel ${KERNEL_PATH} -cpu arm1176 -m 256 -M versatilepb -dtb ${DTB_PATH} -no-reboot -serial stdio -append 'root=/dev/sda2 panic=1 rootfstype=ext4 rw' -hda ${BASE_IMG_PATH} -net nic -net user,hostfwd=tcp::5022-:22 +/usr/bin/qemu-system-arm -kernel ${KERNEL_PATH} -cpu arm1176 -m 256 -M versatilepb -dtb ${DTB_PATH} -no-reboot -serial stdio -append 'root=/dev/sda2 panic=1 rootfstype=ext4 rw' -hda ${BASE_IMG_PATH} -net nic -net user,hostfwd=tcp::5022-:22,hostfwd=tcp::8080-:80,hostfwd=tcp::8443-:443,hostfwd=tcp::5000-:5000 #sudo umount ${BASE_MOUNT_PATH} diff --git a/src/qemu_boot64.sh b/src/qemu_boot64.sh index 713ef250..07422e96 100755 --- a/src/qemu_boot64.sh +++ b/src/qemu_boot64.sh @@ -36,7 +36,7 @@ if [ ! -f "${BASE_IMG_PATH}" ]; then rm -rf /tmp/debian_bootpart || true mkdir /tmp/debian_bootpart cp kernel8.img /tmp/debian_bootpart - DTB_PATH="$(ls *-3-b.dtb | head)" + DTB_PATH="$(ls *.dtb | head)" # sudo bash -c 'cp initrd.img*-arm64 /tmp/debian_bootpart' cp ${DTB_PATH} /tmp/debian_bootpart # sudo bash -c 'cp cmdline.txt /tmp/debian_bootpart' @@ -55,12 +55,12 @@ fi #/usr/bin/qemu-system-arm -kernel ${KERNEL_PATH} -cpu arm1176 -m 256 -M versatilepb -dtb ${DTB_PATH} -no-reboot -serial stdio -append 'root=/dev/sda2 panic=1 rootfstype=ext4 rw' -hda ${BASE_IMG_PATH} -net nic -net user,hostfwd=tcp::5022-:22 -DTB_PATH=$(ls /tmp/debian_bootpart/*-3-b.dtb | head) +DTB_PATH=$(ls /tmp/debian_bootpart/*-4-b.dtb | head) qemu-system-aarch64 \ -kernel /tmp/debian_bootpart/kernel8.img \ -dtb "${DTB_PATH}" \ --m 1024 -M raspi3 \ +-m 2G -M raspi4b \ -cpu cortex-a53 \ -serial stdio \ -append "rw earlycon=pl011,0x3f201000 console=ttyAMA0 loglevel=8 root=/dev/mmcblk0p2 fsck.repair=yes net.ifnames=0 rootwait memtest=1" \