diff --git a/.github/workflows/build-local-slurm-image.yml b/.github/workflows/build-local-slurm-image.yml new file mode 100644 index 0000000..b9bcb13 --- /dev/null +++ b/.github/workflows/build-local-slurm-image.yml @@ -0,0 +1,39 @@ +name: Build local-slurm image + +on: + push: + branches: + - main + paths: + - 'tests/integration/stacks/_images/slurm/**' + - '.github/workflows/build-local-slurm-image.yml' + schedule: + # Weekly rebuild so base-image security updates flow through even when + # the Dockerfile itself hasn't changed. + - cron: '0 4 * * 1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v5 + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Build image + run: | + podman build \ + -t ghcr.io/diracgrid/intercede-testenv/slurm:latest \ + tests/integration/stacks/_images/slurm + + - name: Push image + run: podman push ghcr.io/diracgrid/intercede-testenv/slurm:latest diff --git a/.github/workflows/slurm-integration.yml b/.github/workflows/slurm-integration.yml new file mode 100644 index 0000000..c3af9be --- /dev/null +++ b/.github/workflows/slurm-integration.yml @@ -0,0 +1,106 @@ +name: Slurm Integration Tests + +on: + push: + branches: + - main + paths: + - 'tests/integration/stacks/local-slurm/**' + - 'tests/integration/stacks/_images/slurm/**' + - 'src/**' + pull_request: + branches: + - main + paths: + - 'tests/integration/stacks/local-slurm/**' + - 'tests/integration/stacks/_images/slurm/**' + - 'src/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: read + +jobs: + slurm-integration: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install podman-compose + run: pip install podman-compose + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Pull slurm image + # Pulls the exact digest pinned in compose.yml (Renovate-managed). This is a separate, + # explicit step so a missing/retired digest fails loudly here instead of `up` silently + # falling back to the `build:` directive. + working-directory: tests/integration/stacks/local-slurm + run: podman-compose pull + + - name: Start cluster + working-directory: tests/integration/stacks/local-slurm + run: podman-compose up -d + + - name: Wait for containers to be healthy + working-directory: tests/integration/stacks/local-slurm + run: | + # podman-compose's own `up --wait` never returns even once both + # containers report healthy, so poll container health directly. + for i in $(seq 1 20); do + ctld=$(podman inspect --format='{{.State.Health.Status}}' intercede-slurmctld) + c1=$(podman inspect --format='{{.State.Health.Status}}' intercede-c1) + [ "$ctld" = "healthy" ] && [ "$c1" = "healthy" ] && break + echo "Waiting for containers to be healthy... (slurmctld=$ctld, c1=$c1, attempt $i/20)" + sleep 3 + done + podman-compose ps + + - name: Show cluster state + working-directory: tests/integration/stacks/local-slurm + run: podman-compose exec intercede-slurmctld sinfo + + - name: Submit test jobs + working-directory: tests/integration/stacks/local-slurm + run: | + podman-compose exec intercede-slurmctld sbatch /test-jobs/simple.sh + podman-compose exec intercede-slurmctld sbatch --array=1-4 /test-jobs/array.sh + podman-compose exec intercede-slurmctld sbatch /test-jobs/collatz.sh + + - name: Wait for all jobs to complete + working-directory: tests/integration/stacks/local-slurm + run: | + for i in $(seq 1 30); do + PENDING=$(podman-compose exec intercede-slurmctld squeue --noheader 2>/dev/null | wc -l) + if [ "$PENDING" -eq 0 ]; then + echo "All jobs completed" + break + fi + echo "Waiting... $PENDING job(s) still in queue (attempt $i/30)" + podman-compose exec intercede-slurmctld squeue + sleep 5 + done + + - name: Show job output + working-directory: tests/integration/stacks/local-slurm + run: podman-compose exec intercede-c1 bash -c 'cat /tmp/slurm-*.out' + + - name: Show logs on failure + if: failure() + working-directory: tests/integration/stacks/local-slurm + run: | + echo "=== slurmctld log ===" + podman-compose exec intercede-slurmctld cat /var/log/slurm/slurmctld.log || true + echo "=== slurmd log ===" + podman-compose exec intercede-c1 cat /var/log/slurm/slurmd.log || true + + - name: Tear down + if: always() + working-directory: tests/integration/stacks/local-slurm + run: podman-compose down -v diff --git a/renovate.json b/renovate.json index 437d827..ba3e02a 100644 --- a/renovate.json +++ b/renovate.json @@ -7,7 +7,23 @@ "enabledManagers": [ "pip_requirements", "pep621", - "github-actions" + "github-actions", + "dockerfile", + "docker-compose", + "custom.regex" + ], + "docker-compose": { + "pinDigests": true + }, + "customManagers": [ + { + "customType": "regex", + "description": "Pin backend package versions in integration-stack Dockerfiles", + "managerFilePatterns": ["/tests/integration/stacks/_images/.+/Dockerfile$/"], + "matchStrings": [ + "# renovate: datasource=(?.*?) depName=(?.*?)\\s+ARG .*_VERSION=(?.*)\\s" + ] + } ], "python": { "rangeStrategy": "auto" @@ -63,6 +79,12 @@ "matchPackagePatterns": ["pytest.*", "ty"], "groupName": "Testing dependencies", "labels": ["dependencies", "testing"] + }, + { + "description": "Group integration-stack image/version updates together", + "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], + "groupName": "Integration test stack images", + "labels": ["dependencies", "integration-tests"] } ] } diff --git a/tests/integration/stacks/_images/slurm/Dockerfile b/tests/integration/stacks/_images/slurm/Dockerfile new file mode 100644 index 0000000..bd58d75 --- /dev/null +++ b/tests/integration/stacks/_images/slurm/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:26.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# renovate: datasource=repology depName=ubuntu_26_04/slurm-wlm +ARG SLURM_VERSION=25.11.2-1ubuntu2 + +RUN apt-get update && \ + apt-get install -y \ + slurmctld=${SLURM_VERSION} \ + slurmd=${SLURM_VERSION} \ + slurm-client=${SLURM_VERSION} \ + munge && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/spool/slurmctld /var/spool/slurmd /var/log/slurm && \ + chown slurm:slurm /var/spool/slurmctld /var/spool/slurmd /var/log/slurm + +# The munge package's postinst auto-generates /etc/munge/munge.key on install; +# remove it so the image ships with none. entrypoint.sh generates the real key +# fresh at container start into a shared volume - a key baked into a pushed +# image would be a committed shared secret usable to impersonate any node. +RUN rm -f /etc/munge/munge.key + +COPY entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/integration/stacks/_images/slurm/entrypoint.sh b/tests/integration/stacks/_images/slurm/entrypoint.sh new file mode 100644 index 0000000..a81b2d0 --- /dev/null +++ b/tests/integration/stacks/_images/slurm/entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -euo pipefail + +ROLE="${1:-controller}" +MUNGE_KEY=/etc/munge/munge.key + +wait_for() { + local desc="$1"; shift + for _ in $(seq 1 30); do + "$@" && return 0 + sleep 0.5 + done + echo "==> timed out waiting for ${desc}" >&2 + return 1 +} + +start_munge() { + mkdir -p /var/run/munge + + if [ "$ROLE" = "controller" ]; then + # Generated fresh per run into the shared munge-key volume - never baked + # into the image. + if [ ! -s "$MUNGE_KEY" ]; then + dd if=/dev/urandom bs=1 count=1024 2>/dev/null > "$MUNGE_KEY" + fi + else + wait_for "munge key from controller" test -s "$MUNGE_KEY" + fi + chown munge:munge "$MUNGE_KEY" + chmod 400 "$MUNGE_KEY" + + chown munge:munge /var/run/munge + runuser -u munge -- munged + wait_for "munged socket" test -S /var/run/munge/munge.socket.2 +} + +case "$ROLE" in + controller) + start_munge + slurmctld + exec tail -f --retry /var/log/slurm/slurmctld.log + ;; + + worker) + start_munge + slurmd + exec tail -f --retry /var/log/slurm/slurmd.log + ;; + + *) + exec "$@" + ;; +esac diff --git a/tests/integration/stacks/local-slurm/README.md b/tests/integration/stacks/local-slurm/README.md new file mode 100644 index 0000000..793e63a --- /dev/null +++ b/tests/integration/stacks/local-slurm/README.md @@ -0,0 +1,167 @@ +# local-slurm stack + +A containerised single-node Slurm cluster for testing job submission locally and in GitHub CI. +This is the `local-slurm` stack described by IC-ADR-002 (integration testing against +containerized backends, currently proposed in PR #4) — it shares its image with the not-yet-built +`ssh-slurm` stack; the only difference is that tests here exec into the container instead of +connecting over SSH. + +## Architecture + +Two containers, both running the shared `slurm` image built from +[`../_images/slurm/Dockerfile`](../_images/slurm/Dockerfile), on a shared network: + +| Container | Hostname | Role | +|---|---|---| +| `intercede-slurmctld` | `intercede-slurmctld` | Controller (`slurmctld`) — submit jobs here | +| `intercede-c1` | `intercede-c1` | Compute node (`slurmd`) | + +The munge authentication key is **not** baked into the image. The controller generates it on +first start into a shared `munge-key` volume; the compute node waits for it to appear. Tearing +the stack down with `podman-compose down -v` removes the volume, so the next `up` starts with a +fresh key. + +Both services declare a `healthcheck` (the daemon process is running) and the compute node's +`depends_on` gates on the controller's healthcheck via `condition: service_healthy` — so +`podman-compose up -d` doesn't start the compute node until the controller is actually up, no +hand-rolled sleeps. (`podman-compose`'s own `up --wait` never returns even once both containers +are healthy — a bug in this version — so wait for readiness with +`podman inspect --format='{{.State.Health.Status}}'` instead, as the CI workflow does.) +`sinfo`/`squeue` polling is still needed after that to observe node registration and job +completion — that's cross-container state convergence, not container startup, and isn't something +a single container's healthcheck can express. + +## Prerequisites + +- **Fedora/RHEL:** `dnf install podman podman-compose` +- **Ubuntu 26.04 / 24.04:** `apt install podman podman-compose` +- **Ubuntu 22.04:** `apt install podman && pip install podman-compose` (the apt package is too old) + +## Local usage + +**Start the cluster** (pulls the prebuilt image from GHCR): + +```bash +podman-compose up -d +``` + +**...or build the image locally instead of pulling, e.g. while iterating on the Dockerfile:** + +```bash +podman-compose up --build -d +``` + +**Check the cluster is up:** + +```bash +podman-compose exec intercede-slurmctld sinfo +``` + +You should see `intercede-c1` with state `idle`. + +**Submit a job:** + +```bash +podman-compose exec intercede-slurmctld sbatch /test-jobs/simple.sh +``` + +**Watch the queue:** + +```bash +podman-compose exec intercede-slurmctld squeue +``` + +**Submit a job array (4 tasks):** + +```bash +podman-compose exec intercede-slurmctld sbatch --array=1-4 /test-jobs/array.sh +``` + +**Read job output** (written to `/tmp/` inside the compute node): + +```bash +podman-compose exec intercede-c1 bash -c 'cat /tmp/slurm-*.out' +``` + +**Tail the Slurm logs:** + +```bash +# Controller log +podman-compose exec intercede-slurmctld tail -f /var/log/slurm/slurmctld.log + +# Compute node log +podman-compose exec intercede-c1 tail -f /var/log/slurm/slurmd.log +``` + +**Tear down** (`-v` also drops the munge-key volume): + +```bash +podman-compose down -v +``` + +## Running your own job script + +Write a batch script with `#SBATCH` directives and submit it via the controller: + +```bash +podman cp my-job.sh intercede-slurmctld:/tmp/my-job.sh +podman-compose exec intercede-slurmctld sbatch /tmp/my-job.sh +``` + +## GitHub CI + +The workflow in `.github/workflows/slurm-integration.yml` runs automatically on push and pull +request when files under `tests/integration/stacks/local-slurm/`, +`tests/integration/stacks/_images/slurm/`, or `src/` change. It: + +1. Pulls the image `compose.yml` pins (`podman-compose pull`) — a digest-exact reference + (`ghcr.io/diracgrid/intercede-testenv/slurm:latest@sha256:…`), not a floating tag, so every run + uses byte-identical content until that pin is deliberately updated. A missing/retired digest + fails this step loudly rather than silently falling back to a rebuild. +2. Starts the cluster with `podman-compose up -d` against the already-pulled image (no rebuild) + and polls both containers' health status until healthy +3. Submits the simple, array, and Collatz test jobs +4. Waits for all jobs to leave the queue +5. Prints job output +6. On failure, dumps the `slurmctld` and `slurmd` logs + +The image itself is built and pushed to `ghcr.io/diracgrid/intercede-testenv/slurm:latest` by a +separate workflow (`.github/workflows/build-local-slurm-image.yml`), triggered when the Dockerfile +changes and on a weekly schedule for base-image security updates. **Renovate owns the digest pin** +in `compose.yml`: it's configured (`renovate.json`, `docker-compose` manager, `pinDigests: true`) +to open a PR whenever the digest that `:latest` resolves to changes, so a version bump becomes a +reviewable, bisectable PR instead of a silent, unreviewed content change (mirroring how ADR-002 +wants backend versions tracked). + +> **Bootstrap note:** the image hasn't been published to GHCR yet, so `compose.yml` currently pins +> a placeholder all-zero digest. Once `build-local-slurm-image.yml` runs and publishes the real +> image, Renovate's next scan will open a PR replacing the placeholder with the actual digest — +> CI here will fail to pull until that first pin PR merges. + +This workflow is intentionally **not** wired into `ci.yml` / required for merge yet: there's no +real interCEde-vs-Slurm contract suite (IC-ADR-002 §2) to gate on, since `src/intercede` doesn't +have backend code yet. The three shell test-jobs here are a stand-in smoke test until that lands. + +## Project structure + +``` +tests/integration/stacks/ +├── _images/slurm/ +│ ├── Dockerfile # Ubuntu 26.04 + pinned Slurm + Munge (shared image) +│ └── entrypoint.sh # Starts the right daemon based on role arg +└── local-slurm/ + ├── compose.yml # image: (GHCR) + build: (local) for either workflow + ├── config/basic/ + │ ├── slurm.conf # Slurm configuration, bind-mounted at runtime + │ └── cgroup.conf # Disables systemd scope creation (container-safe) + └── test-jobs/ + ├── simple.sh # Single-task job + ├── array.sh # 4-task array job + └── collatz.sh # Collatz sequence (starting number = job ID % 1000) +``` + +## Configuration notes + +- `ProctrackType=proctrack/linuxproc` — avoids cgroup kernel requirements inside containers +- `TaskPlugin=task/none` — likewise avoids cgroup task management +- `ReturnToService=2` — nodes automatically return to service after being down, useful when the compute container starts slightly after the controller diff --git a/tests/integration/stacks/local-slurm/compose.yml b/tests/integration/stacks/local-slurm/compose.yml new file mode 100644 index 0000000..0d4e212 --- /dev/null +++ b/tests/integration/stacks/local-slurm/compose.yml @@ -0,0 +1,52 @@ +services: + intercede-slurmctld: + # Placeholder digest — the image hasn't been published to GHCR yet. Renovate will replace + # this with the real digest once build-local-slurm-image.yml pushes it, then keep it current. + image: ghcr.io/diracgrid/intercede-testenv/slurm:latest@sha256:0000000000000000000000000000000000000000000000000000000000000000 + build: ../_images/slurm + container_name: intercede-slurmctld + hostname: intercede-slurmctld + command: ["controller"] + privileged: true + volumes: + - ./config/basic/slurm.conf:/etc/slurm/slurm.conf:ro + - ./config/basic/cgroup.conf:/etc/slurm/cgroup.conf:ro + - ./test-jobs:/test-jobs:ro + - munge-key:/etc/munge + healthcheck: + test: ["CMD", "pgrep", "slurmctld"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: + - slurm + + intercede-c1: + image: ghcr.io/diracgrid/intercede-testenv/slurm:latest@sha256:0000000000000000000000000000000000000000000000000000000000000000 + build: ../_images/slurm + container_name: intercede-c1 + hostname: intercede-c1 + command: ["worker"] + privileged: true + volumes: + - ./config/basic/slurm.conf:/etc/slurm/slurm.conf:ro + - ./config/basic/cgroup.conf:/etc/slurm/cgroup.conf:ro + - munge-key:/etc/munge + healthcheck: + test: ["CMD", "pgrep", "slurmd"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + depends_on: + intercede-slurmctld: + condition: service_healthy + networks: + - slurm + +networks: + slurm: + +volumes: + munge-key: diff --git a/tests/integration/stacks/local-slurm/config/basic/cgroup.conf b/tests/integration/stacks/local-slurm/config/basic/cgroup.conf new file mode 100644 index 0000000..efc385f --- /dev/null +++ b/tests/integration/stacks/local-slurm/config/basic/cgroup.conf @@ -0,0 +1,3 @@ +# Tell slurmstepd not to attempt systemd scope creation via dbus. +# Required when running inside a container without systemd. +IgnoreSystemd=yes diff --git a/tests/integration/stacks/local-slurm/config/basic/slurm.conf b/tests/integration/stacks/local-slurm/config/basic/slurm.conf new file mode 100644 index 0000000..76ddb40 --- /dev/null +++ b/tests/integration/stacks/local-slurm/config/basic/slurm.conf @@ -0,0 +1,31 @@ +ClusterName=intercede-test +SlurmctldHost=intercede-slurmctld + +MpiDefault=none +# linuxproc avoids cgroup kernel requirements — required inside containers +ProctrackType=proctrack/linuxproc +TaskPlugin=task/none +SwitchType=switch/none + +ReturnToService=2 +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmdPidFile=/var/run/slurmd.pid +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation=/var/spool/slurmctld + +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core + +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +JobAcctGatherType=jobacct_gather/none + +SlurmctldDebug=info +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=info +SlurmdLogFile=/var/log/slurm/slurmd.log + +NodeName=intercede-c1 State=UNKNOWN +PartitionName=test Nodes=ALL Default=YES MaxTime=INFINITE State=UP diff --git a/tests/integration/stacks/local-slurm/test-jobs/array.sh b/tests/integration/stacks/local-slurm/test-jobs/array.sh new file mode 100644 index 0000000..b6f914e --- /dev/null +++ b/tests/integration/stacks/local-slurm/test-jobs/array.sh @@ -0,0 +1,10 @@ +#!/bin/bash +#SBATCH --job-name=array-test +#SBATCH --array=1-4 +#SBATCH --ntasks=1 +#SBATCH --time=00:01:00 +#SBATCH --output=/tmp/slurm-%A_%a.out + +echo "Array job $SLURM_ARRAY_JOB_ID task $SLURM_ARRAY_TASK_ID on $(hostname)" +sleep 2 +echo "Task $SLURM_ARRAY_TASK_ID done" diff --git a/tests/integration/stacks/local-slurm/test-jobs/collatz.sh b/tests/integration/stacks/local-slurm/test-jobs/collatz.sh new file mode 100644 index 0000000..b4e07b7 --- /dev/null +++ b/tests/integration/stacks/local-slurm/test-jobs/collatz.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#SBATCH --job-name=collatz +#SBATCH --ntasks=1 +#SBATCH --time=00:01:00 +#SBATCH --output=/tmp/slurm-%j.out + +start=$(( SLURM_JOB_ID % 1000 )) +# Avoid starting at 0 +[ "$start" -eq 0 ] && start=1000 + +echo "Job $SLURM_JOB_ID: Collatz sequence from $start" + +n=$start +steps=0 +sequence="$n" + +while [ "$n" -ne 1 ]; do + if [ $(( n % 2 )) -eq 0 ]; then + n=$(( n / 2 )) + else + n=$(( n * 3 + 1 )) + fi + steps=$(( steps + 1 )) + sequence="$sequence → $n" +done + +echo "Sequence: $sequence" +echo "Steps: $steps" diff --git a/tests/integration/stacks/local-slurm/test-jobs/simple.sh b/tests/integration/stacks/local-slurm/test-jobs/simple.sh new file mode 100644 index 0000000..e861af9 --- /dev/null +++ b/tests/integration/stacks/local-slurm/test-jobs/simple.sh @@ -0,0 +1,10 @@ +#!/bin/bash +#SBATCH --job-name=hello +#SBATCH --ntasks=1 +#SBATCH --time=00:01:00 +#SBATCH --output=/tmp/slurm-%j.out + +echo "Hello from Slurm job $SLURM_JOB_ID on $(hostname)" +date +sleep 3 +echo "Done"