ci: add standalone Slurm test cluster

.github/workflows/build-local-slurm-image.yml

@@ -0,0 +1,39 @@
+name: Build local-slurm image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'tests/integration/stacks/_images/slurm/**'
+      - '.github/workflows/build-local-slurm-image.yml'
+  schedule:
+    # Weekly rebuild so base-image security updates flow through even when
+    # the Dockerfile itself hasn't changed.
+    - cron: '0 4 * * 1'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Log in to GHCR
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Build image
+        run: |
+          podman build \
+            -t ghcr.io/diracgrid/intercede-testenv/slurm:latest \
+            tests/integration/stacks/_images/slurm
+
+      - name: Push image
+        run: podman push ghcr.io/diracgrid/intercede-testenv/slurm:latest

.github/workflows/slurm-integration.yml

@@ -0,0 +1,106 @@
+name: Slurm Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'tests/integration/stacks/local-slurm/**'
+      - 'tests/integration/stacks/_images/slurm/**'
+      - 'src/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'tests/integration/stacks/local-slurm/**'
+      - 'tests/integration/stacks/_images/slurm/**'
+      - 'src/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  slurm-integration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install podman-compose
+        run: pip install podman-compose
+
+      - name: Log in to GHCR
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Pull slurm image
+        # Pulls the exact digest pinned in compose.yml (Renovate-managed). This is a separate,
+        # explicit step so a missing/retired digest fails loudly here instead of `up` silently
+        # falling back to the `build:` directive.
+        working-directory: tests/integration/stacks/local-slurm
+        run: podman-compose pull
+
+      - name: Start cluster
+        working-directory: tests/integration/stacks/local-slurm
+        run: podman-compose up -d
+
+      - name: Wait for containers to be healthy
+        working-directory: tests/integration/stacks/local-slurm
+        run: |
+          # podman-compose's own `up --wait` never returns even once both
+          # containers report healthy, so poll container health directly.
+          for i in $(seq 1 20); do
+            ctld=$(podman inspect --format='{{.State.Health.Status}}' intercede-slurmctld)
+            c1=$(podman inspect --format='{{.State.Health.Status}}' intercede-c1)
+            [ "$ctld" = "healthy" ] && [ "$c1" = "healthy" ] && break
+            echo "Waiting for containers to be healthy... (slurmctld=$ctld, c1=$c1, attempt $i/20)"
+            sleep 3
+          done
+          podman-compose ps
+
+      - name: Show cluster state
+        working-directory: tests/integration/stacks/local-slurm
+        run: podman-compose exec intercede-slurmctld sinfo
+
+      - name: Submit test jobs
+        working-directory: tests/integration/stacks/local-slurm
+        run: |
+          podman-compose exec intercede-slurmctld sbatch /test-jobs/simple.sh
+          podman-compose exec intercede-slurmctld sbatch --array=1-4 /test-jobs/array.sh
+          podman-compose exec intercede-slurmctld sbatch /test-jobs/collatz.sh
+
+      - name: Wait for all jobs to complete
+        working-directory: tests/integration/stacks/local-slurm
+        run: |
+          for i in $(seq 1 30); do
+            PENDING=$(podman-compose exec intercede-slurmctld squeue --noheader 2>/dev/null | wc -l)
+            if [ "$PENDING" -eq 0 ]; then
+              echo "All jobs completed"
+              break
+            fi
+            echo "Waiting... $PENDING job(s) still in queue (attempt $i/30)"
+            podman-compose exec intercede-slurmctld squeue
+            sleep 5
+          done
+
+      - name: Show job output
+        working-directory: tests/integration/stacks/local-slurm
+        run: podman-compose exec intercede-c1 bash -c 'cat /tmp/slurm-*.out'
+
+      - name: Show logs on failure
+        if: failure()
+        working-directory: tests/integration/stacks/local-slurm
+        run: |
+          echo "=== slurmctld log ==="
+          podman-compose exec intercede-slurmctld cat /var/log/slurm/slurmctld.log || true
+          echo "=== slurmd log ==="
+          podman-compose exec intercede-c1 cat /var/log/slurm/slurmd.log || true
+
+      - name: Tear down
+        if: always()
+        working-directory: tests/integration/stacks/local-slurm
+        run: podman-compose down -v

renovate.json

@@ -7,7 +7,23 @@
   "enabledManagers": [
     "pip_requirements",
     "pep621",
-    "github-actions"
+    "github-actions",
+    "dockerfile",
+    "docker-compose",
+    "custom.regex"
+  ],
+  "docker-compose": {
+    "pinDigests": true
+  },
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "Pin backend package versions in integration-stack Dockerfiles",
+      "managerFilePatterns": ["/tests/integration/stacks/_images/.+/Dockerfile$/"],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>.*?) depName=(?<depName>.*?)\\s+ARG .*_VERSION=(?<currentValue>.*)\\s"
+      ]
+    }
   ],
   "python": {
     "rangeStrategy": "auto"
@@ -63,6 +79,12 @@
       "matchPackagePatterns": ["pytest.*", "ty"],
       "groupName": "Testing dependencies",
       "labels": ["dependencies", "testing"]
+    },
+    {
+      "description": "Group integration-stack image/version updates together",
+      "matchManagers": ["dockerfile", "docker-compose", "custom.regex"],
+      "groupName": "Integration test stack images",
+      "labels": ["dependencies", "integration-tests"]
     }
   ]
 }

tests/integration/stacks/_images/slurm/Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:26.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# renovate: datasource=repology depName=ubuntu_26_04/slurm-wlm
+ARG SLURM_VERSION=25.11.2-1ubuntu2
+
+RUN apt-get update && \
+    apt-get install -y \
+        slurmctld=${SLURM_VERSION} \
+        slurmd=${SLURM_VERSION} \
+        slurm-client=${SLURM_VERSION} \
+        munge && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/spool/slurmctld /var/spool/slurmd /var/log/slurm && \
+    chown slurm:slurm /var/spool/slurmctld /var/spool/slurmd /var/log/slurm
+
+# The munge package's postinst auto-generates /etc/munge/munge.key on install;
+# remove it so the image ships with none. entrypoint.sh generates the real key
+# fresh at container start into a shared volume - a key baked into a pushed
+# image would be a committed shared secret usable to impersonate any node.
+RUN rm -f /etc/munge/munge.key
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

tests/integration/stacks/_images/slurm/entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -euo pipefail
+
+ROLE="${1:-controller}"
+MUNGE_KEY=/etc/munge/munge.key
+
+wait_for() {
+    local desc="$1"; shift
+    for _ in $(seq 1 30); do
+        "$@" && return 0
+        sleep 0.5
+    done
+    echo "==> timed out waiting for ${desc}" >&2
+    return 1
+}
+
+start_munge() {
+    mkdir -p /var/run/munge
+
+    if [ "$ROLE" = "controller" ]; then
+        # Generated fresh per run into the shared munge-key volume - never baked
+        # into the image.
+        if [ ! -s "$MUNGE_KEY" ]; then
+            dd if=/dev/urandom bs=1 count=1024 2>/dev/null > "$MUNGE_KEY"
+        fi
+    else
+        wait_for "munge key from controller" test -s "$MUNGE_KEY"
+    fi
+    chown munge:munge "$MUNGE_KEY"
+    chmod 400 "$MUNGE_KEY"
+
+    chown munge:munge /var/run/munge
+    runuser -u munge -- munged
+    wait_for "munged socket" test -S /var/run/munge/munge.socket.2
+}
+
+case "$ROLE" in
+    controller)
+        start_munge
+        slurmctld
+        exec tail -f --retry /var/log/slurm/slurmctld.log
+        ;;
+
+    worker)
+        start_munge
+        slurmd
+        exec tail -f --retry /var/log/slurm/slurmd.log
+        ;;
+
+    *)
+        exec "$@"
+        ;;
+esac