bootstrap xz extractor

.github/workflows/ci.yml

@@ -75,3 +75,20 @@ jobs:
       - uses: actions/checkout@v4
       - uses: denoland/setup-deno@v1
       - run: deno task typecheck
+
+  no-system-xz:
+    runs-on: ubuntu-latest
+    container: debian:bookworm-slim
+    steps:
+      - run: |
+          apt-get update
+          apt-get install --yes ca-certificates curl git unzip
+      - uses: actions/checkout@v4
+      - run: |
+          if command -v xz; then
+            echo "unexpected system xz" >&2
+            exit 1
+          fi
+          curl -fsSL https://deno.land/install.sh | sh
+          echo "$HOME/.deno/bin" >> "$GITHUB_PATH"
+      - run: PKGX_TEST_NO_SYSTEM_XZ=1 deno test --no-check --unstable-fs --unstable-ffi --allow-all src/plumbing/install.no-system-xz.test.ts

src/plumbing/install.no-system-xz.test.ts

@@ -0,0 +1,55 @@
+// deno-lint-ignore-file no-explicit-any
+import { assert, assertEquals, assertFalse } from "@std/assert"
+import { stub } from "@std/testing/mock"
+import { useTestConfig, srcroot } from "../hooks/useTestConfig.ts"
+import { _internals } from "../hooks/useFetch.ts"
+import install from "./install.ts"
+import SemVer from "../utils/semver.ts"
+import host from "../utils/host.ts"
+
+Deno.test("install bootstraps xz when system xz is missing", {
+  ignore: Deno.build.os != 'linux' || Deno.env.get("PKGX_TEST_NO_SYSTEM_XZ") != '1'
+}, async () => {
+  assertFalse(exists("/usr/bin/xz"))
+  assertFalse(exists("/bin/xz"))
+
+  const { arch, platform } = host()
+  const conf = useTestConfig()
+  const pkg = { project: "foo.com", version: new SemVer("5.43.0") }
+  const sha = "12ba4f3a0146f5fb11b6c796bc2e92a154a89bfb9cb2227cb5a9b95f6c2912bb"
+  const fixture = srcroot.join("fixtures/foo.com-5.43.0.tar.xz")
+  const cached = conf.cache.mkdir('p').join(`foo.com-5.43.0+${platform}+${arch}.tar.xz`)
+  fixture.cp({ to: cached })
+
+  const real_fetch = _internals.fetch
+  const fetch_stub = stub(_internals, "fetch", async (opts, init) => {
+    const url = opts as URL
+    const path = url.pathname
+    if (path.startsWith("/tukaani.org/xz/")) {
+      return await real_fetch(opts, init)
+    } else if (path.endsWith(".tar.xz.sha256sum")) {
+      return new Response(`${sha}  ${url.pathname.split("/").at(-1)}\n`)
+    } else if (path.endsWith(".tar.xz")) {
+      return {status: 304, ok: true} as any
+    } else {
+      return new Response(undefined, { status: 404 })
+    }
+  })
+
+  try {
+    const installation = await install(pkg)
+    assertEquals(installation.path, conf.prefix.join(pkg.project, `v${pkg.version}`))
+    assert(installation.path.join("bin/foo").isExecutableFile())
+    assert(conf.prefix.join("tukaani.org/xz/v5.8.3/bin/xz").isExecutableFile())
+  } finally {
+    fetch_stub.restore()
+  }
+})
+
+function exists(path: string): boolean {
+  try {
+    return Deno.statSync(path).isFile
+  } catch {
+    return false
+  }
+}

src/plumbing/install.ts

@@ -8,13 +8,20 @@ import useCache from "../hooks/useCache.ts"
 import useFetch from "../hooks/useFetch.ts"
 import { createHash } from "node:crypto"
 import Path from "../utils/Path.ts"
+import SemVer from "../utils/semver.ts"
 
 type Compression = 'xz' | 'gz'
+const SYSTEM_PATH = Deno.build.os == 'windows' ? "C:\\windows\\system32" : "/usr/bin:/bin"
+const XZ_BOOTSTRAP: Package = {
+  project: "tukaani.org/xz",
+  version: new SemVer("5.8.3")
+}
 
 export default async function install(pkg: Package, logger?: Logger): Promise<Installation> {
   const cellar = useCellar()
   const { prefix: PKGX_DIR, options: { compression } } = useConfig()
   const shelf = PKGX_DIR.join(pkg.project)
+  const candidates: Compression[] = is_xz_bootstrap(pkg) && !has_xz(SYSTEM_PATH) ? ['gz'] : compressions(compression)
 
   logger?.locking?.(pkg)
 
@@ -30,7 +37,7 @@ export default async function install(pkg: Package, logger?: Logger): Promise<In
     }
 
     let last_err: unknown
-    for (const candidate of compressions(compression)) {
+    for (const candidate of candidates) {
       try {
         return await install_bottle(pkg, { compression: candidate, PKGX_DIR, logger })
       } catch (err) {
@@ -67,7 +74,7 @@ async function install_bottle(pkg: Package, opts: {
   logger?.downloading?.({pkg})
 
   const checksum = await remote_SHA(new URL(`${url}.sha256sum`))
-  const PATH = Deno.build.os == 'windows' ? "C:\\windows\\system32" : "/usr/bin:/bin"
+  const env = await extractor_env({ compression, PKGX_DIR, logger })
 
   const tmpdir = Path.mktemp({
     //TODO dir should not be here ofc
@@ -81,7 +88,7 @@ async function install_bottle(pkg: Package, opts: {
     stdin: 'piped', stdout: "inherit", stderr: "inherit",
     cwd: tmpdir.string,
     /// hard coding path to ensure we don’t deadlock trying to use ourselves to untar ourselves
-    env: { PATH }
+    env
   }).spawn()
   const hasher = createHash("sha256")
   const writer = untar.stdin.getWriter()
@@ -142,6 +149,37 @@ function compressions(preferred: Compression): Compression[] {
   }
 }
 
+function is_xz_bootstrap(pkg: Package): boolean {
+  return pkg.project == XZ_BOOTSTRAP.project && pkg.version.eq(XZ_BOOTSTRAP.version)
+}
+
+async function extractor_env(opts: {
+  compression: Compression
+  PKGX_DIR: Path
+  logger?: Logger
+}): Promise<Record<string, string>> {
+  const env: Record<string, string> = { PATH: SYSTEM_PATH }
+
+  if (opts.compression != 'xz') return env
+  if (has_xz(SYSTEM_PATH)) return env
+
+  const xz = await install(XZ_BOOTSTRAP, opts.logger)
+  env.PATH = `${xz.path.join("bin")}:${SYSTEM_PATH}`
+  if (Deno.build.os == 'linux') {
+    env.LD_LIBRARY_PATH = xz.path.join("lib").string
+  }
+  return env
+}
+
+function has_xz(PATH: string): boolean {
+  const sep = Deno.build.os == 'windows' ? ';' : ':'
+  const ext = Deno.build.os == 'windows' ? '.exe' : ''
+  for (const part of PATH.split(sep)) {
+    if (new Path(part).join(`xz${ext}`).isExecutableFile()) return true
+  }
+  return false
+}
+
 function is_not_found(err: unknown): boolean {
   return (
     err instanceof DownloadError && err.status == 404 ||