bump libpkgx to prefer .xz (.gz deprecated)

.github/workflows/ci.yml

@@ -43,6 +43,11 @@ jobs:
 
       - run: deno coverage cov_profile --lcov --output=cov_profile.lcov
 
+      # errors on macOS
+      - run: |
+          brew tap coverallsapp/coveralls
+          brew trust --formula coverallsapp/coveralls/coveralls
+        if: matrix.os == 'macos-latest'
       - uses: coverallsapp/github-action@v2
         with:
           path-to-lcov: cov_profile.lcov

src/hooks/useConfig.test.ts

@@ -20,7 +20,7 @@ Deno.test("useConfig", () => {
   } else {
     assertEquals(config.pantries.map(x => x.string), ["/foo", "/bar"])
   }
-  assertEquals(config.options.compression, "gz")
+  assertEquals(config.options.compression, "xz")
 
   assertFalse(_internals.boolize("false"))
   assertFalse(_internals.boolize("0"))

src/hooks/useConfig.ts

@@ -64,10 +64,8 @@ export function ConfigDefault(env = Deno.env.toObject()): Config {
     ?? platform_data_home_default(home, env)
   ).join("pkgx")
   const dist = env['PKGX_DIST_URL']?.trim() ?? 'https://dist.pkgx.dev'
-  const isCI = boolize(env['CI']) ?? false
   const UserAgent = flatmap(getv(), v => `libpkgx/${v}`) ?? 'libpkgx'
-  //TODO prefer 'xz' on Linux (as well) if supported
-  const compression = !isCI && host().platform == 'darwin' ? 'xz' : 'gz'
+  const compression = 'xz'
 
   return {
     prefix,

src/plumbing/install.test.ts

@@ -18,7 +18,10 @@ Deno.test("install", async runner => {
 
   // deno-lint-ignore require-await
   const fetch_stub = stub(_internals, "fetch", async opts => {
-    if ((opts as URL).pathname.endsWith("sha256sum")) {
+    const path = (opts as URL).pathname
+    if (path.endsWith(".tar.xz.sha256sum")) {
+      return new Response(undefined, { status: 404 })
+    } else if (path.endsWith("sha256sum")) {
       return {
         ok: true,
         status: 200,
@@ -33,12 +36,12 @@ Deno.test("install", async runner => {
 
   try {
     await runner.step("install()", async runner => {
-      const conf = useTestConfig({ CI: "1" }) // CI to force .gz compression
+      const conf = useTestConfig()
 
       /// download() will use the cached version and not do http
       srcroot.join("fixtures/foo.com-5.43.0.tgz").cp({ to:
         conf.cache.mkdir('p').join(`foo.com-5.43.0+${platform}+${arch}.tar.gz`)
-      })
+      }) // only gzip is cached to exercise fallback from the xz default
 
       await runner.step("download & install", async () => {
         // for coverage
@@ -71,12 +74,12 @@ Deno.test("install", async runner => {
 
     await runner.step("install locks", async () => {
 
-      const conf = useTestConfig({ CI: "1" })
+      const conf = useTestConfig()
 
       /// download() will use the cached version and not do http
       srcroot.join("fixtures/foo.com-5.43.0.tgz").cp({ to:
         conf.cache.mkdir('p').join(`foo.com-5.43.0+${platform}+${arch}.tar.gz`)
-      })
+      }) // only gzip is cached to exercise fallback from the xz default
 
       let unlocked_once = false
       const logger: Logger = {

src/plumbing/install.ts

@@ -1,6 +1,6 @@
 import { Package, Installation, StowageNativeBottle } from "../types.ts"
 import useOffLicense from "../hooks/useOffLicense.ts"
-import useDownload from "../hooks/useDownload.ts"
+import useDownload, { DownloadError } from "../hooks/useDownload.ts"
 import { flock } from "../utils/flock.ts"
 import useConfig from "../hooks/useConfig.ts"
 import useCellar from "../hooks/useCellar.ts"
@@ -9,14 +9,11 @@ import useFetch from "../hooks/useFetch.ts"
 import { createHash } from "node:crypto"
 import Path from "../utils/Path.ts"
 
-export default async function install(pkg: Package, logger?: Logger): Promise<Installation> {
-  const { project, version } = pkg
+type Compression = 'xz' | 'gz'
 
+export default async function install(pkg: Package, logger?: Logger): Promise<Installation> {
   const cellar = useCellar()
   const { prefix: PKGX_DIR, options: { compression } } = useConfig()
-  const stowage = StowageNativeBottle({ pkg: { project, version }, compression })
-  const url = useOffLicense('s3').url(stowage)
-  const tarball = useCache().path(stowage)
   const shelf = PKGX_DIR.join(pkg.project)
 
   logger?.locking?.(pkg)
@@ -32,28 +29,64 @@ export default async function install(pkg: Package, logger?: Logger): Promise<In
       return already_installed
     }
 
-    logger?.downloading?.({pkg})
+    let last_err: unknown
+    for (const candidate of compressions(compression)) {
+      try {
+        return await install_bottle(pkg, { compression: candidate, PKGX_DIR, logger })
+      } catch (err) {
+        if (!is_not_found(err)) throw err
+        last_err = err
+      }
+    }
+    throw last_err
+  } finally {
+    logger?.unlocking?.(pkg)
+    await unflock()
+  }
+}
+
+async function remote_SHA(url: URL) {
+  const rsp = await useFetch(url)
+  if (!rsp.ok) throw rsp
+  const txt = await rsp.text()
+  return txt.split(' ')[0]
+}
 
-    const PATH = Deno.build.os == 'windows' ? "C:\\windows\\system32" : "/usr/bin:/bin"
+async function install_bottle(pkg: Package, opts: {
+  compression: Compression
+  PKGX_DIR: Path
+  logger?: Logger
+}): Promise<Installation> {
+  const { project, version } = pkg
+  const { compression, PKGX_DIR, logger } = opts
+  const stowage = StowageNativeBottle({ pkg: { project, version }, compression })
+  const url = useOffLicense('s3').url(stowage)
+  const tarball = useCache().path(stowage)
+  const shelf = PKGX_DIR.join(pkg.project)
 
-    const tmpdir = Path.mktemp({
-      //TODO dir should not be here ofc
-      dir: PKGX_DIR.join(".local/tmp").join(pkg.project),
-      prefix: `v${pkg.version}.`
-      //NOTE ^^ inside pkgx prefix to avoid TMPDIR is on a different volume problems
-    })
-    const tar_args = compression == 'xz' ? 'xJf' : 'xzf'  // laughably confusing
-    const untar = new Deno.Command("tar", {
-      args: [tar_args, "-", "--strip-components", (pkg.project.split("/").length + 1).toString()],
-      stdin: 'piped', stdout: "inherit", stderr: "inherit",
-      cwd: tmpdir.string,
-      /// hard coding path to ensure we don’t deadlock trying to use ourselves to untar ourselves
-      env: { PATH }
-    }).spawn()
-    const hasher = createHash("sha256")
-    const remote_SHA_promise = remote_SHA(new URL(`${url}.sha256sum`))
-    const writer = untar.stdin.getWriter()
+  logger?.downloading?.({pkg})
+
+  const checksum = await remote_SHA(new URL(`${url}.sha256sum`))
+  const PATH = Deno.build.os == 'windows' ? "C:\\windows\\system32" : "/usr/bin:/bin"
+
+  const tmpdir = Path.mktemp({
+    //TODO dir should not be here ofc
+    dir: PKGX_DIR.join(".local/tmp").join(pkg.project),
+    prefix: `v${pkg.version}.`
+    //NOTE ^^ inside pkgx prefix to avoid TMPDIR is on a different volume problems
+  })
+  const tar_args = compression == 'xz' ? 'xJf' : 'xzf'  // laughably confusing
+  const untar = new Deno.Command("tar", {
+    args: [tar_args, "-", "--strip-components", (pkg.project.split("/").length + 1).toString()],
+    stdin: 'piped', stdout: "inherit", stderr: "inherit",
+    cwd: tmpdir.string,
+    /// hard coding path to ensure we don’t deadlock trying to use ourselves to untar ourselves
+    env: { PATH }
+  }).spawn()
+  const hasher = createHash("sha256")
+  const writer = untar.stdin.getWriter()
 
+  try {
     let total: number | undefined
     let n = 0
     await useDownload().download({
@@ -70,15 +103,14 @@ export default async function install(pkg: Package, logger?: Logger): Promise<In
       return writer.write(blob)
     })
 
-    writer.close()
+    await writer.close()
 
     const untar_exit_status = await untar.status
     if (!untar_exit_status.success) {
       throw new Error(`tar exited with status ${untar_exit_status.code}`)
     }
 
     const computed_hash_value = hasher.digest("hex")
-    const checksum = await remote_SHA_promise
 
     if (computed_hash_value != checksum) {
       tarball.rm()
@@ -93,19 +125,28 @@ export default async function install(pkg: Package, logger?: Logger): Promise<In
 
     return install
   } catch (err) {
+    try { await writer.close() } catch { /* already closed */ }
+    await untar.status
     tarball.rm()  //FIXME resumable downloads!
+    tmpdir.rm({ recursive: true })
     throw err
-  } finally {
-    logger?.unlocking?.(pkg)
-    await unflock()
   }
 }
 
-async function remote_SHA(url: URL) {
-  const rsp = await useFetch(url)
-  if (!rsp.ok) throw rsp
-  const txt = await rsp.text()
-  return txt.split(' ')[0]
+function compressions(preferred: Compression): Compression[] {
+  switch (preferred) {
+  case 'xz':
+    return ['xz', 'gz']
+  case 'gz':
+    return ['gz', 'xz']
+  }
+}
+
+function is_not_found(err: unknown): boolean {
+  return (
+    err instanceof DownloadError && err.status == 404 ||
+    err instanceof Response && err.status == 404
+  )
 }