From d43333ad4674bfe7be9c1f28f28a570b1fa9d58d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 7 Jul 2026 16:22:50 -0400 Subject: [PATCH 1/3] Use Fabric DNS endpoint for MASS pull-through cache Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ba10349d5a..738765c83ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,9 @@ variables: DEPENDENCY_CACHE_POLICY: pull BUILD_CACHE_POLICY: pull GRADLE_VERSION: "9.6.1" # must match gradle-wrapper.properties - MASS_READ_URL: "https://mass-read.us1.ddbuild.io" + # Use the Fabric DNS endpoint instead of the ISP URL (mass-read.us1.ddbuild.io), which returns + # transient 503s from the Fabric ISP layer. Per MASS team guidance, the DNS endpoint is more reliable. + MASS_READ_URL: "https://mass-read.rapid-dependency-management-mass.all-clusters.local-dc.fabric.dog:8443" MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the DataDog/images repo From 0c1712095cc361129f6ef9ca19ff29940d107fe6 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 9 Jul 2026 12:09:34 -0400 Subject: [PATCH 2/3] Trust MASS Fabric endpoint TLS cert at runtime Fetch the MASS Fabric endpoint's certificate chain and merge it into a copy of the JDK truststore, exposed via JAVA_TOOL_OPTIONS, so the Gradle wrapper download and build trust the Datadog-internal cert. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85aeb82a107..c0f13790f41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -200,6 +200,36 @@ default: echo "Failed to find base ref for PR" >&2 fi +# The MASS pull-through cache (MASS_READ_URL) is served over the Fabric DNS endpoint, which presents +# a Datadog-internal TLS certificate that the bundled JDKs do not trust out of the box. Fetch the +# endpoint's certificate chain at runtime and merge it into a copy of the active JDK's truststore, +# then point every JVM at that truststore via JAVA_TOOL_OPTIONS so both the Gradle wrapper download +# and the build itself trust it. We copy cacerts first because javax.net.ssl.trustStore *replaces* +# the default store, so the copy keeps all public CAs and merely adds the Datadog CA. +.trust_mass_certs: &trust_mass_certs + - | + mass_host="${MASS_READ_URL#*://}"; mass_host="${mass_host%%/*}" + mass_name="${mass_host%%:*}" + mass_port="${mass_host##*:}"; [ "$mass_port" = "$mass_host" ] && mass_port=443 + java_home="${JAVA_HOME:-$(dirname "$(dirname "$(readlink -f "$(command -v keytool || command -v java)")")")}" + src_cacerts="${java_home}/lib/security/cacerts" + [ -f "$src_cacerts" ] || src_cacerts="${java_home}/jre/lib/security/cacerts" + truststore=/tmp/mass-truststore.p12 + rm -f "$truststore" /tmp/mass-chain.pem /tmp/mass-cert-*.pem + cp "$src_cacerts" "$truststore" && chmod u+w "$truststore" + echo | openssl s_client -showcerts -servername "$mass_name" -connect "${mass_name}:${mass_port}" 2>/dev/null > /tmp/mass-chain.pem + awk '/-----BEGIN CERTIFICATE-----/{n++} /-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/{print > ("/tmp/mass-cert-" n ".pem")}' /tmp/mass-chain.pem + for cert in /tmp/mass-cert-*.pem; do + [ -s "$cert" ] || continue + keytool -importcert -noprompt -trustcacerts -storetype PKCS12 -storepass changeit \ + -keystore "$truststore" -alias "mass-$(basename "$cert" .pem)" -file "$cert" + done + if [ -s "$truststore" ]; then + export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=${truststore} -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12" + else + echo "WARNING: MASS truststore was not created; leaving default truststore in place" >&2 + fi + .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build @@ -258,6 +288,8 @@ default: mass_read_host="${MASS_READ_URL#https://}" mass_read_host="${mass_read_host%/}" sed -i "/^distributionUrl=/ s|services.gradle.org|${mass_read_host}/internal/artifact/services.gradle.org|" gradle/wrapper/gradle-wrapper.properties + # Trust the MASS Fabric endpoint's Datadog-internal TLS cert before the Gradle wrapper downloads the distribution. + - *trust_mass_certs - mkdir -p .mvn/caches # Redirect Spotless's Equo/Solstice P2 cache into the project tree so it is captured by the GitLab cache. # Solstice (https://github.com/equodev/equo-ide) defaults to ~/.m2/repository/dev/equo/p2-data, which is outside $CI_PROJECT_DIR. @@ -817,6 +849,8 @@ muzzle-dep-report: mass_read_host="${MASS_READ_URL#https://}" mass_read_host="${mass_read_host%/}" sed -i "/^distributionUrl=/ s|services.gradle.org|${mass_read_host}/internal/artifact/services.gradle.org|" gradle/wrapper/gradle-wrapper.properties + # Trust the MASS Fabric endpoint's Datadog-internal TLS cert before the Gradle wrapper downloads the distribution. + - *trust_mass_certs - *normalize_node_index - *prepare_test_env # Disable CDS in forked JVMs to avoid SIGSEGVs on Linux arm64. From 0573cb1c84d93ce4c7228b4d9f220327370a9ed9 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 9 Jul 2026 12:27:31 -0400 Subject: [PATCH 3/3] Build MASS truststore as PKCS12 to fix keytool format error The base JDK ships a JKS cacerts; copying it and forcing -storetype PKCS12 made keytool fail with 'DerInputStream.getLength(): lengthTag too big', aborting every Gradle job. Convert cacerts to PKCS12 via importkeystore and make cert fetch/import best-effort so a transient blip cannot abort the job. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0f13790f41..7f1e88287ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -216,13 +216,19 @@ default: [ -f "$src_cacerts" ] || src_cacerts="${java_home}/jre/lib/security/cacerts" truststore=/tmp/mass-truststore.p12 rm -f "$truststore" /tmp/mass-chain.pem /tmp/mass-cert-*.pem - cp "$src_cacerts" "$truststore" && chmod u+w "$truststore" - echo | openssl s_client -showcerts -servername "$mass_name" -connect "${mass_name}:${mass_port}" 2>/dev/null > /tmp/mass-chain.pem - awk '/-----BEGIN CERTIFICATE-----/{n++} /-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/{print > ("/tmp/mass-cert-" n ".pem")}' /tmp/mass-chain.pem + # Convert the JDK's cacerts (JKS on legacy JDKs, PKCS12 on modern ones) into a writable PKCS12 + # copy so a single truststore works across every bundled JDK, matching trustStoreType below. + keytool -importkeystore -noprompt \ + -srckeystore "$src_cacerts" -srcstorepass changeit \ + -destkeystore "$truststore" -deststorepass changeit -deststoretype PKCS12 + # Best-effort fetch/import: a transient connection blip must not abort the whole job (set -e); + # if the cert is missing, the Gradle download simply fails later with a clear TLS error. + echo | openssl s_client -showcerts -servername "$mass_name" -connect "${mass_name}:${mass_port}" 2>/dev/null > /tmp/mass-chain.pem || true + awk '/-----BEGIN CERTIFICATE-----/{n++} /-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/{print > ("/tmp/mass-cert-" n ".pem")}' /tmp/mass-chain.pem || true for cert in /tmp/mass-cert-*.pem; do [ -s "$cert" ] || continue keytool -importcert -noprompt -trustcacerts -storetype PKCS12 -storepass changeit \ - -keystore "$truststore" -alias "mass-$(basename "$cert" .pem)" -file "$cert" + -keystore "$truststore" -alias "mass-$(basename "$cert" .pem)" -file "$cert" || true done if [ -s "$truststore" ]; then export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=${truststore} -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12"