diff --git a/.github/bosh-lite-files/force-delete-env.sh b/.github/bosh-lite-files/force-delete-env.sh new file mode 100644 index 0000000000..f61be60d93 --- /dev/null +++ b/.github/bosh-lite-files/force-delete-env.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Force-tears-down a bosh-lite environment directly via the GCP CPI and +# gcloud, bypassing bbl/bosh's graceful deployment-deletion path entirely. +# +# Use this when `bbl down` hangs or fails - e.g. warden bind-mount +# "target is busy" errors during deployment deletion, or SSH-tunnel +# connectivity issues to an unhealthy director/jumpbox. `bosh delete-env` +# only needs the local state file + the CPI (GCP API) to destroy a VM, so it +# works even when the director/jumpbox is unresponsive or already gone. +# +# Usage: force-delete-env.sh +# Required env vars: BBL_GCP_SERVICE_ACCOUNT_KEY_PATH, GOOGLE_CLOUD_PROJECT, +# BBL_GCP_REGION +# Expects /bbl-state to already be downloaded in the cwd. + +set -uo pipefail + +env_name="$1" +state_dir="${env_name}/bbl-state" +zone="$(jq -r '.gcp.zone' "${state_dir}/bbl-state.json")" + +echo "Force-deleting director VM for ${env_name}" +bosh delete-env \ + "${state_dir}/bosh-deployment/bosh.yml" \ + --state "${state_dir}/vars/bosh-state.json" \ + --vars-store "${state_dir}/vars/director-vars-store.yml" \ + --vars-file "${state_dir}/vars/director-vars-file.yml" \ + -o "${state_dir}/bosh-deployment/gcp/cpi.yml" \ + -o "${state_dir}/bosh-deployment/jumpbox-user.yml" \ + -o "${state_dir}/bosh-deployment/uaa.yml" \ + -o "${state_dir}/bosh-deployment/credhub.yml" \ + -o "${state_dir}/bbl-ops-files/gcp/bosh-director-ephemeral-ip-ops.yml" \ + --var-file gcp_credentials_json="${BBL_GCP_SERVICE_ACCOUNT_KEY_PATH}" \ + -v project_id="${GOOGLE_CLOUD_PROJECT}" \ + -v zone="${zone}" \ + --non-interactive || true + +echo "Force-deleting jumpbox VM for ${env_name}" +bosh delete-env \ + "${state_dir}/jumpbox-deployment/jumpbox.yml" \ + --state "${state_dir}/vars/jumpbox-state.json" \ + --vars-store "${state_dir}/vars/jumpbox-vars-store.yml" \ + --vars-file "${state_dir}/vars/jumpbox-vars-file.yml" \ + -o "${state_dir}/jumpbox-deployment/gcp/cpi.yml" \ + --var-file gcp_credentials_json="${BBL_GCP_SERVICE_ACCOUNT_KEY_PATH}" \ + -v project_id="${GOOGLE_CLOUD_PROJECT}" \ + -v zone="${zone}" \ + --non-interactive || true + +tfstate="${state_dir}/vars/terraform.tfstate" +if [ -f "$tfstate" ]; then + echo "Deleting remaining terraform-managed network resources for ${env_name}" + + jq -r '.resources[] | select(.type=="google_compute_firewall") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute firewall-rules delete "$name" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_dns_record_set") | "\(.instances[0].attributes.name)\t\(.instances[0].attributes.managed_zone)\t\(.instances[0].attributes.type)"' "$tfstate" | \ + while IFS=$'\t' read -r name dns_zone rtype; do gcloud dns record-sets delete "$name" --zone "$dns_zone" --type "$rtype" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_compute_route") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute routes delete "$name" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_compute_router") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute routers delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_compute_address") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute addresses delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_compute_subnetwork") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute networks subnets delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done + + jq -r '.resources[] | select(.type=="google_compute_network") | .instances[0].attributes.name' "$tfstate" | \ + while read -r name; do gcloud compute networks delete "$name" --quiet || true; done +fi + +echo "Force-delete complete for ${env_name}" diff --git a/.github/workflows/create-bosh-lite.yml b/.github/workflows/create-bosh-lite.yml index 3ef7d15fcf..b36ed8f229 100644 --- a/.github/workflows/create-bosh-lite.yml +++ b/.github/workflows/create-bosh-lite.yml @@ -132,6 +132,7 @@ jobs: bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version" - name: Deploy cf + timeout-minutes: 90 run: | env_name="${{ steps.setup-bbl-env.outputs.envName }}" cd $env_name/bbl-state diff --git a/.github/workflows/delete-bosh-lite.yml b/.github/workflows/delete-bosh-lite.yml index 8cfb7dee05..3b66774a08 100644 --- a/.github/workflows/delete-bosh-lite.yml +++ b/.github/workflows/delete-bosh-lite.yml @@ -66,21 +66,41 @@ jobs: - name: Download file from GCS run: | - gsutil -m cp -P -R gs://cf-cli-bosh-lites/${ENV_NAME} . + gcloud storage cp -r gs://cf-cli-bosh-lites/${ENV_NAME} . - name: delete bosh + id: delete-bosh + timeout-minutes: 30 + continue-on-error: true run: | cd ${ENV_NAME}/bbl-state eval "$(bbl print-env --shell-type posix)" # Delete the CF deployment first so warden containers release their - # bind mounts on the director's persistent disk. + # bind mounts on the director's persistent disk. --force makes bosh ignore + # per-instance errors (like a busy unmount) and still delete the + # deployment record. echo "Deleting CF deployment ${BOSH_DEPLOYMENT}" - bosh delete-deployment -d ${BOSH_DEPLOYMENT} --force --skip-drain || true + bosh delete-deployment -d ${BOSH_DEPLOYMENT} --force || true echo "Deleting env ${ENV_NAME}" bbl down --no-confirm - + + - name: force delete env (fallback) + if: steps.delete-bosh.outcome == 'failure' + timeout-minutes: 15 + run: | + echo "${BBL_GCP_SERVICE_ACCOUNT_KEY}" > /tmp/gcp_key.json + export BBL_GCP_SERVICE_ACCOUNT_KEY_PATH=/tmp/gcp_key.json + bash ${GITHUB_WORKSPACE}/cli/.github/bosh-lite-files/force-delete-env.sh "${ENV_NAME}" + rm -f /tmp/gcp_key.json + - name: delete gcs bucket + if: always() + run: | + gcloud storage rm -r gs://cf-cli-bosh-lites/${ENV_NAME} + + - name: warn if delete bosh needed the fallback + if: steps.delete-bosh.outcome == 'failure' run: | - gsutil rm -R gs://cf-cli-bosh-lites/${ENV_NAME} \ No newline at end of file + echo "::warning::The graceful 'delete bosh' step failed for ${ENV_NAME} and the environment was torn down via the force-delete-env.sh fallback instead. This usually means an abandoned BOSH task or a warden mount was still busy - see the 'delete bosh' step log for details." \ No newline at end of file