From 37071fb7658e94def0dfd73bc333dff3adef133f Mon Sep 17 00:00:00 2001
From: Lilith River <lilith@imazen.io>
Date: Wed, 22 Jan 2025 01:25:15 -0700
Subject: [PATCH] fix ci: bucket and url handling

---
 .github/workflows/ci.yml    | 54 ++++++++++++++++++++++++-------------
 .github/workflows/readme.md |  7 ++++-
 2 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 932735af4..432534b6d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,8 +21,10 @@ on:
       - published
 
 env:
-  HTTPS_IMAGEFLOW_NIGHTLIES: "https://s3-us-west-1.amazonaws.com/imageflow-nightlies"
-  IMAGEFLOW_DOWNLOAD_URL_TAR_GZ: "https://s3-us-west-1.amazonaws.com/imageflow-nightlies/commits/${{ github.sha }}/linux64.tar.gz"
+  IMAGEFLOW_RELEASE_BUCKET: ${{ env.IMAGEFLOW_RELEASE_BUCKET || 'imageflow-nightlies' }}
+  IMAGEFLOW_RELEASE_BUCKET_REGION: ${{ env.IMAGEFLOW_RELEASE_BUCKET_REGION || 'us-west-1' }}
+  HTTPS_IMAGEFLOW_BUCKET: "https://s3.${{ env.IMAGEFLOW_RELEASE_BUCKET_REGION }}.amazonaws.com/${{ env.IMAGEFLOW_RELEASE_BUCKET }}"
+  IMAGEFLOW_DOWNLOAD_URL_TAR_GZ: "https://s3.${{ env.IMAGEFLOW_RELEASE_BUCKET_REGION }}.amazonaws.com/${{ env.IMAGEFLOW_RELEASE_BUCKET }}/commits/${{ github.sha }}/linux64.tar.gz"
 
 jobs:
   build:
@@ -196,13 +198,13 @@ jobs:
           echo "IMAGEFLOW_TOOL_PATH=${{ env.REL_BINARIES_DIR }}imageflow_tool${{ contains(matrix.os, 'windows') && '.exe' || '' }}" >> $GITHUB_ENV
 
           # We want to check these are successfully uploaded
-          echo "ESTIMATED_ARTIFACT_URL_COMMITS=${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}/commits/${{ github.sha }}/${{ matrix.commit-suffix }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
-          echo "ESTIMATED_MUSL_VERSIONED_URL=${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}/static/${{ github.ref_name }}/${{ env.IMAGEFLOW_TAG_SHA_SUFFIX }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
-          echo "ESTIMATED_MUSL_LATEST_URL=${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}/static/latest/${{ matrix.target }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
+          echo "ESTIMATED_ARTIFACT_URL_COMMITS=${{ env.HTTPS_IMAGEFLOW_BUCKET }}/commits/${{ github.sha }}/${{ matrix.commit-suffix }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
+          echo "ESTIMATED_MUSL_VERSIONED_URL=${{ env.HTTPS_IMAGEFLOW_BUCKET }}/static/${{ github.ref_name }}/${{ env.IMAGEFLOW_TAG_SHA_SUFFIX }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
+          echo "ESTIMATED_MUSL_LATEST_URL=${{ env.HTTPS_IMAGEFLOW_BUCKET }}/static/latest/${{ matrix.target }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
 
           # These are compiled into the binaries using a build script. We might review to reduce them later.
-          echo "ESTIMATED_DOCS_URL=${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}/doc" >> $GITHUB_ENV
-          echo "ESTIMATED_ARTIFACT_URL=${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}/releases/${{ github.ref_name }}/${{ env.IMAGEFLOW_TAG_SHA_SUFFIX }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
+          echo "ESTIMATED_DOCS_URL=${{ env.HTTPS_IMAGEFLOW_BUCKET }}/doc" >> $GITHUB_ENV
+          echo "ESTIMATED_ARTIFACT_URL=${{ env.HTTPS_IMAGEFLOW_BUCKET }}/releases/${{ github.ref_name }}/${{ env.IMAGEFLOW_TAG_SHA_SUFFIX }}.${{ env.EXTENSION }}" >> $GITHUB_ENV
           echo "CI_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
           echo "CI=True" >> $GITHUB_ENV
           echo "CI_REPO=${{ github.repository }}" >> $GITHUB_ENV
@@ -294,7 +296,7 @@ jobs:
           LIBIMAGEFLOW_DYNAMIC: ${{ env.LIBIMAGEFLOW_DYNAMIC }}
           MATRIX_COMMIT_SUFFIX: ${{ matrix.commit-suffix }}
           MATRIX_TARGET: ${{ matrix.target }}
-          HTTPS_UPLOAD_BASE: ${{ env.HTTPS_IMAGEFLOW_NIGHTLIES }}
+          HTTPS_UPLOAD_BASE: ${{ env.HTTPS_IMAGEFLOW_BUCKET }}
 
 
 
@@ -338,7 +340,7 @@ jobs:
         with:
           aws_key_id: ${{ secrets.AWS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
-          aws_bucket: ${{ secrets.AWS_BUCKET }}
+          aws_bucket: ${{ env.IMAGEFLOW_RELEASE_BUCKET }}
           source_dir: ./artifacts/upload
           destination_dir: ''
 
@@ -360,21 +362,35 @@ jobs:
           done
         shell: bash
 
-      - name: "Verify artifacts are now on S3, and public. Use curl to check, then delete"
+      - name: "Verify artifacts are now on S3, and public"
         if: ${{ env.ALLOW_PUBLISH == 'true' }}
         shell: bash
+        continue-on-error: true
         run: |
-          mkdir -p ./artifacts/verify
-          cd ./artifacts/verify
-          curl -s ${{ env.ESTIMATED_ARTIFACT_URL }}
-          curl -s ${{ env.ESTIMATED_ARTIFACT_URL_COMMITS }}
-          # if musl
+          # Function to test URL silently and return status
+          test_url() {
+            if curl -s -f -I "$1" 2>/dev/null >/dev/null; then
+              echo "✓ OK $1"
+              return 0
+            else
+              echo "✗ FAILED $1"
+              return 1
+            fi
+          }
+
+          # Test all relevant URLs
+          failed=0
+          test_url "${{ env.ESTIMATED_ARTIFACT_URL }}" || failed=1
+          test_url "${{ env.ESTIMATED_ARTIFACT_URL_COMMITS }}" || failed=1
+
+          # Test MUSL-specific URLs if applicable
           if [[ ${{ matrix.target }} == *"musl"* ]]; then
-            curl -s ${{ env.ESTIMATED_MUSL_VERSIONED_URL }}
-            curl -s ${{ env.ESTIMATED_MUSL_LATEST_URL }}
+            test_url "${{ env.ESTIMATED_MUSL_VERSIONED_URL }}" || failed=1
+            test_url "${{ env.ESTIMATED_MUSL_LATEST_URL }}" || failed=1
           fi
-          cd ${{ github.workspace }}
-          rm -rf ./artifacts/verify
+
+          # Exit with failure if any URL test failed
+          exit $failed
 
 
 
diff --git a/.github/workflows/readme.md b/.github/workflows/readme.md
index 208661e44..b65f88468 100644
--- a/.github/workflows/readme.md
+++ b/.github/workflows/readme.md
@@ -3,9 +3,14 @@
 
 * AWS_KEY_ID: Key ID for AWS Bucket.
 * AWS_SECRET_ACCESS_KEY: Key for AWS Bucket
-* AWS_BUCKET: Bucket Name
 * NUGET_API_KEY: Nuget.org API Key
 * DOCKER_USERNAME: Docker hub username
 * DOCKER_PASSWORD: Docker hub password
 * NUGET_UPLOAD_NUGET: true|false, switch for upload to Nuget.org
 * NUGET_UPLOAD_GITHUB: true|false, switch for upload to GitHub package registry
+
+
+## Vars
+
+* IMAGEFLOW_RELEASE_BUCKET: Bucket Name
+* IMAGEFLOW_RELEASE_BUCKET_REGION: Bucket Region