secure-db

.github/workflows/CD_production.yml

@@ -33,14 +33,36 @@ jobs:
           with:
             credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }}
 
-        # Uses Google Cloud Secret Manager to store secret credentials
+        - name: Run Alembic migrations on production database
+          env:
+            DB_DRIVER: "cloudsql"
+            CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
+            CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
+            CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
+            CLOUD_SQL_IAM_AUTH: true
+          run: |
+            uv run alembic upgrade head
+
+        - name: Run backfill script on production database
+          env:
+            DB_DRIVER: "cloudsql"
+            CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
+            CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
+            CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
+            CLOUD_SQL_IAM_AUTH: true
+            GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
+            GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
+          run: |
+            uv run python -m transfers.backfill.backfill
+
         - name: Create app.yaml
           run: |
             cat <<EOF > app.yaml
             service: ocotillo-api
             runtime: python313
             entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
             instance_class: F4
+            service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com"
             handlers:
               - url: /.*
                 secure: always
@@ -51,7 +73,7 @@ jobs:
               CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
               CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
               CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
-              CLOUD_SQL_PASSWORD: "${{ secrets.CLOUD_SQL_PASSWORD }}"
+              CLOUD_SQL_IAM_AUTH: true
               GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
               GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
               AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}"
@@ -91,4 +113,4 @@ jobs:
         - name: Tag commit
           run: |
             git tag -a "production-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "staging gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)"
-            git push origin --tags
+            git push origin --tags

.github/workflows/CD_staging.yml

@@ -57,7 +57,7 @@ jobs:
             GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
             GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
           run: |
-            uv run python -m transfers.backfill.staging
+            uv run python -m transfers.backfill.backfill
 
         # Uses Google Cloud Secret Manager to store secret credentials
         - name: Create app.yaml
@@ -66,6 +66,7 @@ jobs:
             service: ocotillo-api-staging
             runtime: python313
             entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
+            service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com"
             instance_class: F4
             handlers:
               - url: /.*

run_backfill.sh

@@ -19,4 +19,4 @@ set +a
 uv run alembic upgrade head
 
 # Forward any args (e.g., --batch-size 500)
-python -m transfers.backfill.staging "$@"
+python -m transfers.backfill.backfill "$@"

transfers/backfill/staging.py → transfers/backfill/backfill.py

@@ -14,10 +14,10 @@
 # limitations under the License.
 # ===============================================================================
 """
-Orchestrates all backfills used in the staging CD pipeline.
+Orchestrates the backfill pipeline used in CD workflows.
 
 Preferred usage (avoids import path issues):
-    python -m transfers.backfill.staging --batch-size 1000
+    python -m transfers.backfill.backfill --batch-size 1000
 """
 
 import argparse
@@ -67,7 +67,7 @@ def run(batch_size: int = 1000) -> None:
 
 
 def _parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Run staging backfills.")
+    parser = argparse.ArgumentParser(description="Run backfill pipeline.")
     parser.add_argument(
         "--batch-size",
         type=int,