diff --git a/.github/workflows/CD_production.yml b/.github/workflows/CD_production.yml index 0908aa625..83fa0b7fb 100644 --- a/.github/workflows/CD_production.yml +++ b/.github/workflows/CD_production.yml @@ -33,7 +33,28 @@ jobs: with: credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} - # Uses Google Cloud Secret Manager to store secret credentials + - name: Run Alembic migrations on production database + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + run: | + uv run alembic upgrade head + + - name: Run backfill script on production database + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" + run: | + uv run python -m transfers.backfill.backfill + - name: Create app.yaml run: | cat < app.yaml @@ -41,6 +62,7 @@ jobs: runtime: python313 entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app instance_class: F4 + service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com" handlers: - url: /.* secure: always @@ -51,7 +73,7 @@ jobs: CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" - CLOUD_SQL_PASSWORD: "${{ secrets.CLOUD_SQL_PASSWORD }}" + CLOUD_SQL_IAM_AUTH: true GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}" @@ -91,4 +113,4 @@ jobs: - name: Tag commit run: | git tag -a "production-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "staging gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)" - git push origin --tags \ No newline at end of file + git push origin --tags diff --git a/.github/workflows/CD_staging.yml b/.github/workflows/CD_staging.yml index c5fc36e9f..04b1a13f6 100644 --- a/.github/workflows/CD_staging.yml +++ b/.github/workflows/CD_staging.yml @@ -57,7 +57,7 @@ jobs: GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" run: | - uv run python -m transfers.backfill.staging + uv run python -m transfers.backfill.backfill # Uses Google Cloud Secret Manager to store secret credentials - name: Create app.yaml @@ -66,6 +66,7 @@ jobs: service: ocotillo-api-staging runtime: python313 entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app + service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com" instance_class: F4 handlers: - url: /.* diff --git a/run_backfill.sh b/run_backfill.sh index b81149165..e2d0e6444 100755 --- a/run_backfill.sh +++ b/run_backfill.sh @@ -19,4 +19,4 @@ set +a uv run alembic upgrade head # Forward any args (e.g., --batch-size 500) -python -m transfers.backfill.staging "$@" +python -m transfers.backfill.backfill "$@" diff --git a/transfers/backfill/staging.py b/transfers/backfill/backfill.py similarity index 93% rename from transfers/backfill/staging.py rename to transfers/backfill/backfill.py index bea333d2b..44cfa8c56 100644 --- a/transfers/backfill/staging.py +++ b/transfers/backfill/backfill.py @@ -14,10 +14,10 @@ # limitations under the License. # =============================================================================== """ -Orchestrates all backfills used in the staging CD pipeline. +Orchestrates the backfill pipeline used in CD workflows. Preferred usage (avoids import path issues): - python -m transfers.backfill.staging --batch-size 1000 + python -m transfers.backfill.backfill --batch-size 1000 """ import argparse @@ -67,7 +67,7 @@ def run(batch_size: int = 1000) -> None: def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run staging backfills.") + parser = argparse.ArgumentParser(description="Run backfill pipeline.") parser.add_argument( "--batch-size", type=int,