[WIP] Orchestration + GCP modernization: composition refactor, uv migration, OGC Features

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# Don't lint the virtualenv or generated output dirs. extend-exclude keeps
+# flake8's built-in defaults (.git, __pycache__, build, dist, ...).
+extend-exclude =
+    .venv,
+    output,
+    output_1

.github/workflows/cicd.yml

@@ -1,56 +1,39 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
 name: CI/CD
 
 on:
   push:
-    branches: [ "pre-production", "feature/jir", "dev/jab"]
+    branches: [ "pre-production", "feature/jir", "dev/jab", "feature/orchestration-gcp-uv" ]
   pull_request:
-    branches: [ "pre-production"]
+    branches: [ "pre-production" ]
 
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
       with:
-        python-version: "3.10"
-        cache: "pip"
+        version: "latest"
 
-    - name: Cache pip dependencies
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+    - name: Set up Python
+      run: uv python install 3.10
 
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest mypy
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      run: uv sync --extra dev
 
     - name: Lint with flake8
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        uv run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        uv run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
     - name: Type check with mypy
-      run: |
-        mypy . --ignore-missing-imports
+      run: uv run mypy . --ignore-missing-imports
 
     - name: Test with pytest
-      run: |
-        pytest -s tests
+      run: uv run pytest -s tests

.github/workflows/dagster-cloud-branch-deployments.yml

@@ -0,0 +1,86 @@
+name: Dagster+ Serverless Branch Deployments
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
+
+concurrency:
+  group: ${{ github.ref }}/branch_deployments
+  cancel-in-progress: true
+
+env:
+  DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
+  DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+  ENABLE_FAST_DEPLOYS: 'true'
+  PYTHON_VERSION: '3.10'
+  DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
+
+jobs:
+  dagster_cloud_default_deploy:
+    name: Dagster Serverless Deploy
+    runs-on: ubuntu-22.04
+    outputs:
+      build_info: ${{ steps.parse-workspace.outputs.build_info }}
+    steps:
+      - name: Check Dagster+ credentials present
+        id: guard
+        run: |
+          if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy."
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Prerun Checks
+        id: prerun
+        if: steps.guard.outputs.enabled == 'true'
+        uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1
+
+      - name: Launch Docker Deploy
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        id: parse-workspace
+        uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
+        with:
+          dagster_cloud_file: $DAGSTER_CLOUD_FILE
+
+      - name: Checkout for Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+          path: project-repo
+
+      - name: Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        with:
+          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
+          build_output_dir: "$GITHUB_WORKSPACE/build"
+          python_version: "${{ env.PYTHON_VERSION }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  dagster_cloud_docker_deploy:
+    name: Docker Deploy
+    runs-on: ubuntu-22.04
+    if: needs.dagster_cloud_default_deploy.outputs.build_info
+    needs: dagster_cloud_default_deploy
+    strategy:
+      fail-fast: false
+      matrix:
+        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Build and deploy to Dagster+ serverless
+        uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1
+        with:
+          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+          location: ${{ toJson(matrix.location) }}
+          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
+          organization_id: ${{ secrets.ORGANIZATION_ID }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dagster-cloud-deploy.yml

@@ -0,0 +1,89 @@
+name: Dagster+ Serverless Deploy
+
+on:
+  push:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.ref }}/deploy
+  cancel-in-progress: true
+
+env:
+  DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
+  DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+  # PEX fast deploy. No system GDAL needed (only shapely, which ships a wheel
+  # with bundled GEOS), so a Docker build would be pure overhead.
+  ENABLE_FAST_DEPLOYS: 'true'
+  PYTHON_VERSION: '3.10'
+  DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
+
+jobs:
+  dagster_cloud_default_deploy:
+    name: Dagster Serverless Deploy
+    runs-on: ubuntu-22.04
+    outputs:
+      build_info: ${{ steps.parse-workspace.outputs.build_info }}
+    steps:
+      - name: Check Dagster+ credentials present
+        id: guard
+        run: |
+          if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy."
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Prerun Checks
+        id: prerun
+        if: steps.guard.outputs.enabled == 'true'
+        uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1
+
+      - name: Launch Docker Deploy
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        id: parse-workspace
+        uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
+        with:
+          dagster_cloud_file: $DAGSTER_CLOUD_FILE
+
+      - name: Checkout for Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+          path: project-repo
+
+      - name: Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        with:
+          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
+          build_output_dir: "$GITHUB_WORKSPACE/build"
+          python_version: "${{ env.PYTHON_VERSION }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  dagster_cloud_docker_deploy:
+    name: Docker Deploy
+    runs-on: ubuntu-22.04
+    if: needs.dagster_cloud_default_deploy.outputs.build_info
+    needs: dagster_cloud_default_deploy
+    strategy:
+      fail-fast: false
+      matrix:
+        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Build and deploy to Dagster+ serverless
+        uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1
+        with:
+          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+          location: ${{ toJson(matrix.location) }}
+          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
+          organization_id: ${{ secrets.ORGANIZATION_ID }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/orchestration-ci.yml

@@ -0,0 +1,49 @@
+name: Orchestration CI
+
+on:
+  push:
+    paths:
+      - 'orchestration/**'
+      - 'backend/**'
+  pull_request:
+    branches: [ "pre-production" ]
+    paths:
+      - 'orchestration/**'
+
+permissions:
+  contents: read
+
+jobs:
+  lint-and-import-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+
+    - name: Set up Python
+      run: uv python install 3.10
+
+    - name: Install root package deps
+      run: uv sync --extra dev
+
+    - name: Install orchestration deps
+      run: uv pip install dagster dagster-gcp google-cloud-storage google-cloud-secret-manager Jinja2
+
+    - name: Lint orchestration with flake8
+      run: |
+        uv run flake8 orchestration/ --count --select=E9,F63,F7,F82 --show-source --statistics
+        uv run flake8 orchestration/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    - name: Import check — definitions load without error
+      run: |
+        uv run python -c "
+        import sys; sys.path.insert(0, '.')
+        from orchestration.definitions import defs
+        assert len(list(defs.assets)) > 0, 'No assets defined'
+        print(f'OK: {len(list(defs.assets))} assets, {len(defs.schedules)} schedules')
+        "

.github/workflows/publish-to-pypi.yml

@@ -18,10 +18,10 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@master
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.13
 
       - name: Install pypa/build
         run: >-

.gitignore

@@ -8,6 +8,7 @@ frontend/requirements.txt
 frontend/output_timeseries
 backend/test00112233.csv
 frontend/api/cache
+.DS_Store
 
 # Byte-compiled / optimized / DLL files
 __pycache__/