Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
139 commits
Select commit Hold shift + click to select a range
c700ea5
Merge pull request #48 from DataIntegrationGroup/pre-production
jacob-a-brown Apr 29, 2025
932db0e
Merge pull request #50 from DataIntegrationGroup/pre-production
jacob-a-brown Apr 29, 2025
851ae45
Merge pull request #52 from DataIntegrationGroup/pre-production
jacob-a-brown Apr 29, 2025
084a084
Merge pull request #54 from DataIntegrationGroup/pre-production
jacob-a-brown May 2, 2025
c4a54be
Merge pull request #57 from DataIntegrationGroup/pre-production
jacob-a-brown May 21, 2025
85b6227
Merge pull request #59 from DataIntegrationGroup/pre-production
jacob-a-brown Jun 3, 2025
5fd2dd7
Merge pull request #62 from DataIntegrationGroup/pre-production
jacob-a-brown Sep 26, 2025
5192d6b
Merge pull request #64 from DataIntegrationGroup/pre-production
jacob-a-brown Oct 10, 2025
cd6d212
fix: use psycopg2-binary instead of psycopg2 for better compatibility
jacob-a-brown Jan 14, 2026
7ecc5c9
fix: bump version to 0.9.10
jacob-a-brown Jan 14, 2026
9ccdda4
feat(constants): add specific conductance to constants
jacob-a-brown Apr 6, 2026
02750b1
fix(source): decrease chunk size for nmbgmr site source
jacob-a-brown Apr 6, 2026
860a653
feat(parameter): add specific conductance as a parameter with unit uS/cm
jacob-a-brown Apr 6, 2026
40a9f2c
fix(specific conductance): update mappings | remove BOR
jacob-a-brown Apr 6, 2026
6d424cb
fix(specific conductance): remove records where temp not 25C
jacob-a-brown Apr 6, 2026
89537de
feat(parameter): add conductivity to README
jacob-a-brown Apr 7, 2026
2deec34
fix(.gitignore): ignore .DS_Store
jacob-a-brown Apr 7, 2026
fd9c809
feat(parameter): add conductivity to constants
jacob-a-brown Apr 7, 2026
7bafedb
feat(parameter): add conductivity mappings
jacob-a-brown Apr 7, 2026
8e61bf2
fix(mappings): ensure conducitivy and sc in each mapping
jacob-a-brown Apr 7, 2026
9196789
fix(config): make units for conductivity uS/cm
jacob-a-brown Apr 7, 2026
5f60b1a
fix(parameter): add conductivity to parameter list
jacob-a-brown Apr 7, 2026
15d57b9
fix(unit conversion): convert units for conductivity
jacob-a-brown Apr 7, 2026
5b6a5b7
feat(conductivity): filter for conductivity for WQP records
jacob-a-brown Apr 7, 2026
401f1c9
feat(setup): bump version to 0.10.0 for new parameters
jacob-a-brown Apr 7, 2026
b60e5aa
fix(github actions): use python 3.13 and setup-python@v5
jacob-a-brown Apr 7, 2026
c322402
fix(source): enable tag paramter for json requests to be Null
jacob-a-brown Apr 27, 2026
ec9f6e3
fix(config): update config/false agencies for bicarbonate
jacob-a-brown Apr 27, 2026
dcaf8d3
fix(USGS): use new USGS API endpoints for water level retrieval
jacob-a-brown Apr 27, 2026
036e9e7
fix(usgs): update transformer for combined-metadata endpoint
jacob-a-brown Apr 28, 2026
2425094
fix(usgs): update USGS water level retrieval for new APIs
jacob-a-brown Apr 28, 2026
500a0b1
fix(usgs): set limit to 50000 to avoid pagination/rate limits
jacob-a-brown Apr 29, 2026
d93869d
refactor(usgs): check usgs health with get request
jacob-a-brown Apr 29, 2026
12247f5
fix(config): set agencies for bicarbonate
jacob-a-brown Apr 29, 2026
4eaac1c
fix(nmbgmr): set timeout to 30 minutes and retry failures
jacob-a-brown Apr 29, 2026
b342922
fix(usgs): add timeout to usgs api calls and retry logic if errors
jacob-a-brown Apr 29, 2026
3f2d104
fix(usgs): add retry logic to pagination retrieval
jacob-a-brown Apr 29, 2026
6005f70
refactor(usgs): set datum in site transformer
jacob-a-brown Apr 29, 2026
dba8213
fix(usgs): return False if health check fails
jacob-a-brown Apr 29, 2026
b17bbde
fix(usgs): shorten error message
jacob-a-brown Apr 29, 2026
5570c36
fix(usgs): set limit of 1 for health check
jacob-a-brown Apr 29, 2026
836243e
fix(usgs): remove unused imports
jacob-a-brown Apr 29, 2026
4fd3ac6
fix(nmbgmr): retry when status_code!=200
jacob-a-brown Apr 29, 2026
1fcede3
fix(nmbgmr): remove print debugging statement
jacob-a-brown Apr 29, 2026
c1771d7
fix(nmbgmr): call self.warn not warning
jacob-a-brown Apr 29, 2026
a51b13e
fix(usgs): call self.warn not warning
jacob-a-brown Apr 29, 2026
43e2457
refactor(nmbgmr): change note style at top of doc
jacob-a-brown Apr 29, 2026
c17e806
refactor(retries): move rety logic to BaseSource
jacob-a-brown Apr 29, 2026
4a64c6f
refactor(nmbgmr): change timeout to 15 minutes
jacob-a-brown Apr 29, 2026
0e961a4
fix(nmbgmr): update developers note
jacob-a-brown Apr 30, 2026
8dd54d7
fix(source): return null string outside of while loop
jacob-a-brown Apr 30, 2026
32b6b25
fix(source): add URL to error message and log
jacob-a-brown Apr 30, 2026
31271c6
fix(usgs): remove hardcoded API key
jacob-a-brown Apr 30, 2026
deed95f
fix(usgs): have users pass USGS API keys
jacob-a-brown Apr 30, 2026
539fb13
fix(usgs): update README documentation
jacob-a-brown Apr 30, 2026
6897f28
fix(documentation): make README more readable
jacob-a-brown Apr 30, 2026
1d2bac9
fix(documentation): link to USGS API key acquisition
jacob-a-brown Apr 30, 2026
5f886bb
fix(usgs): exit if rate limit exceeded
jacob-a-brown Apr 30, 2026
f9908fd
fix(usgs): disallow partial records if a rate limit error is hit
jacob-a-brown Apr 30, 2026
7082957
fix(usgs): implement retries with exponential backoff
jacob-a-brown Apr 30, 2026
148208b
refactor(exceptions): put custom exceptions in own script
jacob-a-brown Apr 30, 2026
f760bbb
fix(usgs): catch JSONDecodeError and retry
jacob-a-brown Apr 30, 2026
1aff110
fix(usgs): check status_code for health check
jacob-a-brown Apr 30, 2026
0af670e
fix(usgs): revert persister.records on partial completion
jacob-a-brown Apr 30, 2026
690ba41
fix(partial records): don't persist partial records
jacob-a-brown Apr 30, 2026
7424177
fix(usgs): make site source USGS-NWIS to correspond with field measur…
jacob-a-brown Apr 30, 2026
013406d
feature(usgs): add dt params
jacob-a-brown Apr 30, 2026
7d38c37
note(usgs): type hint variables
jacob-a-brown Apr 30, 2026
8183300
test(usgs): use .env for api key for tests
jacob-a-brown Apr 30, 2026
9bf3fc2
fix(usgs): Handle USGS rate limit and partial/no data errors
jacob-a-brown Apr 30, 2026
43409f3
fix(test): don't override parent setup fixture
jacob-a-brown Apr 30, 2026
441ff81
fix(test): remove env var outside of fixture
jacob-a-brown Apr 30, 2026
e524ec7
fix(requirements): download python-dotenv
jacob-a-brown Apr 30, 2026
7af57d6
fix(test): load usgs api key from .env for every test
jacob-a-brown Apr 30, 2026
d7382bd
fix(unifier): remove partial persister.records on failure
jacob-a-brown Apr 30, 2026
245b01f
fix(usgs): use truthiness check for usgs api key header
jacob-a-brown Apr 30, 2026
f5b2e2f
fix(style): two blank lines between classes
jacob-a-brown Apr 30, 2026
85ab811
fix(usgs): fail health check on non 2xx status codes
jacob-a-brown Apr 30, 2026
5cf9904
test(usgs api key): test usgs api key is env var when set via --usgs-…
jacob-a-brown Apr 30, 2026
4f3b913
fix(test): reset usgs api key after each test
jacob-a-brown May 1, 2026
acd3360
Merge branch 'jab-nmbgmr-timeout-fix' into jab-usgs-fix
jacob-a-brown May 18, 2026
8bd3ab4
fix(source): add retry logic to _execute_json_request and _execute_te…
jacob-a-brown May 18, 2026
16201ba
fix(source): add retries and error handling for http requests
jacob-a-brown May 18, 2026
46c15f1
fix(usgs): check for HTTPError instead of HTTPStatusError to for heal…
jacob-a-brown May 18, 2026
3db9946
fix(unifier): Warn when unable to retrieve complete records after mul…
jacob-a-brown May 18, 2026
7f3cfa7
feat(unifier): Add more specific warning messages for incomplete records
jacob-a-brown May 18, 2026
60adf96
feat(usgs): add pagination check for water-level queries
jacob-a-brown May 18, 2026
22b0c33
fix(test): fix USGS API key handling in CLI tests
jacob-a-brown May 18, 2026
869e058
fix(logger): Handle potential file descriptor leaks by removing exist…
jacob-a-brown May 18, 2026
3a3f345
feat(error): log URL on error
jacob-a-brown May 18, 2026
ade753a
fix(tests): reset env var in USGS tests
jacob-a-brown May 18, 2026
e75be0a
fix(health): return False if exception is raised in health check
jacob-a-brown May 18, 2026
49abe13
fix(usgs): correct site collection error message
jacob-a-brown May 18, 2026
bab5252
fix(logger): Ensure only managed handlers are closed
jacob-a-brown May 18, 2026
eafa40f
fix(usgs): Add parameter code filter to USGS data retrieval if not si…
jacob-a-brown May 18, 2026
0c29a8a
feat(version): Update version to 0.10.1
jacob-a-brown May 18, 2026
3d6d41a
fix(usgs): update usgs parameter reader to enable None and number values
jacob-a-brown May 18, 2026
5a7c01f
fix(nmbgmr): Check for features in response to determine if API is wo…
jacob-a-brown May 18, 2026
982b72a
fix(usgs): make sure sites is always list
jacob-a-brown May 18, 2026
b9ac492
fix(health): update health checks to return false if get_records retu…
jacob-a-brown May 18, 2026
9560e0b
fix(st2 health): Update health checks to probe service directly
jacob-a-brown May 18, 2026
e40aa64
fix(usgs health check): decrease timeout for health check to avoid lo…
jacob-a-brown May 18, 2026
1d45571
fix(usgs): remove dead code for pagination
jacob-a-brown May 18, 2026
8b20e34
update(README): added note about USGS API key for die sites
jacob-a-brown May 18, 2026
0c685e1
fix(doc): update source _execute_json_request return type to always b…
jacob-a-brown May 18, 2026
074c17f
fix(documentation): remove repeated word in comment
jacob-a-brown May 18, 2026
45916b2
fix(README): use "for their APIs" instead of "to their APIs"
jacob-a-brown May 18, 2026
de660d8
fix(nmenv): list method hangs up service in health check
jacob-a-brown May 18, 2026
1e3d418
fix(readme): indicate api key for site and water level data from usgs
jacob-a-brown May 19, 2026
e880976
Merge pull request #66 from DataIntegrationGroup/jab-usgs-fix
jacob-a-brown May 19, 2026
c25d97b
feat(doc): log params when an error is encountered
jacob-a-brown May 19, 2026
ad71963
feat(setup): bump version to 0.10.2
jacob-a-brown May 19, 2026
fde585b
fix(persister): fix encoding issue when writing to file
jacob-a-brown May 26, 2026
98403be
feat(version): bump to 0.10.3
jacob-a-brown May 26, 2026
07f28c5
fix(persister): log errors when dumping files
jacob-a-brown May 26, 2026
e29d8ad
fix(persister): use utf-8 instead of utf-8-sig
jacob-a-brown May 26, 2026
746a722
fix(encoding): add error handling to timeseries dumps
jacob-a-brown May 26, 2026
183e8d8
fix(persister): use plain raise in except block
jacob-a-brown May 26, 2026
b06937a
fix(persister): remove trailing whitespace
jacob-a-brown May 26, 2026
17a1bee
fix(persister): include traceback in persister failure
jacob-a-brown May 26, 2026
ed4f371
fix(logger): accept kwargs for log and warn
jacob-a-brown May 26, 2026
2b1abe2
fix(persister): don't reassign variable
jacob-a-brown May 26, 2026
65104d8
feat(README): update README to include info on encoding
jacob-a-brown May 27, 2026
40a29d4
Merge pull request #68 from DataIntegrationGroup/jab-encoding-fix
jacob-a-brown May 27, 2026
8a7f434
feat: Dagster+GCP orchestration, uv migration, OGC Features output
jirhiker Jun 22, 2026
28671f5
spec: add §T.10-§T.15 composition refactor tasks + §V invariants
jirhiker Jun 22, 2026
a18ceaf
spec: add §10 backend improvements + §T.16-§T.26 performance/reliabil…
jirhiker Jun 22, 2026
ff33cdb
T10+T17: Loggable→make_logger factory; class-level polygon cache
jirhiker Jun 22, 2026
778b967
T19: replace all print() with structured logging in backend
jirhiker Jun 22, 2026
78b9072
T16: exponential backoff min(2^n,60)s; structured HTTP request logging
jirhiker Jun 22, 2026
f923603
T20: specific exceptions, coord range validation, fix BaseException→E…
jirhiker Jun 22, 2026
71617cc
T26: PersisterFactory extracted from Unifier
jirhiker Jun 22, 2026
33e4ae3
T11-T15,T18,T21-T25: composition refactor — eliminate inheritance, in…
jirhiker Jun 22, 2026
f844545
Add Dagster+ serverless code location + deploy paths
jirhiker Jun 24, 2026
c2ca243
Fix CI build: flake8 .venv exclude + resolve mypy type errors
jirhiker Jun 24, 2026
39a8f4c
Merge branch 'feature/orchestration-gcp-uv' into feature/dagster-plus…
jirhiker Jun 24, 2026
1cd29cb
Remove dead code: unused imports, functions, classes
jirhiker Jun 24, 2026
307119e
Merge pull request #70 from DataIntegrationGroup/feature/dagster-plus…
jirhiker Jun 24, 2026
6de6df3
Gate Dagster+ deploy on credential presence
jirhiker Jun 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[flake8]
# Don't lint the virtualenv or generated output dirs. extend-exclude keeps
# flake8's built-in defaults (.git, __pycache__, build, dist, ...).
extend-exclude =
.venv,
output,
output_1
41 changes: 12 additions & 29 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
@@ -1,56 +1,39 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: CI/CD

on:
push:
branches: [ "pre-production", "feature/jir", "dev/jab"]
branches: [ "pre-production", "feature/jir", "dev/jab", "feature/orchestration-gcp-uv" ]
pull_request:
branches: [ "pre-production"]
branches: [ "pre-production" ]

permissions:
contents: read

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Set up Python 3.10
uses: actions/setup-python@v4
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
python-version: "3.10"
cache: "pip"
version: "latest"

- name: Cache pip dependencies
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Set up Python
run: uv python install 3.10

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 pytest mypy
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
run: uv sync --extra dev

- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
uv run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
uv run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

- name: Type check with mypy
run: |
mypy . --ignore-missing-imports
run: uv run mypy . --ignore-missing-imports

- name: Test with pytest
run: |
pytest -s tests
run: uv run pytest -s tests
86 changes: 86 additions & 0 deletions .github/workflows/dagster-cloud-branch-deployments.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
name: Dagster+ Serverless Branch Deployments

on:
pull_request:
types: [opened, synchronize, reopened, closed]

concurrency:
group: ${{ github.ref }}/branch_deployments
cancel-in-progress: true

env:
DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
ENABLE_FAST_DEPLOYS: 'true'
PYTHON_VERSION: '3.10'
DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'

jobs:
dagster_cloud_default_deploy:
name: Dagster Serverless Deploy
runs-on: ubuntu-22.04
outputs:
build_info: ${{ steps.parse-workspace.outputs.build_info }}
steps:
- name: Check Dagster+ credentials present
id: guard
run: |
if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then
echo "enabled=true" >> "$GITHUB_OUTPUT"
else
echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy."
echo "enabled=false" >> "$GITHUB_OUTPUT"
fi

- name: Prerun Checks
id: prerun
if: steps.guard.outputs.enabled == 'true'
uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1

- name: Launch Docker Deploy
if: steps.prerun.outputs.result == 'docker-deploy'
id: parse-workspace
uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
with:
dagster_cloud_file: $DAGSTER_CLOUD_FILE

- name: Checkout for Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: actions/checkout@v6
with:
ref: ${{ github.head_ref }}
path: project-repo

- name: Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
with:
dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
build_output_dir: "$GITHUB_WORKSPACE/build"
python_version: "${{ env.PYTHON_VERSION }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dagster_cloud_docker_deploy:
name: Docker Deploy
runs-on: ubuntu-22.04
if: needs.dagster_cloud_default_deploy.outputs.build_info
needs: dagster_cloud_default_deploy
strategy:
fail-fast: false
matrix:
location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.head_ref }}
- name: Build and deploy to Dagster+ serverless
uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1
with:
dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
location: ${{ toJson(matrix.location) }}
base_image: "python:${{ env.PYTHON_VERSION }}-slim"
organization_id: ${{ secrets.ORGANIZATION_ID }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
89 changes: 89 additions & 0 deletions .github/workflows/dagster-cloud-deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: Dagster+ Serverless Deploy

on:
push:
branches:
- "main"

concurrency:
group: ${{ github.ref }}/deploy
cancel-in-progress: true

env:
DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
# PEX fast deploy. No system GDAL needed (only shapely, which ships a wheel
# with bundled GEOS), so a Docker build would be pure overhead.
ENABLE_FAST_DEPLOYS: 'true'
PYTHON_VERSION: '3.10'
DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'

jobs:
dagster_cloud_default_deploy:
name: Dagster Serverless Deploy
runs-on: ubuntu-22.04
outputs:
build_info: ${{ steps.parse-workspace.outputs.build_info }}
steps:
- name: Check Dagster+ credentials present
id: guard
run: |
if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then
echo "enabled=true" >> "$GITHUB_OUTPUT"
else
echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy."
echo "enabled=false" >> "$GITHUB_OUTPUT"
fi

- name: Prerun Checks
id: prerun
if: steps.guard.outputs.enabled == 'true'
uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1

- name: Launch Docker Deploy
if: steps.prerun.outputs.result == 'docker-deploy'
id: parse-workspace
uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
with:
dagster_cloud_file: $DAGSTER_CLOUD_FILE

- name: Checkout for Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: actions/checkout@v6
with:
ref: ${{ github.head_ref }}
path: project-repo

- name: Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
with:
dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
build_output_dir: "$GITHUB_WORKSPACE/build"
python_version: "${{ env.PYTHON_VERSION }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dagster_cloud_docker_deploy:
name: Docker Deploy
runs-on: ubuntu-22.04
if: needs.dagster_cloud_default_deploy.outputs.build_info
needs: dagster_cloud_default_deploy
strategy:
fail-fast: false
matrix:
location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.head_ref }}
- name: Build and deploy to Dagster+ serverless
uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1
with:
dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
location: ${{ toJson(matrix.location) }}
base_image: "python:${{ env.PYTHON_VERSION }}-slim"
organization_id: ${{ secrets.ORGANIZATION_ID }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
49 changes: 49 additions & 0 deletions .github/workflows/orchestration-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: Orchestration CI

on:
push:
paths:
- 'orchestration/**'
- 'backend/**'
pull_request:
branches: [ "pre-production" ]
paths:
- 'orchestration/**'

permissions:
contents: read

jobs:
lint-and-import-check:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Install uv
uses: astral-sh/setup-uv@v4
with:
version: "latest"

- name: Set up Python
run: uv python install 3.10

- name: Install root package deps
run: uv sync --extra dev

- name: Install orchestration deps
run: uv pip install dagster dagster-gcp google-cloud-storage google-cloud-secret-manager Jinja2

- name: Lint orchestration with flake8
run: |
uv run flake8 orchestration/ --count --select=E9,F63,F7,F82 --show-source --statistics
uv run flake8 orchestration/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

- name: Import check — definitions load without error
run: |
uv run python -c "
import sys; sys.path.insert(0, '.')
from orchestration.definitions import defs
assert len(list(defs.assets)) > 0, 'No assets defined'
print(f'OK: {len(list(defs.assets))} assets, {len(defs.schedules)} schedules')
"
6 changes: 3 additions & 3 deletions .github/workflows/publish-to-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
id-token: write
steps:
- uses: actions/checkout@master
- name: Set up Python 3.9
uses: actions/setup-python@v1
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: 3.9
python-version: 3.13

- name: Install pypa/build
run: >-
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ frontend/requirements.txt
frontend/output_timeseries
backend/test00112233.csv
frontend/api/cache
.DS_Store

# Byte-compiled / optimized / DLL files
__pycache__/
Expand Down
Loading
Loading