Add Dagster+ serverless code location + deploy paths

.github/workflows/dagster-cloud-branch-deployments.yml

@@ -0,0 +1,75 @@
+name: Dagster+ Serverless Branch Deployments
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
+
+concurrency:
+  group: ${{ github.ref }}/branch_deployments
+  cancel-in-progress: true
+
+env:
+  DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
+  DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+  ENABLE_FAST_DEPLOYS: 'true'
+  PYTHON_VERSION: '3.10'
+  DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
+
+jobs:
+  dagster_cloud_default_deploy:
+    name: Dagster Serverless Deploy
+    runs-on: ubuntu-22.04
+    outputs:
+      build_info: ${{ steps.parse-workspace.outputs.build_info }}
+    steps:
+      - name: Prerun Checks
+        id: prerun
+        uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1
+
+      - name: Launch Docker Deploy
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        id: parse-workspace
+        uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
+        with:
+          dagster_cloud_file: $DAGSTER_CLOUD_FILE
+
+      - name: Checkout for Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+          path: project-repo
+
+      - name: Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        with:
+          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
+          build_output_dir: "$GITHUB_WORKSPACE/build"
+          python_version: "${{ env.PYTHON_VERSION }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  dagster_cloud_docker_deploy:
+    name: Docker Deploy
+    runs-on: ubuntu-22.04
+    if: needs.dagster_cloud_default_deploy.outputs.build_info
+    needs: dagster_cloud_default_deploy
+    strategy:
+      fail-fast: false
+      matrix:
+        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Build and deploy to Dagster+ serverless
+        uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1
+        with:
+          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+          location: ${{ toJson(matrix.location) }}
+          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
+          organization_id: ${{ secrets.ORGANIZATION_ID }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dagster-cloud-deploy.yml

@@ -0,0 +1,78 @@
+name: Dagster+ Serverless Deploy
+
+on:
+  push:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.ref }}/deploy
+  cancel-in-progress: true
+
+env:
+  DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
+  DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+  # PEX fast deploy. No system GDAL needed (only shapely, which ships a wheel
+  # with bundled GEOS), so a Docker build would be pure overhead.
+  ENABLE_FAST_DEPLOYS: 'true'
+  PYTHON_VERSION: '3.10'
+  DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
+
+jobs:
+  dagster_cloud_default_deploy:
+    name: Dagster Serverless Deploy
+    runs-on: ubuntu-22.04
+    outputs:
+      build_info: ${{ steps.parse-workspace.outputs.build_info }}
+    steps:
+      - name: Prerun Checks
+        id: prerun
+        uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1
+
+      - name: Launch Docker Deploy
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        id: parse-workspace
+        uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1
+        with:
+          dagster_cloud_file: $DAGSTER_CLOUD_FILE
+
+      - name: Checkout for Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+          path: project-repo
+
+      - name: Python Executable Deploy
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        with:
+          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
+          build_output_dir: "$GITHUB_WORKSPACE/build"
+          python_version: "${{ env.PYTHON_VERSION }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  dagster_cloud_docker_deploy:
+    name: Docker Deploy
+    runs-on: ubuntu-22.04
+    if: needs.dagster_cloud_default_deploy.outputs.build_info
+    needs: dagster_cloud_default_deploy
+    strategy:
+      fail-fast: false
+      matrix:
+        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Build and deploy to Dagster+ serverless
+        uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1
+        with:
+          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
+          location: ${{ toJson(matrix.location) }}
+          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
+          organization_id: ${{ secrets.ORGANIZATION_ID }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

backend/bounding_polygons.py

@@ -15,113 +15,14 @@
 # ===============================================================================
 import json
 import os
-from pprint import pprint
 
 import click
 import httpx
-from shapely import Polygon, box
 from shapely.geometry import shape
 
 from backend.geo_utils import transform_srid, SRID_WGS84, SRID_UTM_ZONE_13N
 
 
-# polygon retrivial functions
-# multiple polygons
-def get_congressional_district_boundaries(state, district):
-    pass
-
-
-def get_tribal_boundaries(state=None):
-    state, statefp = _get_statefp(state)
-
-    # use the processes service to get all tribal boundaries that intersect the state
-    def func():
-        payload = {
-            "inputs": {
-                "collection": f"aiannh",
-                "url": f"https://geoconnex.us/ref/states/{statefp}",
-            }
-        }
-        resp = httpx.post(
-            "https://reference.geoconnex.us/processes/intersector/execution",
-            json=payload,
-        )
-        return resp.json()
-
-    obj = _get_cached_object(f"{state}.aiannh", f"{state} AIANNH", func)
-
-    return obj
-
-
-def get_state_hucs_boundaries(state=None, level=8):
-    state, statefp = _get_statefp(state)
-
-    # use the processes service to get all hucs from this level that intersect the state of NM
-    def func():
-        payload = {
-            "inputs": {
-                "collection": f"hu{level:02n}",
-                "url": f"https://geoconnex.us/ref/states/{statefp}",
-            }
-        }
-        resp = httpx.post(
-            "https://reference.geoconnex.us/processes/intersector/execution",
-            json=payload,
-        )
-        return resp.json()
-
-    obj = _get_cached_object(f"{state}.hucs.{level}", f"{state} HU{level:02n}", func)
-
-    return obj
-
-
-def get_state_pwss_boundaries(state=None):
-    state, statefp = _get_statefp(state)
-    obj = _get_cached_object(
-        f"{state}.pws",
-        f"{state} PWSs",
-        f"https://reference.geoconnex.us/collections/pws/items?f=json&state_code={state}",
-    )
-
-    return obj
-
-
-# single polygons
-
-
-def get_pws_polygon(pwsid, as_wkt=True):
-    obj = _get_cached_object(
-        pwsid,
-        pwsid,
-        f"https://reference.geoconnex.us/collections/pws/items/{pwsid}?f=json",
-    )
-    return _make_shape(obj, as_wkt)
-
-
-def get_huc_polygon(huc, as_wkt=True):
-    if len(huc) == 2:
-        collection = "hu02"
-    elif len(huc) == 4:
-        collection = "hu04"
-    elif len(huc) == 6:
-        collection = "hu06"
-    elif len(huc) == 8:
-        collection = "hu08"
-    elif len(huc) == 10:
-        collection = "hu10"
-    else:
-        _warning(f"Invalid HUC {huc}. length must be 2, 4, 6, 8, or 10")
-        return
-
-    obj = _get_cached_object(
-        huc,
-        huc,
-        f"https://reference.geoconnex.us/collections/{collection}/items/{huc}?f=json",
-    )
-
-    return _make_shape(obj, as_wkt)
-
-
 def get_county_polygon(name, as_wkt=True):
     if ":" in name:
         state, county = name.split(":")

backend/config.py

@@ -16,7 +16,6 @@
 import os
 import sys
 from datetime import datetime, timedelta
-from enum import Enum
 import shapely.wkt
 import yaml
 

backend/connectors/bor/source.py

@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-import pprint
-from json import JSONDecodeError
 
-import httpx
 
 from backend.connectors.bor.transformer import BORSiteTransformer, BORAnalyteTransformer
 from backend.connectors.mappings import BOR_ANALYTE_MAPPING
@@ -27,12 +24,9 @@
     SOURCE_PARAMETER_NAME,
     SOURCE_PARAMETER_UNITS,
     DT_MEASURED,
-    EARLIEST,
-    LATEST,
 )
 
 from backend.source import (
-    BaseSource,
     BaseSiteSource,
     BaseAnalyteSource,
     get_terminal_record,

backend/connectors/bor/transformer.py

@@ -13,13 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-import pprint
-import json
 
-from backend.record import SiteRecord
 from backend.transformer import (
-    BaseTransformer,
-    WaterLevelTransformer,
     SiteTransformer,
     AnalyteTransformer,
 )

backend/connectors/ckan/source.py

@@ -15,7 +15,6 @@
 # ===============================================================================
 from itertools import groupby
 
-import httpx
 
 from backend.connectors import (
     OSE_ROSWELL_HONDO_BOUNDING_POLYGON,
@@ -34,7 +33,6 @@
 from backend.constants import (
     FEET,
     DTW,
-    DTW_UNITS,
     DT_MEASURED,
     PARAMETER_NAME,
     PARAMETER_UNITS,
@@ -43,7 +41,6 @@
     SOURCE_PARAMETER_UNITS,
 )
 from backend.source import (
-    BaseSource,
     BaseSiteSource,
     BaseWaterLevelSource,
     get_terminal_record,

backend/connectors/ckan/transformer.py

@@ -13,15 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-import pprint
 
-from backend.record import SiteRecord
-from backend.transformer import BaseTransformer, WaterLevelTransformer, SiteTransformer
+from backend.transformer import WaterLevelTransformer, SiteTransformer
 
 
 class OSERoswellSiteTransformer(SiteTransformer):
     def _transform(self, record):
-        # pprint.pprint(record)
         lat = float(record["DD_lat"])
         lng = float(record["DD_lon"])
         # if not self.contained(lng, lat):

backend/connectors/isc_seven_rivers/source.py

@@ -15,7 +15,6 @@
 # ===============================================================================
 from datetime import datetime
 
-import httpx
 
 from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON
 from backend.connectors.mappings import ISC_SEVEN_RIVERS_ANALYTE_MAPPING
@@ -28,16 +27,13 @@
     PARAMETER_UNITS,
     SOURCE_PARAMETER_NAME,
     SOURCE_PARAMETER_UNITS,
-    EARLIEST,
-    LATEST,
 )
 from backend.connectors.isc_seven_rivers.transformer import (
     ISCSevenRiversSiteTransformer,
     ISCSevenRiversWaterLevelTransformer,
     ISCSevenRiversAnalyteTransformer,
 )
 from backend.source import (
-    BaseSource,
     BaseSiteSource,
     BaseWaterLevelSource,
     BaseAnalyteSource,

backend/connectors/isc_seven_rivers/transformer.py

@@ -13,12 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-import shapely.wkt
-from shapely import Point
 
-from backend.record import SiteRecord
 from backend.transformer import (
-    BaseTransformer,
     WaterLevelTransformer,
     SiteTransformer,
     AnalyteTransformer,