Bdms 227 jab updates to pass tests

.github/workflows/tests.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Tests
+name: Test Suite
 
 on:
   pull_request:
@@ -16,11 +16,8 @@ jobs:
 
     services:
       postgis:
-#        image: ghcr.io/dataintegrationgroup/nmdms:latest
-#        image: postgres
         image: postgis/postgis:latest
-#        image: postgis/postgis:17-3.5
-#        image: timescale/timescaledb:2.18.0-pg17
+        # image: postgis/postgis:17-3.5
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -36,11 +33,6 @@ jobs:
     - name: Check out source repository
       uses: actions/checkout@v4
 
-#    - name: Install SpatiaLite
-#      run: |
-#          sudo apt-get update
-#          sudo apt-get install -y libsqlite3-mod-spatialite libspatialite-dev
-
     - name: Install uv
       uses: astral-sh/setup-uv@v5
       with:
@@ -62,26 +54,38 @@ jobs:
         POSTGRES_USER: postgres
         POSTGRES_PASSWORD: postgres
         DB_DRIVER: postgres
-#        SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so
 
       run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml
 
-#    - name: Checkout BDD repo (features only)
-#      uses: actions/checkout@v4
-#      with:
-#        repository: DataIntegrationGroup/OcotilloBDD
-#        path: bdd
-#
-#    - name: Copy BDD features into backend test directory
-#      run: |
-#        mkdir -p tests/features
-#        cp -r bdd/features/backend/* tests/features/
-#
-#    - name: Run BDD tests
-#      env:
-#        BASE_URL: ${{ secrets.BACKEND_URL }}
-#      run: |
-#        uv run behave tests/features --tags=@backend,@approved --no-capture
+    - name: Checkout BDD repo (features only)
+      uses: actions/checkout@v4
+      with:
+        repository: DataIntegrationGroup/OcotilloBDD
+        path: bdd
+
+    - name: Copy BDD features into backend test directory
+      run: |
+        mkdir -p tests/features
+        cp -r bdd/features/backend/* tests/features/
+
+    - name: Run BDD tests
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+        DB_DRIVER: postgres
+        BASE_URL: http://localhost:8000
+      run: |
+        uv run behave tests/features --tags="@backend and @production" --no-capture
+#        uv run behave tests/features/transducer-data-response.feature \
+#             tests/features/thing-type-path-parameters.feature \
+#             tests/features/thing-query-parameters.feature \
+#             tests/features/well-notes.feature \
+#             tests/features/location-notes.feature \
+#             tests/features/geojson-response.feature
+#        use this when we have consensus on tag nomenclature
+#        uv run behave tests/features --tags="@backend and @production" --no-capture
 
     - name: Upload results to Codecov
       uses: codecov/codecov-action@v4

.gitignore

@@ -32,6 +32,7 @@ transfers/data/nma_csv_cache/*
 tests/features/*.feature
 transfers/metrics/*
 transfers/logs/*
+run_bdd-local.sh
 
 
 # deployment files

.pre-commit-config.yaml

@@ -16,15 +16,15 @@ repos:
           '--statistics'
           ]
         exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly
-  - repo: local
-    hooks:
-      - id: pytest
-        name: pytest
-        entry: pytest  # Or your specific test command, e.g., poetry run pytest
-        language: system
-        types: [python] # Specify relevant file types for your tests
-        pass_filenames: false
-        always_run: true
+  # - repo: local
+  #   hooks:
+  #     - id: pytest
+  #       name: pytest
+  #       entry: pytest  # Or your specific test command, e.g., poetry run pytest
+  #       language: system
+  #       types: [python] # Specify relevant file types for your tests
+  #       pass_filenames: false
+  #       always_run: true
 
   # - repo: https://github.com/pre-commit/mirrors-mypy
   #   rev: v1.10.0  # Use the latest stable version or pin to your preference

api/observation.py

@@ -25,7 +25,7 @@
     amp_editor_dependency,
     amp_viewer_dependency,
 )
-from db import Observation
+from db import Observation, Parameter
 from schemas.observation import (
     CreateGroundwaterLevelObservation,
     GroundwaterLevelObservationResponse,
@@ -113,8 +113,6 @@ async def update_water_chemistry_observation(
 
 
 # ============= Get ==============================================
-
-
 @router.get(
     "/transducer-groundwater-level",
     summary="Get transducer groundwater level observations",
@@ -124,12 +122,19 @@ async def get_transducer_groundwater_level_observations(
     session: session_dependency,
     user: amp_viewer_dependency,
     thing_id: int | None = None,
-    parameter_id: int | None = None,
     start_time: datetime | None = None,
     end_time: datetime | None = None,
 ) -> CustomPage[TransducerObservationWithBlockResponse]:
+
+    groundwater_parameter_id = (
+        session.query(Parameter)
+        .filter(Parameter.parameter_name == "groundwater level")
+        .one()
+        .id
+    )
+
     return get_transducer_observations(
-        session, thing_id, parameter_id, start_time, end_time
+        session, thing_id, groundwater_parameter_id, start_time, end_time
     )
 
 

core/app.py

@@ -24,7 +24,10 @@
 )
 from fastapi.openapi.utils import get_openapi
 
-from .initializers import init_db, init_lexicon, init_parameter, register_routes
+from .initializers import (
+    register_routes,
+    erase_and_rebuild_db,
+)
 from .settings import settings
 
 
@@ -34,9 +37,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     Application lifespan event handler to initialize the database and lexicon.
     """
     if settings.get_enum("MODE") == "development":
-        init_db()
-        init_lexicon()
-        init_parameter()
+        erase_and_rebuild_db()
 
     register_routes(app)
     yield

core/enums.py

@@ -24,7 +24,9 @@
 )
 CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material")
 CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method")
-ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method")
+WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category(
+    "well_construction_method"
+)
 ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type")
 CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method")
 WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose")
@@ -48,6 +50,7 @@
 MonitoringStatus: type[Enum] = build_enum_from_lexicon_category("monitoring_status")
 ParameterName: type[Enum] = build_enum_from_lexicon_category("parameter_name")
 Organization: type[Enum] = build_enum_from_lexicon_category("organization")
+OriginSource: type[Enum] = build_enum_from_lexicon_category("origin_source")
 ParameterType: type[Enum] = build_enum_from_lexicon_category("parameter_type")
 PhoneType: type[Enum] = build_enum_from_lexicon_category("phone_type")
 PublicationType: type[Enum] = build_enum_from_lexicon_category("publication_type")
@@ -67,4 +70,10 @@
 Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum")
 ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type")
 SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type")
+WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type")
+PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type")
+GroupType: type[Enum] = build_enum_from_lexicon_category("group_type")
+MonitoringFrequency: type[Enum] = build_enum_from_lexicon_category(
+    "monitoring_frequency"
+)
 # ============= EOF =============================================

core/initializers.py

@@ -18,49 +18,13 @@
 from fastapi_pagination import add_pagination
 from sqlalchemy import text
 from sqlalchemy.exc import DatabaseError
-from sqlalchemy.orm import Session
 
 from db import Base
-from db.engine import engine, session_ctx
+from db.engine import session_ctx
 from db.parameter import Parameter
 from services.lexicon_helper import add_lexicon_term, add_lexicon_category
 
 
-# ============= EOF =============================================
-def init_db():
-    """
-    Initialize the database by creating all tables.
-    This function is called during application startup.
-    """
-
-    from sqlalchemy import text
-
-    with engine.connect() as conn:
-        conn.execute(text("DROP SCHEMA public CASCADE"))
-        conn.execute(text("CREATE SCHEMA public"))
-        conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
-        conn.commit()
-
-    Base.metadata.drop_all(engine)
-    Base.metadata.create_all(engine)
-
-
-def init_hypertables():
-    """
-    Initialize hypertables for time-series data.
-    This function is called during application startup.
-    """
-    # session = next(get_db_session())
-    # Create hypertables for time-series data
-    with session_ctx() as session:
-        session.execute(
-            text("select create_hypertable('observation', 'observation_datetime');")
-        )
-
-    # session.commit()
-    # session.close()
-
-
 def init_parameter(path: str = None) -> None:
     """
     Populate the parameter table to allow their use in creating and editing
@@ -91,17 +55,17 @@ def init_parameter(path: str = None) -> None:
                 session.rollback()
 
 
-def erase_and_rebuild_db(session: Session):
-    from sqlalchemy import text
-
-    with session.bind.connect() as conn:
-        conn.execute(text("DROP SCHEMA public CASCADE"))
-        conn.execute(text("CREATE SCHEMA public"))
-        conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
-        conn.commit()
+def erase_and_rebuild_db():
+    with session_ctx() as session:
+        session.execute(text("DROP SCHEMA public CASCADE"))
+        session.execute(text("CREATE SCHEMA public"))
+        session.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
+        session.commit()
+        Base.metadata.drop_all(session.bind)
+        Base.metadata.create_all(session.bind)
 
-    Base.metadata.drop_all(session.bind)
-    Base.metadata.create_all(session.bind)
+    init_lexicon()
+    init_parameter()
 
 
 def init_lexicon(path: str = None) -> None:
@@ -174,3 +138,6 @@ def register_routes(app):
     app.include_router(search_router)
     app.include_router(thing_router)
     add_pagination(app)
+
+
+# ============= EOF =============================================