feat: add transducer observation handling and improve database initialization

.github/workflows/tests.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Tests
+name: Test Suite
 
 on:
   pull_request:
@@ -16,11 +16,8 @@ jobs:
 
     services:
       postgis:
-#        image: ghcr.io/dataintegrationgroup/nmdms:latest
-#        image: postgres
         image: postgis/postgis:latest
-#        image: postgis/postgis:17-3.5
-#        image: timescale/timescaledb:2.18.0-pg17
+        # image: postgis/postgis:17-3.5
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -36,11 +33,6 @@ jobs:
     - name: Check out source repository
       uses: actions/checkout@v4
 
-#    - name: Install SpatiaLite
-#      run: |
-#          sudo apt-get update
-#          sudo apt-get install -y libsqlite3-mod-spatialite libspatialite-dev
-
     - name: Install uv
       uses: astral-sh/setup-uv@v5
       with:
@@ -62,26 +54,34 @@ jobs:
         POSTGRES_USER: postgres
         POSTGRES_PASSWORD: postgres
         DB_DRIVER: postgres
-#        SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so
 
       run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml
 
-#    - name: Checkout BDD repo (features only)
-#      uses: actions/checkout@v4
-#      with:
-#        repository: DataIntegrationGroup/OcotilloBDD
-#        path: bdd
-#
-#    - name: Copy BDD features into backend test directory
-#      run: |
-#        mkdir -p tests/features
-#        cp -r bdd/features/backend/* tests/features/
-#
-#    - name: Run BDD tests
-#      env:
-#        BASE_URL: ${{ secrets.BACKEND_URL }}
-#      run: |
-#        uv run behave tests/features --tags=@backend,@approved --no-capture
+    - name: Checkout BDD repo (features only)
+      uses: actions/checkout@v4
+      with:
+        repository: DataIntegrationGroup/OcotilloBDD
+        path: bdd
+
+    - name: Copy BDD features into backend test directory
+      run: |
+        mkdir -p tests/features
+        cp -r bdd/features/backend/* tests/features/
+
+    - name: Run BDD tests
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+        DB_DRIVER: postgres
+        BASE_URL: http://localhost:8000
+      run: |
+        uv run behave tests/features/transducer-data-response.feature \
+             tests/features/thing-type-path-parameters.feature \
+             tests/features/thing-query-parameters.feature
+#        use this when we have consensus on tag nomenclature
+#        uv run behave tests/features --tags="@backend and @production" --no-capture
 
     - name: Upload results to Codecov
       uses: codecov/codecov-action@v4

api/observation.py

@@ -25,7 +25,7 @@
     amp_editor_dependency,
     amp_viewer_dependency,
 )
-from db import Observation
+from db import Observation, Parameter
 from schemas.observation import (
     CreateGroundwaterLevelObservation,
     GroundwaterLevelObservationResponse,
@@ -113,8 +113,6 @@ async def update_water_chemistry_observation(
 
 
 # ============= Get ==============================================
-
-
 @router.get(
     "/transducer-groundwater-level",
     summary="Get transducer groundwater level observations",
@@ -124,12 +122,19 @@ async def get_transducer_groundwater_level_observations(
     session: session_dependency,
     user: amp_viewer_dependency,
     thing_id: int | None = None,
-    parameter_id: int | None = None,
     start_time: datetime | None = None,
     end_time: datetime | None = None,
 ) -> CustomPage[TransducerObservationWithBlockResponse]:
+
+    groundwater_parameter_id = (
+        session.query(Parameter)
+        .filter(Parameter.parameter_name == "groundwater level")
+        .one()
+        .id
+    )
+
     return get_transducer_observations(
-        session, thing_id, parameter_id, start_time, end_time
+        session, thing_id, groundwater_parameter_id, start_time, end_time
     )
 
 

core/app.py

@@ -24,7 +24,12 @@
 )
 from fastapi.openapi.utils import get_openapi
 
-from .initializers import init_db, init_lexicon, init_parameter, register_routes
+from .initializers import (
+    init_lexicon,
+    init_parameter,
+    register_routes,
+    erase_and_rebuild_db,
+)
 from .settings import settings
 
 
@@ -34,7 +39,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     Application lifespan event handler to initialize the database and lexicon.
     """
     if settings.get_enum("MODE") == "development":
-        init_db()
+        erase_and_rebuild_db()
         init_lexicon()
         init_parameter()
 

core/initializers.py

@@ -21,46 +21,11 @@
 from sqlalchemy.orm import Session
 
 from db import Base
-from db.engine import engine, session_ctx
+from db.engine import session_ctx
 from db.parameter import Parameter
 from services.lexicon_helper import add_lexicon_term, add_lexicon_category
 
 
-# ============= EOF =============================================
-def init_db():
-    """
-    Initialize the database by creating all tables.
-    This function is called during application startup.
-    """
-
-    from sqlalchemy import text
-
-    with engine.connect() as conn:
-        conn.execute(text("DROP SCHEMA public CASCADE"))
-        conn.execute(text("CREATE SCHEMA public"))
-        conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
-        conn.commit()
-
-    Base.metadata.drop_all(engine)
-    Base.metadata.create_all(engine)
-
-
-def init_hypertables():
-    """
-    Initialize hypertables for time-series data.
-    This function is called during application startup.
-    """
-    # session = next(get_db_session())
-    # Create hypertables for time-series data
-    with session_ctx() as session:
-        session.execute(
-            text("select create_hypertable('observation', 'observation_datetime');")
-        )
-
-    # session.commit()
-    # session.close()
-
-
 def init_parameter(path: str = None) -> None:
     """
     Populate the parameter table to allow their use in creating and editing
@@ -92,14 +57,10 @@ def init_parameter(path: str = None) -> None:
 
 
 def erase_and_rebuild_db(session: Session):
-    from sqlalchemy import text
-
-    with session.bind.connect() as conn:
-        conn.execute(text("DROP SCHEMA public CASCADE"))
-        conn.execute(text("CREATE SCHEMA public"))
-        conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
-        conn.commit()
-
+    session.execute(text("DROP SCHEMA public CASCADE"))
+    session.execute(text("CREATE SCHEMA public"))
+    session.execute(text("CREATE EXTENSION IF NOT EXISTS postgis"))
+    session.commit()
     Base.metadata.drop_all(session.bind)
     Base.metadata.create_all(session.bind)
 
@@ -174,3 +135,6 @@ def register_routes(app):
     app.include_router(search_router)
     app.include_router(thing_router)
     add_pagination(app)
+
+
+# ============= EOF =============================================

run_bdd.sh

@@ -59,7 +59,13 @@ export BASE_URL=${BASE_URL:-http://localhost:8000}
 #uv run behave tests/features --tags=@backend
 #uv run behave tests/features/sensor-notes.feature --tags=@backend
 
+uv run behave tests/features/transducer-data-response.feature
+
+#uv run behave tests/features/transducer-data-response.feature \
+#  tests/features/thing-type-path-parameters.feature \
+#  tests/features/thing-query-parameters.feature
+
+#uv run behave tests/features/well-inventory-csv.feature
 
-uv run behave tests/features --tags=@backend --tags=@production
 
 echo "✅ BDD test run complete."

services/observation_helper.py

@@ -52,6 +52,12 @@ def get_transducer_observations(
     order: str | None = None,
     filter_: str = Query(alias="filter", default=None),
 ):
+    if thing_id:
+        item = session.get(Thing, thing_id)
+        if item is None:
+            empty_query = select(TransducerObservation).where(False)
+            return paginate(query=empty_query, conn=session)
+
     # Subquery to get latest block for each observation
     block_subq = (
         select(TransducerObservationBlock.id)

tests/features/environment.py

@@ -24,12 +24,12 @@
     Group,
     GroupThingAssociation,
     Sensor,
-    LexiconTerm,
     TransducerObservation,
     Parameter,
     Deployment,
     TransducerObservationBlock,
 )
+
 from db.engine import session_ctx
 
 
@@ -187,12 +187,24 @@ def add_block(context, session, parameter):
     return block
 
 
+@add_context_object_container("transducer_observations")
+def add_transducer_observation(context, session, block, deployment_id, value):
+    obs = TransducerObservation(
+        parameter_id=block.parameter_id,
+        deployment_id=deployment_id,
+        observation_datetime=datetime.now(),
+        value=value,
+    )
+    session.add(obs)
+    context.objects["transducer_observations"].append(obs)
+    return obs
+
+
 def before_all(context):
     context.objects = {}
-
-    force = False
+    rebuild = False
     with session_ctx() as session:
-        if session.query(LexiconTerm).count() == 0 or force:
+        if rebuild:
             erase_and_rebuild_db(session)
             init_lexicon()
             init_parameter()
@@ -211,16 +223,12 @@ def before_all(context):
 
         # parameter ID can be hardcoded because init_parameter always creates the same one
         parameter = session.get(Parameter, 1)
-        add_obs = add_block(context, session, parameter)
-        if add_obs:
-            for i in range(1, 10):
-                obs = TransducerObservation(
-                    parameter_id=parameter.id,
-                    deployment_id=deployment.id,
-                    observation_datetime=datetime.now(),
-                    value=random.random(),
-                )
-                session.add(obs)
+        block = add_block(context, session, parameter)
+        for i in range(1, 10):
+            add_transducer_observation(
+                context, session, block, deployment.id, random.random()
+            )
+
         session.commit()
 
 

tests/features/steps/common.py

@@ -102,4 +102,12 @@ def step_impl(context):
     ), f"Unexpected response type {context.response.headers['Content-Type']}"
 
 
+@then("the items should be an empty list")
+def step_impl(context):
+    data = context.response.json()
+    assert len(data["items"]) == 0, f'Unexpected items {data["items"]}'
+    assert data["total"] == 0, f'Unexpected total {data["total"]}'
+    assert data["page"] == 1, f'Unexpected page {data["page"]}'
+
+
 # ============= EOF =============================================