diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b71716d6e..5098607b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Tests +name: Test Suite on: pull_request: @@ -16,11 +16,8 @@ jobs: services: postgis: -# image: ghcr.io/dataintegrationgroup/nmdms:latest -# image: postgres image: postgis/postgis:latest -# image: postgis/postgis:17-3.5 -# image: timescale/timescaledb:2.18.0-pg17 + # image: postgis/postgis:17-3.5 env: POSTGRES_PASSWORD: postgres options: >- @@ -36,11 +33,6 @@ jobs: - name: Check out source repository uses: actions/checkout@v4 -# - name: Install SpatiaLite -# run: | -# sudo apt-get update -# sudo apt-get install -y libsqlite3-mod-spatialite libspatialite-dev - - name: Install uv uses: astral-sh/setup-uv@v5 with: @@ -62,26 +54,34 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres DB_DRIVER: postgres -# SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml -# - name: Checkout BDD repo (features only) -# uses: actions/checkout@v4 -# with: -# repository: DataIntegrationGroup/OcotilloBDD -# path: bdd -# -# - name: Copy BDD features into backend test directory -# run: | -# mkdir -p tests/features -# cp -r bdd/features/backend/* tests/features/ -# -# - name: Run BDD tests -# env: -# BASE_URL: ${{ secrets.BACKEND_URL }} -# run: | -# uv run behave tests/features --tags=@backend,@approved --no-capture + - name: Checkout BDD repo (features only) + uses: actions/checkout@v4 + with: + repository: DataIntegrationGroup/OcotilloBDD + path: bdd + + - name: Copy BDD features into backend test directory + run: | + mkdir -p tests/features + cp -r bdd/features/backend/* tests/features/ + + - name: Run BDD tests + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + DB_DRIVER: postgres + BASE_URL: http://localhost:8000 + run: | + uv run behave tests/features/transducer-data-response.feature \ + tests/features/thing-type-path-parameters.feature \ + tests/features/thing-query-parameters.feature +# use this when we have consensus on tag nomenclature +# uv run behave tests/features --tags="@backend and @production" --no-capture - name: Upload results to Codecov uses: codecov/codecov-action@v4 diff --git a/api/observation.py b/api/observation.py index 3372a2349..90970e5a9 100644 --- a/api/observation.py +++ b/api/observation.py @@ -25,7 +25,7 @@ amp_editor_dependency, amp_viewer_dependency, ) -from db import Observation +from db import Observation, Parameter from schemas.observation import ( CreateGroundwaterLevelObservation, GroundwaterLevelObservationResponse, @@ -113,8 +113,6 @@ async def update_water_chemistry_observation( # ============= Get ============================================== - - @router.get( "/transducer-groundwater-level", summary="Get transducer groundwater level observations", @@ -124,12 +122,19 @@ async def get_transducer_groundwater_level_observations( session: session_dependency, user: amp_viewer_dependency, thing_id: int | None = None, - parameter_id: int | None = None, start_time: datetime | None = None, end_time: datetime | None = None, ) -> CustomPage[TransducerObservationWithBlockResponse]: + + groundwater_parameter_id = ( + session.query(Parameter) + .filter(Parameter.parameter_name == "groundwater level") + .one() + .id + ) + return get_transducer_observations( - session, thing_id, parameter_id, start_time, end_time + session, thing_id, groundwater_parameter_id, start_time, end_time ) diff --git a/core/app.py b/core/app.py index c5b6bb226..377734e20 100644 --- a/core/app.py +++ b/core/app.py @@ -24,7 +24,12 @@ ) from fastapi.openapi.utils import get_openapi -from .initializers import init_db, init_lexicon, init_parameter, register_routes +from .initializers import ( + init_lexicon, + init_parameter, + register_routes, + erase_and_rebuild_db, +) from .settings import settings @@ -34,7 +39,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: Application lifespan event handler to initialize the database and lexicon. """ if settings.get_enum("MODE") == "development": - init_db() + erase_and_rebuild_db() init_lexicon() init_parameter() diff --git a/core/initializers.py b/core/initializers.py index 3da41018b..6b0d7920c 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -21,46 +21,11 @@ from sqlalchemy.orm import Session from db import Base -from db.engine import engine, session_ctx +from db.engine import session_ctx from db.parameter import Parameter from services.lexicon_helper import add_lexicon_term, add_lexicon_category -# ============= EOF ============================================= -def init_db(): - """ - Initialize the database by creating all tables. - This function is called during application startup. - """ - - from sqlalchemy import text - - with engine.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() - - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) - - -def init_hypertables(): - """ - Initialize hypertables for time-series data. - This function is called during application startup. - """ - # session = next(get_db_session()) - # Create hypertables for time-series data - with session_ctx() as session: - session.execute( - text("select create_hypertable('observation', 'observation_datetime');") - ) - - # session.commit() - # session.close() - - def init_parameter(path: str = None) -> None: """ Populate the parameter table to allow their use in creating and editing @@ -92,14 +57,10 @@ def init_parameter(path: str = None) -> None: def erase_and_rebuild_db(session: Session): - from sqlalchemy import text - - with session.bind.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() - + session.execute(text("DROP SCHEMA public CASCADE")) + session.execute(text("CREATE SCHEMA public")) + session.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + session.commit() Base.metadata.drop_all(session.bind) Base.metadata.create_all(session.bind) @@ -174,3 +135,6 @@ def register_routes(app): app.include_router(search_router) app.include_router(thing_router) add_pagination(app) + + +# ============= EOF ============================================= diff --git a/run_bdd.sh b/run_bdd.sh index eb1eaa57f..1f30a4432 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -59,7 +59,13 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} #uv run behave tests/features --tags=@backend #uv run behave tests/features/sensor-notes.feature --tags=@backend +uv run behave tests/features/transducer-data-response.feature + +#uv run behave tests/features/transducer-data-response.feature \ +# tests/features/thing-type-path-parameters.feature \ +# tests/features/thing-query-parameters.feature + +#uv run behave tests/features/well-inventory-csv.feature -uv run behave tests/features --tags=@backend --tags=@production echo "✅ BDD test run complete." diff --git a/services/observation_helper.py b/services/observation_helper.py index ac5877381..af24af05f 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -52,6 +52,12 @@ def get_transducer_observations( order: str | None = None, filter_: str = Query(alias="filter", default=None), ): + if thing_id: + item = session.get(Thing, thing_id) + if item is None: + empty_query = select(TransducerObservation).where(False) + return paginate(query=empty_query, conn=session) + # Subquery to get latest block for each observation block_subq = ( select(TransducerObservationBlock.id) diff --git a/tests/features/environment.py b/tests/features/environment.py index ac97355c1..aad9b0dc2 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -24,12 +24,12 @@ Group, GroupThingAssociation, Sensor, - LexiconTerm, TransducerObservation, Parameter, Deployment, TransducerObservationBlock, ) + from db.engine import session_ctx @@ -187,12 +187,24 @@ def add_block(context, session, parameter): return block +@add_context_object_container("transducer_observations") +def add_transducer_observation(context, session, block, deployment_id, value): + obs = TransducerObservation( + parameter_id=block.parameter_id, + deployment_id=deployment_id, + observation_datetime=datetime.now(), + value=value, + ) + session.add(obs) + context.objects["transducer_observations"].append(obs) + return obs + + def before_all(context): context.objects = {} - - force = False + rebuild = False with session_ctx() as session: - if session.query(LexiconTerm).count() == 0 or force: + if rebuild: erase_and_rebuild_db(session) init_lexicon() init_parameter() @@ -211,16 +223,12 @@ def before_all(context): # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) - add_obs = add_block(context, session, parameter) - if add_obs: - for i in range(1, 10): - obs = TransducerObservation( - parameter_id=parameter.id, - deployment_id=deployment.id, - observation_datetime=datetime.now(), - value=random.random(), - ) - session.add(obs) + block = add_block(context, session, parameter) + for i in range(1, 10): + add_transducer_observation( + context, session, block, deployment.id, random.random() + ) + session.commit() diff --git a/tests/features/steps/common.py b/tests/features/steps/common.py index af44c8095..e724a6016 100644 --- a/tests/features/steps/common.py +++ b/tests/features/steps/common.py @@ -102,4 +102,12 @@ def step_impl(context): ), f"Unexpected response type {context.response.headers['Content-Type']}" +@then("the items should be an empty list") +def step_impl(context): + data = context.response.json() + assert len(data["items"]) == 0, f'Unexpected items {data["items"]}' + assert data["total"] == 0, f'Unexpected total {data["total"]}' + assert data["page"] == 1, f'Unexpected page {data["page"]}' + + # ============= EOF ============================================= diff --git a/tests/features/steps/transducer.py b/tests/features/steps/transducer.py index 4b84834ed..9030ba029 100644 --- a/tests/features/steps/transducer.py +++ b/tests/features/steps/transducer.py @@ -24,45 +24,44 @@ def step_impl(context): with session_ctx() as session: sql = select(Thing).where(Thing.thing_type == "water well") - well = session.execute(sql).scalars().first() - context.well = well + wells = session.execute(sql).unique().scalars().all() + assert len(wells) > 0, "No wells found in db" sql = select(TransducerObservation) transducer_observations = session.execute(sql).scalars().all() - context.transducer_observations = transducer_observations - assert len(transducer_observations) > 0 + assert len(transducer_observations) > 0, "No transducer observations found db" @when("the user requests transducer data for a non-existing well") def step_impl(context): context.response = context.client.get( - "/observation/transducer-groundwater-level", params={"thing_id": 9999} + "/observation/transducer-groundwater-level?thing_id=9999" ) @when("the user requests transducer data for a well") def step_impl(context): context.response = context.client.get( - "/observation/transducer-groundwater-level", - params={"thing_id": context.well.id}, + f"/observation/transducer-groundwater-level?thing_id={context.objects['wells'][0].id}", ) @then("each page should be an array of transducer data") def step_impl(context): data = context.response.json() - context.data = data["items"] - assert len(context.data) > 0 + assert len(data["items"]) > 0, "Expected at least one transducer data entry" @then("each transducer data entry should include a timestamp, value, status") def step_impl(context): - item = context.data[0]["observation"] - block = context.data[0]["block"] + data = context.response.json() + items = data["items"][0] + item = items["observation"] + block = items["block"] - assert "observation_datetime" in item - assert "value" in item - assert "review_status" in block + assert "observation_datetime" in item, f"Expected a timestamp in the data {item}" + assert "value" in item, f"Expected a value in the data {item}" + assert "review_status" in block, f"Expected a review_status in the block {block}" context.timestamp = item["observation_datetime"] context.value = item["value"] @@ -75,7 +74,9 @@ def step_impl(context): from datetime import datetime dt = datetime.fromisoformat(context.timestamp) - assert isinstance(dt, datetime) + assert isinstance( + dt, datetime + ), f"Timestamp is not in ISO 8601 format: {context.timestamp}" @then("the value should be a numeric type") @@ -83,9 +84,12 @@ def step_impl(context): assert isinstance(context.value, (int, float)) -@then('the status should be one of "Draft", "Corrected"') +@then('the status should be one of "approved", "not reviewed"') def step_impl(context): - assert context.status in ("not reviewed",) + assert context.status in ( + "approved", + "not reviewed", + ), f'Unexpected status: {context.status} not in "approved", "not reviewed"' # ============= EOF ============================================= diff --git a/tests/test_observation.py b/tests/test_observation.py index 50307b5ca..3a9c7cf10 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -211,6 +211,7 @@ def test_patch_water_chemistry_observation_404_wrong_activity_type( # ============= Get tests ================= +@pytest.mark.skip(reason="No longer supported") def test_get_transducer_observations(): response = client.get("/observation/transducer-groundwater-level") assert response.status_code == 200