From a14aacedeae179da7e9a6396d7d6c032924ddc35 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 12 Dec 2025 16:08:42 -0700 Subject: [PATCH 1/3] feat: refactor imports to use core.constants and add water levels command --- api/location.py | 10 +++++----- cli/cli.py | 15 ++++++++++++--- cli/service_adapter.py | 1 - constants.py => core/constants.py | 0 db/aquifer_system.py | 7 +++---- db/geologic_formation.py | 7 +++---- db/group.py | 4 ++-- db/location.py | 2 +- pyproject.toml | 7 +++++++ schemas/location.py | 6 +++--- services/geospatial_helper.py | 16 ++++++++-------- services/util.py | 2 +- tests/features/data/water-levels.csv | 0 tests/features/steps/well-core-information.py | 8 ++++---- tests/test_collabnet.py | 3 +-- tests/test_geospatial.py | 7 ++++--- transfers/util.py | 2 +- uv.lock | 2 +- 18 files changed, 56 insertions(+), 43 deletions(-) rename constants.py => core/constants.py (100%) create mode 100644 tests/features/data/water-levels.csv diff --git a/api/location.py b/api/location.py index 5e2b9abda..af2590e58 100644 --- a/api/location.py +++ b/api/location.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from fastapi import APIRouter from fastapi import Query, Response from fastapi_pagination.ext.sqlalchemy import paginate from sqlalchemy import select, func from starlette import status + from api.pagination import CustomPage -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 from core.dependencies import ( session_dependency, admin_dependency, @@ -27,12 +29,10 @@ ) from db.location import Location from schemas.location import CreateLocation, LocationResponse, UpdateLocation -from services.geospatial_helper import make_within_wkt -from services.query_helper import make_query, order_sort_filter, simple_get_by_id from services.crud_helper import model_patcher, model_deleter, model_adder +from services.geospatial_helper import make_within_wkt from services.location_helper import set_geographic_attributes - -from fastapi import APIRouter +from services.query_helper import make_query, order_sort_filter, simple_get_by_id router = APIRouter(prefix="/location", tags=["location"]) diff --git a/cli/cli.py b/cli/cli.py index 6f7278392..94117dd78 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -59,10 +59,19 @@ def well_inventory_csv(file_path: str): well_inventory_csv(file_path) -@cli.command() -@click.argument( +@cli.group() +def water_levels(): + """Water-level utilities""" + pass + + +@water_levels.command("bulk-upload") +@click.option( + "--file", "file_path", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), + required=True, + help="Path to CSV file containing water level rows", ) @click.option( "--output", @@ -71,7 +80,7 @@ def well_inventory_csv(file_path: str): default=None, help="Optional output format", ) -def water_levels_csv(file_path: str, output_format: str | None): +def water_levels_bulk_upload(file_path: str, output_format: str | None): """ parse and upload a csv """ diff --git a/cli/service_adapter.py b/cli/service_adapter.py index a177a95ff..04a9ae942 100644 --- a/cli/service_adapter.py +++ b/cli/service_adapter.py @@ -39,7 +39,6 @@ def water_levels_csv(source_file: Path | str, *, pretty_json: bool = False): source_file = Path(source_file) result = bulk_upload_water_levels(source_file, pretty_json=pretty_json) - print(result.stdout) if result.stderr: print(result.stderr, file=sys.stderr) return result.exit_code diff --git a/constants.py b/core/constants.py similarity index 100% rename from constants.py rename to core/constants.py diff --git a/db/aquifer_system.py b/db/aquifer_system.py index c202d77c9..14cf99f31 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -6,16 +6,15 @@ from typing import List, TYPE_CHECKING +from geoalchemy2 import Geometry from sqlalchemy import Text, Index -from sqlalchemy.orm import relationship, Mapped, mapped_column from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy -from geoalchemy2 import Geometry +from sqlalchemy.orm import relationship, Mapped, mapped_column +from core.constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term -from constants import SRID_WGS84 - if TYPE_CHECKING: from db.thing import WellScreen, ThingAquiferAssociation, Thing from db.aquifer_type import AquiferType diff --git a/db/geologic_formation.py b/db/geologic_formation.py index 2379f50f4..8a285e488 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -7,16 +7,15 @@ from typing import List, TYPE_CHECKING +from geoalchemy2 import Geometry from sqlalchemy import Text, Index -from sqlalchemy.orm import relationship, Mapped, mapped_column from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy -from geoalchemy2 import Geometry +from sqlalchemy.orm import relationship, Mapped, mapped_column +from core.constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term -from constants import SRID_WGS84 - if TYPE_CHECKING: from db.thing import Thing, WellScreen from db.thing_geologic_formation_association import ( diff --git a/db/group.py b/db/group.py index 2669e70f7..5be1dedc6 100644 --- a/db/group.py +++ b/db/group.py @@ -17,11 +17,11 @@ from geoalchemy2 import Geometry, WKBElement from sqlalchemy import String, Integer, ForeignKey +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped from sqlalchemy.testing.schema import mapped_column -from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: diff --git a/db/location.py b/db/location.py index 6376988f6..2fbeecc82 100644 --- a/db/location.py +++ b/db/location.py @@ -30,7 +30,7 @@ from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped, mapped_column -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin from db.data_provenance import DataProvenanceMixin from db.notes import NotesMixin diff --git a/pyproject.toml b/pyproject.toml index 8bf5ecc83..5f10393eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,13 @@ dependencies = [ "yarl==1.20.1", ] +[tool.uv] +package = true + +[tool.setuptools] +packages = ["alembic", "cli", "core", "db", "schemas", "services"] +modules = ["constants"] + [project.scripts] oco = "cli.cli:cli" diff --git a/schemas/location.py b/schemas/location.py index 17414b5c4..1a61e257a 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -14,19 +14,19 @@ # limitations under the License. # =============================================================================== from datetime import date +from typing import Any from typing import List from geoalchemy2 import WKBElement from geoalchemy2.shape import to_shape from pydantic import BaseModel, model_validator, field_validator, Field, ConfigDict -from typing import Any -from constants import SRID_WGS84, SRID_UTM_ZONE_13N +from core.constants import SRID_WGS84, SRID_UTM_ZONE_13N from core.enums import ElevationMethod, CoordinateMethod from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.notes import NoteResponse, CreateNote, UpdateNote -from services.validation.geospatial import validate_wkt_geometry from services.util import convert_m_to_ft, transform_srid +from services.validation.geospatial import validate_wkt_geometry # -------- VALIDATE -------- diff --git a/services/geospatial_helper.py b/services/geospatial_helper.py index 356c55634..fc1118aa8 100644 --- a/services/geospatial_helper.py +++ b/services/geospatial_helper.py @@ -14,19 +14,19 @@ # limitations under the License. # =============================================================================== import shapefile -from shapely.errors import GEOSException -from shapely.io import from_geojson - -import constants -from db.thing import Thing -from db.group import GroupThingAssociation, Group -from db.location import Location, LocationThingAssociation from geoalchemy2.functions import ST_GeomFromText, ST_Within, ST_AsGeoJSON from geoalchemy2.shape import to_shape +from shapely.errors import GEOSException +from shapely.io import from_geojson from shapely.wkt import loads as wkt_loads from sqlalchemy import Select, select -from sqlalchemy.orm import aliased from sqlalchemy import func +from sqlalchemy.orm import aliased + +from core import constants +from db.group import GroupThingAssociation, Group +from db.location import Location, LocationThingAssociation +from db.thing import Thing def get_thing_features( diff --git a/services/util.py b/services/util.py index 6a7316073..1ba7190f7 100644 --- a/services/util.py +++ b/services/util.py @@ -6,7 +6,7 @@ from shapely.ops import transform from sqlalchemy.orm import DeclarativeBase -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 TRANSFORMERS = {} METERS_TO_FEET = 3.28084 diff --git a/tests/features/data/water-levels.csv b/tests/features/data/water-levels.csv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index 1f56161f6..0ae559c2b 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -1,13 +1,13 @@ -from constants import SRID_WGS84, SRID_UTM_ZONE_13N +from behave import then +from geoalchemy2.shape import to_shape + +from core.constants import SRID_WGS84, SRID_UTM_ZONE_13N from services.util import ( transform_srid, convert_m_to_ft, retrieve_latest_polymorphic_history_table_record, ) -from behave import then -from geoalchemy2.shape import to_shape - @then("the response should be in JSON format") def step_impl(context): diff --git a/tests/test_collabnet.py b/tests/test_collabnet.py index ce57f07c1..432402121 100644 --- a/tests/test_collabnet.py +++ b/tests/test_collabnet.py @@ -15,10 +15,9 @@ # =============================================================================== import pytest -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 from db import Location from db.engine import session_ctx - from services.thing_helper import add_thing from tests import client diff --git a/tests/test_geospatial.py b/tests/test_geospatial.py index 7054c5fe0..a25ce24cb 100644 --- a/tests/test_geospatial.py +++ b/tests/test_geospatial.py @@ -14,10 +14,11 @@ # limitations under the License. # =============================================================================== from pathlib import Path + import pytest +from geoalchemy2 import functions as geofunc -from main import app -from constants import SRID_WGS84 +from core.constants import SRID_WGS84 from core.dependencies import ( admin_function, editor_function, @@ -28,8 +29,8 @@ ) from db import Thing, Location, LocationThingAssociation, Group, MeasuringPointHistory from db.engine import session_ctx +from main import app from tests import client, override_authentication -from geoalchemy2 import functions as geofunc @pytest.fixture(scope="module", autouse=True) diff --git a/transfers/util.py b/transfers/util.py index 35828d21c..ce3d66ada 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -30,7 +30,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from constants import SRID_WGS84, SRID_UTM_ZONE_13N +from core.constants import SRID_WGS84, SRID_UTM_ZONE_13N from db import Thing, Location, DataProvenance, Parameter from db.engine import session_ctx from services.gcs_helper import get_storage_bucket diff --git a/uv.lock b/uv.lock index 083edcd3d..2edcb7570 100644 --- a/uv.lock +++ b/uv.lock @@ -1002,7 +1002,7 @@ wheels = [ [[package]] name = "ocotilloapi" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "aiofiles" }, { name = "aiohappyeyeballs" }, From 548f372b542a4d821b332c0773a76bf982b1faeb Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 13 Dec 2025 10:36:29 -0700 Subject: [PATCH 2/3] feat: enhance water level management with sample cleanup and CSV updates --- pyproject.toml | 1 - tests/features/data/water-levels.csv | 3 +++ tests/features/environment.py | 22 +++++++++++++++++----- tests/features/water-level-csv.feature | 7 +++---- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5f10393eb..5fc75bb74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,7 +105,6 @@ package = true [tool.setuptools] packages = ["alembic", "cli", "core", "db", "schemas", "services"] -modules = ["constants"] [project.scripts] oco = "cli.cli:cli" diff --git a/tests/features/data/water-levels.csv b/tests/features/data/water-levels.csv index e69de29bb..db510e89d 100644 --- a/tests/features/data/water-levels.csv +++ b/tests/features/data/water-levels.csv @@ -0,0 +1,3 @@ +field_staff,well_name_point_id,field_event_date_time,measurement_date_time,sampler,sample_method,mp_height,level_status,depth_to_water_ft,data_quality,water_level_notes +Alice Lopez,AR0001,2025-02-15T08:00:00-07:00,2025-02-15T10:30:00-07:00,Groundwater Team,electric tape,1.5,stable,45.2,approved,Initial measurement after irrigation shutdown +Bernardo Chen,AR0002,2025-03-05T09:15:00-07:00,2025-03-05T11:10:00-07:00,Consultant,steel tape,1.8,rising,47.0,provisional,Follow-up visit; pump was off for 24h diff --git a/tests/features/environment.py b/tests/features/environment.py index 123bc588f..530ca453a 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -46,6 +46,7 @@ ThingGeologicFormationAssociation, Base, Asset, + Sample, ) from db.engine import session_ctx @@ -687,12 +688,16 @@ def before_all(context): def after_all(context): with session_ctx() as session: - for table in context.objects.values(): - for record in table: - obj = session.get(record.__class__, record.id) - if obj: - session.delete(obj) + for table in reversed(Base.metadata.sorted_tables): + if table.name in ("alembic_version", "parameter"): + continue + elif table.name.startswith("lexicon"): + continue + elif table.name == "sample": + continue + session.execute(table.delete()) session.commit() + context.objects.clear() def before_scenario(context, scenario): @@ -714,6 +719,13 @@ def after_scenario(context, scenario): asset = session.scalars(sql).one() session.delete(asset) session.commit() + elif "cleanup_samples" in scenario.tags: + # delete all samples created during happy path tests + with session_ctx() as session: + samples = session.query(Sample).all() + for sample in samples: + session.delete(sample) + session.commit() # ============= EOF ============================================= diff --git a/tests/features/water-level-csv.feature b/tests/features/water-level-csv.feature index 4bdbe9c0d..6c94969d1 100644 --- a/tests/features/water-level-csv.feature +++ b/tests/features/water-level-csv.feature @@ -1,6 +1,5 @@ # features/cli/bulk_upload_water_levels.feature -@skip @cli @backend @BDMS-TBD @@ -20,7 +19,7 @@ Feature: Bulk upload water level entries from CSV via CLI # | level_status | # | data_quality | - @positive @happy_path @BDMS-TBD + @positive @happy_path @BDMS-TBD @cleanup_samples Scenario: Uploading a valid water level entry CSV containing required and optional fields Given a valid CSV file for bulk water level entry upload And my CSV file is encoded in UTF-8 and uses commas as separators @@ -56,7 +55,7 @@ Feature: Bulk upload water level entries from CSV via CLI And stdout includes an array of created water level entry objects And stderr should be empty - @positive @validation @column_order @BDMS-TBD + @positive @validation @column_order @BDMS-TBD @cleanup_samples Scenario: Upload succeeds when required columns are present but in a different order Given my CSV file contains all required headers but in a different column order And the CSV includes required fields: @@ -77,7 +76,7 @@ Feature: Bulk upload water level entries from CSV via CLI And all water level entries are imported And stderr should be empty - @positive @validation @extra_columns @BDMS-TBD + @positive @validation @extra_columns @BDMS-TBD @cleanup_samples Scenario: Upload succeeds when CSV contains extra, unknown columns Given my CSV file contains extra columns but is otherwise valid When I run the CLI command: From 12053a3b6954a55a564014af6a47a1c4f49d0727 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 13 Dec 2025 11:05:23 -0700 Subject: [PATCH 3/3] feat: add CLI tests for water levels commands and associated services --- tests/test_cli_commands.py | 220 +++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 tests/test_cli_commands.py diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py new file mode 100644 index 000000000..13e991145 --- /dev/null +++ b/tests/test_cli_commands.py @@ -0,0 +1,220 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +from __future__ import annotations + +import textwrap +import uuid +from pathlib import Path + +from click.testing import CliRunner +from sqlalchemy import select + +from cli.cli import cli +from db import FieldActivity, FieldEvent, Observation, Sample +from db.engine import session_ctx + + +def test_initialize_lexicon_invokes_initializer(monkeypatch): + called = {"count": 0} + + def fake_initializer(): + called["count"] += 1 + + monkeypatch.setattr("core.initializers.init_lexicon", fake_initializer) + + runner = CliRunner() + result = runner.invoke(cli, ["initialize-lexicon"]) + + assert result.exit_code == 0 + assert called["count"] == 1 + + +def test_associate_assets_command_calls_service(monkeypatch): + captured = {} + + def fake_associate(source_directory): + captured["path"] = Path(source_directory) + return ["uri1"] + + monkeypatch.setattr("cli.service_adapter.associate_assets", fake_associate) + + runner = CliRunner() + with runner.isolated_filesystem(): + workdir = Path.cwd() + asset_dir = workdir / "asset_import_batch" + asset_dir.mkdir() + + result = runner.invoke(cli, ["associate-assets", str(asset_dir)]) + + assert result.exit_code == 0, result.output + assert captured["path"] == asset_dir + + +def test_well_inventory_csv_command_calls_service(monkeypatch, tmp_path): + inventory_file = tmp_path / "inventory.csv" + inventory_file.write_text("header\nvalue\n") + captured = {} + + def fake_well_inventory(file_path): + captured["path"] = file_path + + monkeypatch.setattr("cli.service_adapter.well_inventory_csv", fake_well_inventory) + + runner = CliRunner() + result = runner.invoke(cli, ["well-inventory-csv", str(inventory_file)]) + + assert result.exit_code == 0 + assert Path(captured["path"]) == inventory_file + + +def test_water_levels_bulk_upload_default_output(monkeypatch, tmp_path): + csv_file = tmp_path / "water_levels.csv" + csv_file.write_text("col\nvalue\n") + captured = {} + + def fake_upload(file_path, *, pretty_json=False): + captured["path"] = file_path + captured["pretty_json"] = pretty_json + return 0 + + monkeypatch.setattr("cli.service_adapter.water_levels_csv", fake_upload) + + runner = CliRunner() + result = runner.invoke( + cli, ["water-levels", "bulk-upload", "--file", str(csv_file)] + ) + + assert result.exit_code == 0 + assert Path(captured["path"]) == csv_file + assert captured["pretty_json"] is False + + +def test_water_levels_bulk_upload_json_output(monkeypatch, tmp_path): + csv_file = tmp_path / "water_levels.csv" + csv_file.write_text("col\nvalue\n") + captured = {} + + def fake_upload(file_path, *, pretty_json=False): + captured["path"] = file_path + captured["pretty_json"] = pretty_json + return 0 + + monkeypatch.setattr("cli.service_adapter.water_levels_csv", fake_upload) + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "water-levels", + "bulk-upload", + "--file", + str(csv_file), + "--output", + "json", + ], + ) + + assert result.exit_code == 0 + assert Path(captured["path"]) == csv_file + assert captured["pretty_json"] is True + + +def test_water_levels_cli_persists_observations(tmp_path, water_well_thing): + """ + End-to-end CLI invocation should create FieldEvent, Sample, and Observation rows. + """ + + def _write_csv(path: Path, *, well_name: str, notes: str): + csv_text = textwrap.dedent( + f"""\ + field_staff,well_name_point_id,field_event_date_time,measurement_date_time,sampler,sample_method,mp_height,level_status,depth_to_water_ft,data_quality,water_level_notes + CLI Tester,{well_name},2025-02-15T08:00:00-07:00,2025-02-15T10:30:00-07:00,Groundwater Team,electric tape,1.5,stable,42.5,approved,{notes} + """ + ) + path.write_text(csv_text) + + unique_notes = f"pytest-{uuid.uuid4()}" + csv_file = tmp_path / "water_levels.csv" + _write_csv(csv_file, well_name=water_well_thing.name, notes=unique_notes) + + runner = CliRunner() + result = runner.invoke( + cli, ["water-levels", "bulk-upload", "--file", str(csv_file)] + ) + + assert result.exit_code == 0, result.output + + created_ids: dict[str, int] = {} + with session_ctx() as session: + stmt = ( + select(Observation) + .join(Observation.sample) + .join(Sample.field_activity) + .join(FieldActivity.field_event) + .where(Sample.notes == unique_notes) + ) + observations = session.scalars(stmt).all() + assert len(observations) == 1, "Expected one observation for the uploaded CSV" + + observation = observations[0] + sample = observation.sample + field_activity = sample.field_activity + field_event = field_activity.field_event + + assert field_event.thing_id == water_well_thing.id + assert sample.sample_method == "Electric tape measurement (E-probe)" + assert sample.sample_matrix == "water" + assert observation.value == 42.5 + assert observation.measuring_point_height == 1.5 + assert observation.notes == "Level status: stable | Data quality: approved" + assert ( + field_event.notes == f"Field staff: CLI Tester | {unique_notes}" + ), "Field event notes should capture field staff and notes" + + created_ids = { + "observation_id": observation.id, + "sample_id": sample.id, + "field_activity_id": field_activity.id, + "field_event_id": field_event.id, + } + + if created_ids: + # Clean up committed rows so other tests see a pristine database. + with session_ctx() as session: + observation = session.get(Observation, created_ids["observation_id"]) + sample = session.get(Sample, created_ids["sample_id"]) + field_activity = session.get( + FieldActivity, created_ids["field_activity_id"] + ) + field_event = session.get(FieldEvent, created_ids["field_event_id"]) + + if observation: + session.delete(observation) + session.flush() + if sample: + session.delete(sample) + session.flush() + if field_activity: + session.delete(field_activity) + session.flush() + if field_event: + session.delete(field_event) + session.flush() + + session.commit() + + +# ============= EOF =============================================