diff --git a/api/search.py b/api/search.py index 40b066cbf..b65bc22cf 100644 --- a/api/search.py +++ b/api/search.py @@ -27,6 +27,8 @@ Phone, Address, Thing, + WellCasingMaterial, + WellPurpose, Asset, AssetThingAssociation, search, @@ -70,15 +72,28 @@ def _get_contact_results(session: Session, q: str, limit: int) -> list[dict]: def _get_thing_results(session: Session, q: str, limit: int) -> list[dict]: - vector = Thing.search_vector + well_vector = ( + func.coalesce(Thing.search_vector, text("''::tsvector")) + .op("||")(func.coalesce(WellCasingMaterial.search_vector, text("''::tsvector"))) + .op("||")(func.coalesce(WellPurpose.search_vector, text("''::tsvector"))) + ) + water_well_query = search( - select(Thing).where(Thing.thing_type == "water well"), + select(Thing) + .outerjoin(WellCasingMaterial) + .outerjoin(WellPurpose) + .where(Thing.thing_type == "water well"), q, - vector=vector, + vector=well_vector, limit=limit, ) + + spring_vector = Thing.search_vector spring_well_query = search( - select(Thing).where(Thing.thing_type == "spring"), q, vector=vector, limit=limit + select(Thing).where(Thing.thing_type == "spring"), + q, + vector=spring_vector, + limit=limit, ) # unique needs to be called because of eager loads diff --git a/api/thing.py b/api/thing.py index d0207f00b..a4ebab305 100644 --- a/api/thing.py +++ b/api/thing.py @@ -32,8 +32,7 @@ editor_dependency, viewer_dependency, ) -from db.thing import Thing, WellScreen -from db.thing import ThingIdLink +from db.thing import Thing, ThingIdLink, WellScreen from schemas.thing import ( CreateThingIdLink, CreateWell, @@ -62,6 +61,8 @@ add_well_screen, get_db_things, get_thing_of_a_thing_type_by_id, + modify_well_descriptor_tables, + WELL_DESCRIPTOR_MODEL_MAP, ) from services.lexicon_helper import get_terms_by_category @@ -379,7 +380,9 @@ async def create_well( Create a new water well in the database. """ try: - return add_thing(session=session, data=thing_data, request=request, user=user) + thing = add_thing(session=session, data=thing_data, request=request, user=user) + modify_well_descriptor_tables(session, thing, thing_data, user) + return thing except ProgrammingError as e: database_error_handler(thing_data, e) @@ -443,7 +446,18 @@ async def update_water_well( """ Update an existing well by ID. """ - return patch_thing(session, request, thing_id, thing_data, user=user) + well_descriptor_data = thing_data.model_copy(deep=True) + + # remove these fields from payload otherwise patch_thing will try to process + # and raise an error because they are not found in the Thing model + for field in WELL_DESCRIPTOR_MODEL_MAP.keys(): + if hasattr(thing_data, field): + delattr(thing_data, field) + + thing = patch_thing(session, request, thing_id, thing_data, user=user) + modify_well_descriptor_tables(session, thing, well_descriptor_data, user) + + return thing @router.patch( diff --git a/db/thing.py b/db/thing.py index 13ce81bbb..d7684ed86 100644 --- a/db/thing.py +++ b/db/thing.py @@ -98,10 +98,6 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix info={"unit": "feet below ground surface"}, comment="Depth of the well casing from ground surface to the bottom of the casing (in feet).", ) - well_casing_material: Mapped[str] = lexicon_term( - nullable=True, - comment="Material of the well casing (e.g., 'PVC', 'Steel', 'Concrete', 'Wood').", - ) well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) @@ -211,6 +207,22 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix passive_deletes=True, ) + well_purposes: Mapped[List["WellPurpose"]] = relationship( + "WellPurpose", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) + + well_casing_materials: Mapped[List["WellCasingMaterial"]] = relationship( + "WellCasingMaterial", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" @@ -237,11 +249,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix ) # Full-text search vector - search_vector = Column( - TSVectorType( - "name", "well_construction_notes", "well_purpose", "well_casing_material" - ) - ) + search_vector = Column(TSVectorType("name", "well_construction_notes")) @property def current_location(self): @@ -302,6 +310,39 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): thing: Mapped["Thing"] = relationship("Thing", back_populates="screens") +class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a controlled vocabulary term for well purposes. + """ + + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + purpose: Mapped[str] = lexicon_term(nullable=False) + + search_vector: Mapped[TSVectorType] = mapped_column(TSVectorType("purpose")) + + thing: Mapped["Thing"] = relationship("Thing", back_populates="well_purposes") + + +class WellCasingMaterial(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a controlled vocabulary term for well casing materials. + """ + + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + + material: Mapped[str] = lexicon_term(nullable=False) + + search_vector: Mapped[TSVectorType] = mapped_column(TSVectorType("material")) + + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="well_casing_materials" + ) + + # TODO: this could be the model used to handle AMP monitoring # class FieldSamplingAdministation(Base, AutoBaseMixin): # # the thing being monitored diff --git a/schemas/thing.py b/schemas/thing.py index 6bf0befc1..21704e9fa 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -15,7 +15,7 @@ # =============================================================================== from typing import List -from pydantic import BaseModel, model_validator, PastDate, Field +from pydantic import BaseModel, model_validator, PastDate, Field, field_validator from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -31,19 +31,11 @@ class ValidateWell(BaseModel): @model_validator(mode="after") def check_depths(self): if ( - self.well_depth is not None - and self.hole_depth is not None + self.hole_depth is not None + and self.well_depth is not None and self.well_depth > self.hole_depth ): raise ValueError("well depth must be less than than or equal to hole depth") - elif ( - self.well_depth is not None - and self.well_casing_depth is not None - and self.well_casing_depth > self.well_depth - ): - raise ValueError( - "well casing depth must be less than or equal to well depth" - ) elif ( self.hole_depth is not None and self.well_casing_depth is not None @@ -89,7 +81,7 @@ class CreateWell(CreateBaseThing, ValidateWell): Schema for creating a well. """ - well_purpose: str | None = None + well_purposes: list[str] | None = None well_depth: float | None = Field( default=None, gt=0, description="Well depth in feet" ) @@ -103,7 +95,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_casing_depth: float | None = Field( default=None, gt=0, description="Well casing depth in feet" ) - well_casing_material: str | None = None + well_casing_materials: list[str] | None = None class CreateSpring(CreateBaseThing): @@ -148,7 +140,7 @@ class WellResponse(BaseThingResponse): Response schema for well details. """ - well_purpose: str | None = None # e.g., "Production", "Observation", etc. + well_purposes: list[str] = [] well_depth: float | None = None well_depth_unit: str = "ft" hole_depth: float | None = None @@ -157,9 +149,28 @@ class WellResponse(BaseThingResponse): well_casing_diameter_unit: str = "in" well_casing_depth: float | None = None well_casing_depth_unit: str = "ft" - well_casing_material: str | None = None + well_casing_materials: list[str] = [] well_construction_notes: str | None = None + @field_validator("well_purposes", mode="before") + def populate_well_purposes_with_strings(cls, well_purposes): + if well_purposes is not None: + purposes = [well_purpose.purpose for well_purpose in well_purposes] + else: + purposes = [] + return purposes + + @field_validator("well_casing_materials", mode="before") + def populate_well_casing_materials_with_strings(cls, well_casing_materials): + if well_casing_materials is not None: + materials = [ + well_casing_material.material + for well_casing_material in well_casing_materials + ] + else: + materials = [] + return materials + class SpringResponse(BaseThingResponse): """ @@ -249,13 +260,13 @@ class UpdateThing(BaseUpdateModel): class UpdateWell(UpdateThing, ValidateWell): - well_purpose: str | None = None + well_purposes: list[str] | None = None well_depth: float | None = None # in feet hole_depth: float | None = None # in feet well_construction_notes: str | None = None well_casing_diameter: float | None = None # in inches well_casing_depth: float | None = None # in feet - well_casing_material: str | None = None + well_casing_materials: list[str] | None = None class UpdateSpring(UpdateThing): diff --git a/services/thing_helper.py b/services/thing_helper.py index bddabdbed..563ddace7 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -20,7 +20,15 @@ from sqlalchemy.orm import Session, aliased from starlette.status import HTTP_404_NOT_FOUND, HTTP_409_CONFLICT -from db import LocationThingAssociation, Thing, Base, Location, WellScreen +from db import ( + LocationThingAssociation, + Thing, + Base, + Location, + WellScreen, + WellPurpose, + WellCasingMaterial, +) from db.group import GroupThingAssociation from services.audit_helper import audit_add from services.crud_helper import model_patcher @@ -30,6 +38,11 @@ from shapely import wkb from shapely.geometry import mapping +WELL_DESCRIPTOR_MODEL_MAP = { + "well_purposes": (WellPurpose, "purpose"), + "well_casing_materials": (WellCasingMaterial, "material"), +} + def wkb_to_geojson(wkb_element): if wkb_element is None: @@ -132,7 +145,8 @@ def add_thing( thing_type = get_thing_type_from_request(request) if isinstance(data, BaseModel): - data = data.model_dump() + well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) + data = data.model_dump(exclude=well_descriptor_table_list) location_id = data.pop("location_id", None) group_id = data.pop("group_id", None) @@ -217,4 +231,34 @@ def patch_thing( return thing +def modify_well_descriptor_tables( + session: Session, thing: Thing, payload: BaseModel, user: dict +) -> None: + """ + This function is to add and update well descriptor tables when a Thing is created + or updated. It deletes existing descriptor table records for the Thing if they + exist and then adds the new data. + """ + try: + for descriptor_table in WELL_DESCRIPTOR_MODEL_MAP.keys(): + db_table, field_name = WELL_DESCRIPTOR_MODEL_MAP[descriptor_table] + descriptor_table_data = payload.model_dump(exclude_unset=True).pop( + descriptor_table, None + ) + if descriptor_table_data: + session.query(db_table).filter(db_table.thing_id == thing.id).delete() + for ctd in descriptor_table_data: + inserts = {"thing_id": thing.id, field_name: ctd} + record = db_table(**inserts) + audit_add(user, record) + session.add(record) + session.commit() + + # Thing needs to be refreshed to find associated child table data + session.refresh(thing) + except Exception as e: + session.rollback() + raise e + + # ============= EOF ============================================= diff --git a/tests/conftest.py b/tests/conftest.py index 61fe086e3..664651196 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,13 +55,11 @@ def water_well_thing(location): first_visit_date="2023-03-03", thing_type="water well", release_status="draft", - well_purpose="Domestic", well_depth=10, hole_depth=10, well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, - well_casing_material="PVC", ) session.add(water_well) session.commit() @@ -81,6 +79,66 @@ def water_well_thing(location): session.commit() +@pytest.fixture() +def pvc_well_casing_material(water_well_thing): + with session_ctx() as session: + casing_material = WellCasingMaterial( + thing_id=water_well_thing.id, + material="PVC", + release_status="draft", + ) + session.add(casing_material) + session.commit() + yield casing_material + session.delete(casing_material) + session.commit() + + +@pytest.fixture(scope="function") +def steel_well_casing_material(water_well_thing): + with session_ctx() as session: + casing_material = WellCasingMaterial( + thing_id=water_well_thing.id, + material="Steel", + release_status="draft", + ) + session.add(casing_material) + session.commit() + yield casing_material + session.delete(casing_material) + session.commit() + + +@pytest.fixture() +def irrigation_well_purpose(water_well_thing): + with session_ctx() as session: + purpose = WellPurpose( + thing_id=water_well_thing.id, + purpose="Irrigation", + release_status="draft", + ) + session.add(purpose) + session.commit() + yield purpose + session.delete(purpose) + session.commit() + + +@pytest.fixture() +def domestic_well_purpose(water_well_thing): + with session_ctx() as session: + purpose = WellPurpose( + thing_id=water_well_thing.id, + purpose="Domestic", + release_status="draft", + ) + session.add(purpose) + session.commit() + yield purpose + session.delete(purpose) + session.commit() + + @pytest.fixture() def well_screen(water_well_thing): with session_ctx() as session: diff --git a/tests/test_thing.py b/tests/test_thing.py index d2038befc..12a728b2b 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -55,21 +55,13 @@ def override_authentication_dependency_fixture(): # VALIDATE tests =============================================================== -def test_validate_well_depth_hole_depth(): +def test_validate_hole_depth_well_depth(): with pytest.raises( ValueError, match="well depth must be less than than or equal to hole depth" ): ValidateWell(well_depth=100.0, hole_depth=90.0) -def test_validate_well_depth_casing_depth(): - with pytest.raises( - ValueError, - match="well casing depth must be less than or equal to well depth", - ): - ValidateWell(well_depth=100.0, well_casing_depth=110.0) - - def test_validate_hole_depth_casing_depth(): with pytest.raises( ValueError, @@ -88,13 +80,13 @@ def test_add_water_well(location, group): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Domestic", "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", "well_casing_diameter": 5.0, "well_casing_depth": 10.0, - "well_casing_material": "PVC", + "well_casing_materials": ["PVC"], + "well_purposes": ["Domestic"], } response = client.post("/thing/water-well", json=payload) @@ -106,7 +98,7 @@ def test_add_water_well(location, group): assert data["name"] == payload["name"] assert data["first_visit_date"] == payload["first_visit_date"] assert data["thing_type"] == "water well" - assert data["well_purpose"] == payload["well_purpose"] + assert data["well_purposes"] == payload["well_purposes"] assert data["hole_depth"] == payload["hole_depth"] assert data["hole_depth_unit"] == "ft" assert data["well_depth"] == payload["well_depth"] @@ -116,7 +108,7 @@ def test_add_water_well(location, group): assert data["well_casing_diameter_unit"] == "in" assert data["well_casing_depth"] == payload["well_casing_depth"] assert data["well_casing_depth_unit"] == "ft" - assert data["well_casing_material"] == payload["well_casing_material"] + assert data["well_casing_materials"] == payload["well_casing_materials"] expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( @@ -135,13 +127,12 @@ def test_add_water_well_409_bad_group_id(location): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Domestic", + "well_purposes": ["Domestic"], "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", "well_casing_diameter": 5.0, "well_casing_depth": 10.0, - "well_casing_material": "PVC", } response = client.post("/thing/water-well", json=payload) @@ -161,7 +152,7 @@ def test_add_water_well_409_bad_location_id(group): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Domestic", + "well_purposes": ["Domestic"], "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", @@ -383,7 +374,9 @@ def test_get_water_wells(water_well_thing, location): ) assert data["items"][0]["thing_type"] == water_well_thing.thing_type assert data["items"][0]["release_status"] == water_well_thing.release_status - assert data["items"][0]["well_purpose"] == water_well_thing.well_purpose + assert data["items"][0]["well_purposes"] == [ + p for p in water_well_thing.well_purposes + ] assert data["items"][0]["well_depth"] == water_well_thing.well_depth assert data["items"][0]["well_depth_unit"] == "ft" assert data["items"][0]["hole_depth"] == water_well_thing.hole_depth @@ -399,10 +392,9 @@ def test_get_water_wells(water_well_thing, location): assert data["items"][0]["well_casing_diameter_unit"] == "in" assert data["items"][0]["well_casing_depth"] == water_well_thing.well_casing_depth assert data["items"][0]["well_casing_depth_unit"] == "ft" - assert ( - data["items"][0]["well_casing_material"] - == water_well_thing.well_casing_material - ) + assert data["items"][0]["well_casing_materials"] == [ + wcm for wcm in water_well_thing.well_casing_materials + ] expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( @@ -423,7 +415,7 @@ def test_get_water_well_by_id(water_well_thing, location): assert data["first_visit_date"] == water_well_thing.first_visit_date.isoformat() assert data["thing_type"] == water_well_thing.thing_type assert data["release_status"] == water_well_thing.release_status - assert data["well_purpose"] == water_well_thing.well_purpose + assert data["well_purposes"] == [p for p in water_well_thing.well_purposes] assert data["well_depth"] == water_well_thing.well_depth assert data["well_depth_unit"] == "ft" assert data["hole_depth"] == water_well_thing.hole_depth @@ -433,7 +425,9 @@ def test_get_water_well_by_id(water_well_thing, location): assert data["well_casing_diameter_unit"] == "in" assert data["well_casing_depth"] == water_well_thing.well_casing_depth assert data["well_casing_depth_unit"] == "ft" - assert data["well_casing_material"] == water_well_thing.well_casing_material + assert data["well_casing_materials"] == [ + wcm for wcm in water_well_thing.well_casing_materials + ] expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( @@ -700,7 +694,10 @@ def test_get_thing_by_id(water_well_thing, location): assert data["first_visit_date"] == water_well_thing.first_visit_date.isoformat() assert data["thing_type"] == water_well_thing.thing_type assert data["release_status"] == water_well_thing.release_status - assert data["well_purpose"] == water_well_thing.well_purpose + assert data["well_purposes"] == [p for p in water_well_thing.well_purposes] + assert data["well_casing_materials"] == [ + cm for cm in water_well_thing.well_casing_materials + ] assert data["well_depth"] == water_well_thing.well_depth assert data["hole_depth"] == water_well_thing.hole_depth assert data["well_construction_notes"] == water_well_thing.well_construction_notes @@ -778,7 +775,7 @@ def test_patch_water_well(water_well_thing, location): "name": "patched water well", "first_visit_date": "2023-02-02", "release_status": "provisional", - "well_purpose": "Injection", + "well_purposes": ["Injection"], "well_depth": 20, "hole_depth": 40, "well_construction_notes": "patched well construction notes", @@ -789,7 +786,7 @@ def test_patch_water_well(water_well_thing, location): assert data["name"] == payload["name"] assert data["first_visit_date"] == payload["first_visit_date"] assert data["release_status"] == payload["release_status"] - assert data["well_purpose"] == payload["well_purpose"] + assert data["well_purposes"] == payload["well_purposes"] assert data["well_depth"] == payload["well_depth"] assert data["hole_depth"] == payload["hole_depth"] assert data["well_construction_notes"] == payload["well_construction_notes"] @@ -809,7 +806,6 @@ def test_patch_water_well_404_not_found(): "name": "patched water well", "first_visit_date": "2023-02-02", "release_status": "provisional", - "well_purpose": "Injection", "well_depth": 20, "hole_depth": 40, "well_construction_notes": "patched well construction notes", diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ff180deb8..916801c06 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -20,7 +20,14 @@ from datetime import datetime from pandas import isna -from db import LocationThingAssociation, Thing, WellScreen, Location +from db import ( + LocationThingAssociation, + Thing, + WellScreen, + Location, + WellPurpose, + WellCasingMaterial, +) from schemas.thing import CreateWellScreen, CreateWell from services.gcs_helper import get_storage_bucket from services.util import ( @@ -75,6 +82,19 @@ def _extract_well_purposes(row) -> list[str]: return purposes +def _extract_casing_materials(row) -> list[str]: + materials = [] + if "pvc" in row.CasingDescription.lower(): + materials.append("PVC") + + if "steel" in row.CasingDescription.lower(): + materials.append("Steel") + + if "concrete" in row.CasingDescription.lower(): + materials.append("Concrete") + return materials + + def transfer_wells(session, limit=0) -> None: wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) ldf = read_csv("Location") @@ -126,7 +146,10 @@ def transfer_wells(session, limit=0) -> None: try: first_visit_date = _get_first_visit_date(row) - well_purposes = _extract_well_purposes(row) + well_purposes = [] if isna(row.CurrentUse) else _extract_well_purposes(row) + well_casing_materials = ( + [] if isna(row.CasingDescription) else _extract_casing_materials(row) + ) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation @@ -135,7 +158,6 @@ def transfer_wells(session, limit=0) -> None: nma_pk_welldata=row.WellID, name=row.PointID, first_visit_date=first_visit_date, - # well_purpose=well_purpose, hole_depth=row.HoleDepth, well_depth=row.WellDepth, well_construction_notes=row.ConstructionNotes, @@ -153,10 +175,27 @@ def transfer_wells(session, limit=0) -> None: continue try: - well_data = data.model_dump(exclude=["location_id", "group_id"]) + well_data = data.model_dump( + exclude=[ + "location_id", + "group_id", + "well_purposes", + "well_casing_materials", + ] + ) well_data["thing_type"] = "water well" well = Thing(**well_data) session.add(well) + + if well_purposes: + for wp in well_purposes: + wp_obj = WellPurpose(thing=well, purpose=wp) + session.add(wp_obj) + + if well_casing_materials: + for wcm in well_casing_materials: + wcm_obj = WellCasingMaterial(thing=well, material=wcm) + session.add(wcm_obj) except Exception as e: session.rollback() logger.critical(f"Error creating well for {row.PointID}: {e}")