From 656e159d4179a1d5ae826d5bd25266491166d6ce Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 13:27:45 -0700 Subject: [PATCH 1/5] fix: use regex to check for pump type in well construction notes regex is a more robust way to check for the presence of pump types in construction notes, handling variations in spacing and punctuation. --- transfers/well_transfer.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 9e684cc8d..22209741a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -16,7 +16,7 @@ import json import time from datetime import datetime, UTC - +import re import pandas as pd from pandas import isna from pydantic import ValidationError @@ -118,19 +118,26 @@ def _extract_casing_materials(row) -> list[str]: return materials +pattern = re.compile( + r"\b(?Pjet|hand|submersible)\b|\b(?Pline[-\s]+shaft)\b", re.IGNORECASE +) + + +def first_matched_term(text: str): + m = pattern.search(text) + if not m: + return None + return m.group("term") or m.group("phrase") + + +PUMP_MAPPING = {"jet": "Jet", "hand": "Hand", "submersible": "Submersible"} + + def _extract_well_pump_type(row) -> str | None: + if isna(row.ConstructionNotes): + return None construction_notes = row.ConstructionNotes.lower() - if "pump" in construction_notes: - if "submersible" in construction_notes: - return "Submersible" - elif "jet" in construction_notes: - return "Jet" - elif "line shaft" in construction_notes or "lineshaft" in construction_notes: - return "Line Shaft" - elif "hand" in construction_notes: - return "Hand" - else: - return None + return PUMP_MAPPING.get(first_matched_term(construction_notes), None) def get_wells_to_transfer( @@ -252,9 +259,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) - well_pump_type = ( - _extract_well_pump_type(row) if row.ConstructionNotes else None - ) + well_pump_type = _extract_well_pump_type(row) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation From 183d275525c5c323d90f46fbb6cf828d7136bb4b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 14:20:27 -0700 Subject: [PATCH 2/5] fix: ensure permissions are not null during transfer check to make sure that permissions are set before transferring them from one entity to another to avoid potential errors. --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 1dcca2b8f..fa9ecf8e7 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -1,5 +1,6 @@ from sqlalchemy.orm import Session from datetime import datetime +from pandas import isna from db import Thing, PermissionHistory from transfers.util import read_csv, logger, replace_nans @@ -15,8 +16,10 @@ def transfer_permissions(session: Session): """ - The transferred wells and contacts need to be queried to know who gave - permission to which well since contact_id is required for PermissionHistory + The transferred wells and contacts need to be transferred first + - to access the auto-generated well IDs + - to know who gave permission to which well since contact_id is required for + PermissionHistory """ wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) wdf = replace_nans(wdf) @@ -38,7 +41,9 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + if len(allow_water_level_samples) > 0 and not isna( + allow_water_level_samples[0] + ): try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -61,9 +66,8 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if ( - len(allow_water_chemistry_samples) > 0 - and allow_water_chemistry_samples is not None + if len(allow_water_chemistry_samples) > 0 and not isna( + allow_water_chemistry_samples[0] ): try: permission_allowed = bool(allow_water_chemistry_samples[0]) From 2d0f2b6177980836ca7ba895ea7df72a3f13f5eb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 15:10:15 -0700 Subject: [PATCH 3/5] refactor: check for existence of permissions before converting --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index fa9ecf8e7..18daa1040 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -41,9 +41,11 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and not isna( - allow_water_level_samples[0] - ): + if len(allow_water_level_samples) == 0: + pass + elif isna(allow_water_level_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -66,9 +68,11 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if len(allow_water_chemistry_samples) > 0 and not isna( - allow_water_chemistry_samples[0] - ): + if len(allow_water_chemistry_samples) == 0: + pass + elif isna(allow_water_chemistry_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_chemistry_samples[0]) permission = PermissionHistory( From 64eb2b7d698d669f14fa04a4f211a552c74cc5cb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 15:22:06 -0700 Subject: [PATCH 4/5] refactor: use formation_completion_code in well schema and tests use this instead of listing out the formations through which the bore hole passes --- schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 9 +++++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 2ee35ad6c..ebc011466 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -254,6 +254,7 @@ class WellResponse(BaseThingResponse): measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None permissions: list[PermissionHistoryResponse] + formation_completion_code: FormationCode | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 5fca63bf0..a89eacd00 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -101,6 +101,7 @@ def add_well(context, session, location, name_num): well_pump_type="Submersible", well_pump_depth=8, is_suitable_for_datalogger=True, + formation_completion_code="000EXRV", ) session.add(well) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 5658f2ccc..8b00f7eb7 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -237,10 +237,11 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "geologic_formations" in context.water_well_data - assert context.water_well_data["geologic_formations"] == [ - context.objects["geologic_formations"][0].formation_code - ] + assert "formation_completion_code" in context.water_well_data + assert ( + context.water_well_data["formation_completion_code"] + == context.objects["wells"][0].formation_completion_code + ) @then( From 23c6ecc66133a91e71b447cbc4c62009a92d4bab Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 15:24:08 -0700 Subject: [PATCH 5/5] fix: remove outdated geologic_formations field from ThingResponse --- schemas/thing.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index ebc011466..7a7982494 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -249,7 +249,6 @@ class WellResponse(BaseThingResponse): measuring_point_height_unit: str = "ft" measuring_point_description: str | None aquifers: list[dict] = [] - geologic_formations: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -275,14 +274,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("geologic_formations", mode="before") - def populate_geologic_formations_with_strings(cls, geologic_formations): - if geologic_formations is not None: - formations = [formation.formation_code for formation in geologic_formations] - else: - formations = [] - return formations - @field_validator("permissions", mode="before") def populate_permission_history_with_latest_records(cls, permissions): """