From e20876849531d0e42260e8ebd2b4d82b302ff97b Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 09:46:45 -0700 Subject: [PATCH 1/7] feat: add open status and datalogger installation status to lexicon this will allow the refactor from fields to the status history since these statuses can change for a well over time --- core/lexicon.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index 90ead61b9..d18d0f678 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -335,12 +335,18 @@ {"categories": ["status_type"], "term": "Well Status", "definition": "Defines the well's operational condition as reported by the owner"}, {"categories": ["status_type"], "term": "Monitoring Status", "definition": "Defines the well's current monitoring status by NMBGMR."}, {"categories": ["status_type"], "term": "Access Status", "definition": "Defines the well's access status for field personnel."}, + {"categories": ["status_type"], "term": "Open Status", "definition": "Defines if the well is open or closed"}, + {"categories": ["status_type"], "term": "Datalogger Installation Status", "definition": "Defines if a datalogger can or cannot be installed at the well"}, {"categories": ["status_value"], "term": "Abandoned", "definition": "The well has been properly decommissioned."}, {"categories": ["status_value"], "term": "Active, pumping well", "definition": "This well is in use."}, {"categories": ["status_value"], "term": "Destroyed, exists but not usable", "definition": "The well structure is physically present but is damaged, collapsed, or otherwise compromised to the point that it is non-functional."}, {"categories": ["status_value"], "term": "Inactive, exists but not used", "definition": "The well is not currently in use but is believed to be in a usable condition; it has not been permanently decommissioned/abandoned."}, {"categories": ["status_value"], "term": "Currently monitored", "definition": "The well is currently being monitored by AMMP."}, {"categories": ["status_value"], "term": "Not currently monitored", "definition": "The well is not currently being monitored by AMMP."}, + {"categories": ["status_value"], "term": "Open", "definition": "The well is open."}, + {"categories": ["status_value"], "term": "Closed", "definition": "The well is closed."}, + {"categories": ["status_value"], "term": "Datalogger can be installed", "definition": "A datalogger can be installed at the well"}, + {"categories": ["status_value"], "term": "Datalogger cannot be installed", "definition": "A datalogger cannot be installed at the well"}, {"categories": ["sample_method"], "term": "Airline measurement", "definition": "Airline measurement"}, {"categories": ["sample_method"], "term": "Analog or graphic recorder", "definition": "Analog or graphic recorder"}, {"categories": ["sample_method"], "term": "Calibrated airline measurement", "definition": "Calibrated airline measurement"}, From c0c743e1947efdb86aaaa3384d7ac3055b1b424c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 13:43:36 -0700 Subject: [PATCH 2/7] refactor: use the nomenclature 'Datalogger Suitability Status' for clarity --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index d18d0f678..d25eae897 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -336,7 +336,7 @@ {"categories": ["status_type"], "term": "Monitoring Status", "definition": "Defines the well's current monitoring status by NMBGMR."}, {"categories": ["status_type"], "term": "Access Status", "definition": "Defines the well's access status for field personnel."}, {"categories": ["status_type"], "term": "Open Status", "definition": "Defines if the well is open or closed"}, - {"categories": ["status_type"], "term": "Datalogger Installation Status", "definition": "Defines if a datalogger can or cannot be installed at the well"}, + {"categories": ["status_type"], "term": "Datalogger Suitability Status", "definition": "Defines if a datalogger can or cannot be installed at the well"}, {"categories": ["status_value"], "term": "Abandoned", "definition": "The well has been properly decommissioned."}, {"categories": ["status_value"], "term": "Active, pumping well", "definition": "This well is in use."}, {"categories": ["status_value"], "term": "Destroyed, exists but not usable", "definition": "The well structure is physically present but is damaged, collapsed, or otherwise compromised to the point that it is non-functional."}, From a8718c6dcc12bb4f9ec246653b4ef74c477ae164 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 13:50:19 -0700 Subject: [PATCH 3/7] refactor: store open status and datalogger suitability status in status history table these statuses are changeable, so they should be in the status history table rather than as standalone fields in the thing table --- db/thing.py | 26 +++++++++++++++++ schemas/thing.py | 4 +-- tests/features/environment.py | 29 +++++++++++++++---- .../steps/well-additional-information.py | 13 ++++++--- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/db/thing.py b/db/thing.py index 35d7482ba..0c2754d6f 100644 --- a/db/thing.py +++ b/db/thing.py @@ -394,6 +394,32 @@ def monitoring_status(self) -> str | None: ) return latest_status.status_value if latest_status else None + @property + def open_status(self) -> str | None: + """ + Returns the open status from the most recent status history entry + where status_type is "Open Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Open Status" + ) + return latest_status.status_value if latest_status else None + + @property + def datalogger_suitability_status(self) -> str | None: + """ + Returns the datalogger installation status from the most recent status history entry + where status_type is "Datalogger Suitability Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Datalogger Suitability Status" + ) + return latest_status.status_value if latest_status else None + @property def measuring_point_height(self) -> int | None: """ diff --git a/schemas/thing.py b/schemas/thing.py index 9f2a084e3..a2b089089 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -138,7 +138,6 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None - is_suitable_for_datalogger: bool | None formation_completion_code: FormationCode | None = None @@ -238,8 +237,9 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" - is_suitable_for_datalogger: bool | None well_status: str | None + open_status: str | None + datalogger_suitability_status: str | None measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None diff --git a/tests/features/environment.py b/tests/features/environment.py index 123bc588f..64645d1c1 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -501,9 +501,9 @@ def add_geologic_formation(context, session, formation_code, well): def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True - erase_data = True + erase_data = False if rebuild: erase_and_rebuild_db() elif erase_data: @@ -581,14 +581,31 @@ def before_all(context): target_table="thing", ) - for value, start, end in ( - ("Currently monitored", datetime(2020, 1, 1), datetime(2021, 1, 1)), - ("Not currently monitored", datetime(2021, 1, 1), None), + for value, status_type, start, end in ( + ( + "Currently monitored", + "Monitoring Status", + datetime(2020, 1, 1), + datetime(2021, 1, 1), + ), + ( + "Not currently monitored", + "Monitoring Status", + datetime(2021, 1, 1), + None, + ), + ("Open", "Open Status", datetime(2020, 1, 1), None), + ( + "Datalogger can be installed", + "Datalogger Suitability Status", + datetime(2020, 1, 1), + None, + ), ): add_status_history( context, session, - status_type="Monitoring Status", + status_type=status_type, status_value=value, start_date=start, end_date=end, diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 8b00f7eb7..690068807 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -78,7 +78,7 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - permission_type = "Datalogger Installation" + permission_type = "Datalogger Suitability" assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( @@ -221,10 +221,15 @@ def step_impl(context): "the response should include whether the well is open and suitable for a datalogger" ) def step_impl(context): - assert "is_suitable_for_datalogger" in context.water_well_data + assert "datalogger_installation_status" in context.water_well_data + assert "open_status" in context.water_well_data assert ( - context.water_well_data["is_suitable_for_datalogger"] - == context.objects["wells"][0].is_suitable_for_datalogger + context.water_well_data["datalogger_installation_status"] + == context.objects["wells"][0].datalogger_installation_status + ) + assert ( + context.water_well_data["open_status"] + == context.objects["wells"][0].open_status ) From 5ef51df78c13690d0e16a1a562f8bee8466b5f48 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 13:51:48 -0700 Subject: [PATCH 4/7] refactor: transfer datalogger suitability to status history table --- transfers/well_transfer.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 02d6b1c69..b011a5991 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -279,10 +279,6 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): row, f"LU_ConstructionMethod:{row.ConstructionMethod}", "Unknown" ) - is_suitable_for_datalogger = False - if notna(row.OpenWellLoggerOK): - is_suitable_for_datalogger = bool(row.OpenWellLoggerOK) - mpheight = row.MPHeight mpheight_description = row.MeasuringPoint if mpheight is None: @@ -321,7 +317,6 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): well_driller_name=row.DrillerName, well_construction_method=wcm, well_pump_type=well_pump_type, - is_suitable_for_datalogger=is_suitable_for_datalogger, ) CreateWell.model_validate(data) @@ -659,6 +654,7 @@ def _process_chunk(chunk_index: int, wells_chunk: list[Thing]): try: session.bulk_save_objects(all_objects, return_defaults=False) session.commit() + print("ADDED AFTER HOOK OBJECTS TO DATABASE") except DatabaseError as e: session.rollback() self._capture_database_error("MultiplePointIDs", e) @@ -819,7 +815,6 @@ def _after_hook_chunk(self, well, formations): ) if notna(row.Status): - status_value = self._get_lexicon_value(row, f"LU_Status:{row.Status}") if status_value is not None: status_history = StatusHistory( @@ -835,6 +830,26 @@ def _after_hook_chunk(self, well, formations): logger.info( f" Added well status for well {well.name}: {status_value}" ) + + if notna(row.OpenWellLoggerOK): + if bool(row.OpenWellLoggerOK): + status_value = "Datalogger can be installed" + else: + status_value = "Datalogger cannot be installed" + status_history = StatusHistory( + status_type="Datalogger Suitability Status", + status_value=status_value, + reason=None, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + objs.append(status_history) + if self.verbose: + logger.info( + f" Added datalogger suitability status for well {well.name}: {status_value}" + ) + return objs From 2fc4493b7e344a449b8ff1e01ff973831f4209d0 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 15:25:40 -0700 Subject: [PATCH 5/7] feat: map open unequipped wells to status history this is a status of the well not a well purpose --- docker-compose.yml | 1 + transfers/well_transfer.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1c6dec4ef..30d22b9d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_HOST=db + - POSTGRES_PORT=5432 - MODE=${MODE} - AUTHENTIK_DISABLE_AUTHENTICATION=${AUTHENTIK_DISABLE_AUTHENTICATION} ports: diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index b011a5991..8a0ef30a4 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -420,6 +420,9 @@ def _extract_well_purposes(self, row) -> list[str]: else: purposes = [] for cui in cu: + if cui == "A": + # skip "Open, unequipped well" as that gets mapped to the status_history table + continue p = self._get_lexicon_value(row, f"LU_CurrentUse:{cui}") if p is not None: purposes.append(p) @@ -850,6 +853,19 @@ def _after_hook_chunk(self, well, formations): f" Added datalogger suitability status for well {well.name}: {status_value}" ) + if notna(row.CurrentUse) and "A" in row.CurrentUse: + status_history = StatusHistory( + status_type="Open Status", + status_value="Open", + reason=None, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + objs.append(status_history) + if self.verbose: + logger.info(f" Added open open status for well {well.name}") + return objs From 9cb7464f662aa814d2053a4a0ec00d76cd4d0daf Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 15:29:42 -0700 Subject: [PATCH 6/7] fix: permission should be datalogger installation not suitability in test --- tests/features/steps/well-additional-information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 690068807..8eecef159 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -78,7 +78,7 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - permission_type = "Datalogger Suitability" + permission_type = "Datalogger Installation" assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( From 1c1a050e04e10caaaedb4607662ce5f5f88039e9 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 12 Dec 2025 15:31:56 -0700 Subject: [PATCH 7/7] fix: remove print debugging error --- transfers/well_transfer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 8a0ef30a4..c1105d8b1 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -657,7 +657,6 @@ def _process_chunk(chunk_index: int, wells_chunk: list[Thing]): try: session.bulk_save_objects(all_objects, return_defaults=False) session.commit() - print("ADDED AFTER HOOK OBJECTS TO DATABASE") except DatabaseError as e: session.rollback() self._capture_database_error("MultiplePointIDs", e)