From 0ac45c14c80e6f7c9724a581d93c3a4e3911e59e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 4 Nov 2025 17:31:56 -0700 Subject: [PATCH 001/119] WIP: first stab at well additional information --- .../steps/well-additional-information.py | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tests/features/steps/well-additional-information.py diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py new file mode 100644 index 000000000..89f1c38d9 --- /dev/null +++ b/tests/features/steps/well-additional-information.py @@ -0,0 +1,167 @@ +from behave import when, then + + +@when("the user retrieves the well by ID via path parameter") +def step_impl_retrieve_well_by_id(context): + well_id = 1 + context.response = context.client.get(f"/thing/water-well/{well_id}") + + +# ------------------------------------------------------------------------------ +# Permissions / Operational OK flags +# ------------------------------------------------------------------------------ +# TODO: the API needs to be updated to include Permissions +# TODO: the schema and test data need to be updated +# TODO: should the testing data and tests contain multiple permissions, one that has expired? + + +@then( + "the response should include whether repeat measurement permission is granted for the well" +) +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_repeat_sampling"] is True + + +@then("the response should include whether sampling permission is granted for the well") +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_sampling"] is True + + +# TODO: should this be datalogger specific? +@then( + "the response should include whether datalogger installation permission is granted for the well" +) +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_installation"] is True + + +# ------------------------------------------------------------------------------ +# Well Construction Information +# ------------------------------------------------------------------------------ + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the completion date of the well") +def step_impl(context): + data = context.response.json() + assert data["completion_date"] == "2020-05-15" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the source of the completion information") +def step_impl(context): + data = context.response.json() + assert data["completion_info_source"] == "Driller Report" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the driller name") +def step_impl(context): + data = context.response.json() + assert data["driller_name"] == "John Doe" + + +# TODO: needs to be added to model, schemas, test data +# TODO: needs to be an enum and added to lexicon +@then("the response should include the construction method") +def step_impl(context): + data = context.response.json() + assert data["construction_method"] == "Rotary Drilling" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the source of the construction information") +def step_impl(context): + data = context.response.json() + assert data["construction_info_source"] == "Driller Report" + + +# ------------------------------------------------------------------------------ +# Additional Well Physical Properties +# ------------------------------------------------------------------------------ + + +# TODO: the transfer script needs to convert ft to in +@then("the response should include the casing diameter in inches") +def step_impl(context): + data = context.response.json() + assert data["casing_diameter"] == 10 + assert data["casing_diameter_unit"] == "in" + + +@then("the response should include the casing depth in feet below ground surface") +def step_impl(context): + data = context.response.json() + assert data["well_casing_depth"] == 30 + assert data["well_casing_depth_unit"] == "ft" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the casing description (previously casing notes field)" +) +def step_impl(context): + data = context.response.json() + assert data["well_casing_description"] == "test description" + + +# TODO: needs to be added to model, schemas, test data +# TODO: needs to be added to lexicon and an enum should be created +@then("the response should include the well pump type (previously well_type field)") +def step_impl(context): + data = context.response.json() + assert data["well_pump_type"] == "Submersible" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the well pump depth in feet (new field)") +def step_impl(context): + data = context.response.json() + assert data["well_pump_depth"] == 100 + assert data["well_pump_depth_unit"] == "ft" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include whether the well is open and suitable for a datalogger" +) +def step_impl(context): + data = context.response.json() + assert data["well_open"] is True + assert data["well_suitable_for_datalogger"] is True + + +# ------------------------------------------------------------------------------ +# Aquifer/ Geology Information +# ------------------------------------------------------------------------------ + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the formation as the formation zone of well completion" +) +def step_impl(context): + data = context.response.json() + assert data["formation"] == "Sandstone" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the aquifer class code to classify the aquifer into aquifer system." +) +def step_impl(context): + data = context.response.json() + assert data["aquifer_class_code"] == "A1" + + +# TODO: needs to be added to model, schemas, test data +# TODO: should this be plural? that is, a descriptor model of the well +@then( + "the response should include the aquifer type as the type of aquifers penetrated by the well" +) +def step_impl(context): + data = context.response.json() + assert data["aquifer_type"] == "Confined" From edf3c38096f101badc8dfe6abdf64f203daa5604 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 7 Nov 2025 16:19:08 -0700 Subject: [PATCH 002/119] WIP: well additional information bdd tests --- .../steps/well-additional-information.py | 127 ++++++++++++------ 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 89f1c38d9..c0f8949ca 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -3,8 +3,9 @@ @when("the user retrieves the well by ID via path parameter") def step_impl_retrieve_well_by_id(context): - well_id = 1 - context.response = context.client.get(f"/thing/water-well/{well_id}") + context.well = context.objects["wells"][0] + context.response = context.client.get(f"/thing/water-well/{context.well.id}") + context.data = context.response.json() # ------------------------------------------------------------------------------ @@ -13,20 +14,45 @@ def step_impl_retrieve_well_by_id(context): # TODO: the API needs to be updated to include Permissions # TODO: the schema and test data need to be updated # TODO: should the testing data and tests contain multiple permissions, one that has expired? +# TODO: what are the permission_types that will be used? after they have been determined update these tests @then( "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_repeat_sampling"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "water level measurement" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_water_level_measurements"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_sampling"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "water chemistry sample" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_sampling"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) # TODO: should this be datalogger specific? @@ -34,8 +60,20 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_installation"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "data logger installation" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_data_logger_installation"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) # ------------------------------------------------------------------------------ @@ -46,37 +84,42 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): - data = context.response.json() - assert data["completion_date"] == "2020-05-15" + assert "completion_date" in context.data + assert context.data["completion_date"] == context.well.completion_date.strftime( + "%Y-%m-%d" + ) # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): - data = context.response.json() - assert data["completion_info_source"] == "Driller Report" + assert "completion_info_source" in context.data + assert context.data["completion_info_source"] == context.well.completion_info_source # TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): - data = context.response.json() - assert data["driller_name"] == "John Doe" + assert "driller_name" in context.data + assert context.data["driller_name"] == context.well.driller_name # TODO: needs to be added to model, schemas, test data # TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): - data = context.response.json() - assert data["construction_method"] == "Rotary Drilling" + assert "construction_method" in context.data + assert context.data["construction_method"] == context.well.construction_method # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): - data = context.response.json() - assert data["construction_info_source"] == "Driller Report" + assert "construction_info_source" in context.data + assert ( + context.data["construction_info_source"] + == context.well.construction_info_source + ) # ------------------------------------------------------------------------------ @@ -87,16 +130,20 @@ def step_impl(context): # TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): - data = context.response.json() - assert data["casing_diameter"] == 10 - assert data["casing_diameter_unit"] == "in" + assert "casing_diameter" in context.data + assert "casing_diameter_unit" in context.data + + assert context.data["casing_diameter"] == context.well.casing_diameter + assert context.data["casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") def step_impl(context): - data = context.response.json() - assert data["well_casing_depth"] == 30 - assert data["well_casing_depth_unit"] == "ft" + assert "well_casing_depth" in context.data + assert "well_casing_depth_unit" in context.data + + assert context.data["well_casing_depth"] == context.well.well_casing_depth + assert context.data["well_casing_depth_unit"] == "ft" # TODO: needs to be added to model, schemas, test data @@ -104,24 +151,28 @@ def step_impl(context): "the response should include the casing description (previously casing notes field)" ) def step_impl(context): - data = context.response.json() - assert data["well_casing_description"] == "test description" + assert "well_casing_description" in context.data + assert ( + context.data["well_casing_description"] == context.well.well_casing_description + ) # TODO: needs to be added to model, schemas, test data # TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): - data = context.response.json() - assert data["well_pump_type"] == "Submersible" + assert "well_pump_type" in context.data + assert context.data["well_pump_type"] == context.well.well_pump_type # TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): - data = context.response.json() - assert data["well_pump_depth"] == 100 - assert data["well_pump_depth_unit"] == "ft" + assert "well_pump_depth" in context.data + assert "well_pump_depth_unit" in context.data + + assert context.data["well_pump_depth"] == context.well.well_pump_depth + assert context.data["well_pump_depth_unit"] == "ft" # TODO: needs to be added to model, schemas, test data @@ -144,17 +195,17 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - data = context.response.json() - assert data["formation"] == "Sandstone" + assert "formation" in context.data + assert context.data["formation"] == context.well.formation -# TODO: needs to be added to model, schemas, test data +# TODO: needs to be added to model, schemas, test data, lexicon @then( "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - data = context.response.json() - assert data["aquifer_class_code"] == "A1" + assert "aquifer_class_code" in context.data + assert context.data["aquifer_class_code"] == context.well.aquifer_class_code # TODO: needs to be added to model, schemas, test data @@ -163,5 +214,5 @@ def step_impl(context): "the response should include the aquifer type as the type of aquifers penetrated by the well" ) def step_impl(context): - data = context.response.json() - assert data["aquifer_type"] == "Confined" + assert "aquifer_type" in context.data + assert context.data["aquifer_type"] == context.well.aquifer_type From 7197edea3d47266e9f54817d52c0c2df72205070 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 10 Nov 2025 09:58:37 -0700 Subject: [PATCH 003/119] feat: use function to retrieve polymoprhic records --- services/util.py | 40 +++++++++++++++++++ .../steps/well-additional-information.py | 32 +++++---------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/services/util.py b/services/util.py index cb3d8826c..36c1bf7a6 100644 --- a/services/util.py +++ b/services/util.py @@ -5,6 +5,7 @@ import httpx from constants import SRID_WGS84 +from db import Base TRANSFORMERS = {} @@ -115,6 +116,45 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] +def retrieve_polymorphic_table_record( + target_record: Base, + polymorphic_relationship: str, + polymorphic_type: str, + latest=True, +) -> Base: + """ + Retrieve a record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. + + Parameters: + ---------- + target_record : Base + The parent record from which to retrieve the polymorphic child record. + + polymorphic_relationship : str + The name of the relationship attribute on the parent record that corresponds to the polymorphic table. + + polymorphic_type : str + The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). + + latest : bool, optional + If True, retrieves the latest record based on start_date. Defaults to True. + """ + if polymorphic_relationship == "permissions": + type_field = "permission_type" + elif polymorphic_relationship == "status_history": + type_field = "status_type" + + polymorphic_records = getattr(target_record, polymorphic_relationship) + type_polymorphic_records = [ + r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + ] + sorted_type_polymorphic_records = sorted( + type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + ) + return sorted_type_polymorphic_records[0] + + if __name__ == "__main__": x = -106.904107 y = 34.068198 diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index c0f8949ca..ead66efaf 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -1,5 +1,7 @@ from behave import when, then +from services.util import retrieve_polymorphic_table_record + @when("the user retrieves the well by ID via path parameter") def step_impl_retrieve_well_by_id(context): @@ -23,17 +25,13 @@ def step_impl_retrieve_well_by_id(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "water level measurement" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_water_level_measurements", latest=True ) assert ( context.data["permissions"]["allow_water_level_measurements"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) @@ -41,17 +39,13 @@ def step_impl(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "water chemistry sample" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_water_chemistry_sample", latest=True ) assert ( context.data["permissions"]["allow_sampling"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) @@ -62,17 +56,13 @@ def step_impl(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "data logger installation" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_data_logger_installation", latest=True ) assert ( context.data["permissions"]["allow_data_logger_installation"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) From 826a6b64c95ce19f14ee13b71045e652ccbf31ea Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 11:23:47 -0700 Subject: [PATCH 004/119] feat: pass bdd test for well completion date --- .pre-commit-config.yaml | 18 +++++++++--------- db/thing.py | 4 ++++ schemas/thing.py | 1 + tests/features/environment.py | 1 + .../steps/well-additional-information.py | 18 +++++++++++++----- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d74e6a6c..d708a9010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - - repo: local - hooks: - - id: pytest - name: pytest - entry: pytest # Or your specific test command, e.g., poetry run pytest - language: system - types: [python] # Specify relevant file types for your tests - pass_filenames: false - always_run: true + # - repo: local + # hooks: + # - id: pytest + # name: pytest + # entry: pytest # Or your specific test command, e.g., poetry run pytest + # language: system + # types: [python] # Specify relevant file types for your tests + # pass_filenames: false + # always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/db/thing.py b/db/thing.py index bedc4430d..8c7a2f9e6 100644 --- a/db/thing.py +++ b/db/thing.py @@ -101,6 +101,10 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) + well_completion_date: Mapped[str] = mapped_column( + Date, nullable=True, comment="the date the well was completed if known" + ) + # Spring-related columns spring_type: Mapped[str] = lexicon_term( nullable=True, diff --git a/schemas/thing.py b/schemas/thing.py index cd741c758..540c5484f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -153,6 +153,7 @@ class WellResponse(BaseThingResponse): well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None + well_completion_date: PastDate | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index ac97355c1..a792f18ed 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -78,6 +78,7 @@ def add_well(context, session, location, name_num): well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, + well_completion_date="2013-05-15", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index ead66efaf..475a4b5a5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -10,6 +10,15 @@ def step_impl_retrieve_well_by_id(context): context.data = context.response.json() +@then( + "null values in the response should be represented as JSON null (not placeholder strings)" +) +def step_impl(context): + for key, value in context.data.items(): + if value is None: + assert value is None # JSON null is represented as None in Python + + # ------------------------------------------------------------------------------ # Permissions / Operational OK flags # ------------------------------------------------------------------------------ @@ -71,13 +80,12 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): - assert "completion_date" in context.data - assert context.data["completion_date"] == context.well.completion_date.strftime( - "%Y-%m-%d" - ) + assert "well_completion_date" in context.data + assert context.data[ + "well_completion_date" + ] == context.well.well_completion_date.strftime("%Y-%m-%d") # TODO: needs to be added to model, schemas, test data From b34f62e52f930917f9fa05e1eede8c65594e3733 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 11:50:22 -0700 Subject: [PATCH 005/119] feat: pass well driller name bdd test --- db/thing.py | 3 +++ schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 4 ++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/db/thing.py b/db/thing.py index 8c7a2f9e6..c47b08811 100644 --- a/db/thing.py +++ b/db/thing.py @@ -104,6 +104,9 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_completion_date: Mapped[str] = mapped_column( Date, nullable=True, comment="the date the well was completed if known" ) + well_driller_name: Mapped[str] = mapped_column( + String(200), nullable=True, comment="Name of the well driller." + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 540c5484f..254643207 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -154,6 +154,7 @@ class WellResponse(BaseThingResponse): well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None well_completion_date: PastDate | None + well_driller_name: str | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index a792f18ed..b1e8da88f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -79,6 +79,7 @@ def add_well(context, session, location, name_num): well_casing_diameter=5.0, well_casing_depth=10.0, well_completion_date="2013-05-15", + well_driller_name="Jonsi", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 475a4b5a5..18b32ae28 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -98,8 +98,8 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): - assert "driller_name" in context.data - assert context.data["driller_name"] == context.well.driller_name + assert "well_driller_name" in context.data + assert context.data["well_driller_name"] == context.well.well_driller_name # TODO: needs to be added to model, schemas, test data From 3d89a3ecef303429646f4a53954d65c733bf782a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:01:43 -0700 Subject: [PATCH 006/119] feat: implement well construction method --- core/enums.py | 4 +++- core/lexicon.json | 18 +++++++++--------- db/thing.py | 1 + schemas/thing.py | 9 ++++++++- tests/features/environment.py | 1 + .../steps/well-additional-information.py | 7 +++++-- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/core/enums.py b/core/enums.py index 52e37d805..e8538ade3 100644 --- a/core/enums.py +++ b/core/enums.py @@ -24,7 +24,9 @@ ) CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material") CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method") -ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method") +WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category( + "well_construction_method" +) ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type") CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method") WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose") diff --git a/core/lexicon.json b/core/lexicon.json index f1a77ed24..d027c5400 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -4,7 +4,7 @@ {"name": "analysis_method_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, - {"name": "construction_method", "description": null}, + {"name": "well_construction_method", "description": null}, {"name": "contact_type", "description": null}, {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, @@ -75,14 +75,14 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, - {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, - {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, - {"categories": ["construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, - {"categories": ["construction_method"], "term": "Air percussion", "definition": "Air percussion"}, - {"categories": ["construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, - {"categories": ["construction_method"], "term": "Driven", "definition": "Driven"}, - {"categories": ["construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, + {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, + {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, + {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, + {"categories": ["well_construction_method"], "term": "Air percussion", "definition": "Air percussion"}, + {"categories": ["well_construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, + {"categories": ["well_construction_method"], "term": "Driven", "definition": "Driven"}, + {"categories": ["well_construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, {"categories": ["coordinate_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["coordinate_method"], "term": "Survey-grade global positioning system (SGPS)", "definition": "Survey-grade global positioning system (SGPS)"}, {"categories": ["coordinate_method"], "term": "GPS, uncorrected", "definition": "GPS, uncorrected"}, diff --git a/db/thing.py b/db/thing.py index c47b08811..3e5f3e7cc 100644 --- a/db/thing.py +++ b/db/thing.py @@ -107,6 +107,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_driller_name: Mapped[str] = mapped_column( String(200), nullable=True, comment="Name of the well driller." ) + well_construction_method = lexicon_term(nullable=True) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 254643207..38ed12b5a 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -17,7 +17,13 @@ from pydantic import BaseModel, model_validator, PastDate, Field, field_validator -from core.enums import WellPurpose, CasingMaterial, SpringType, ScreenType +from core.enums import ( + WellPurpose, + CasingMaterial, + SpringType, + ScreenType, + WellConstructionMethod, +) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -155,6 +161,7 @@ class WellResponse(BaseThingResponse): well_construction_notes: str | None = None well_completion_date: PastDate | None well_driller_name: str | None + well_construction_method: WellConstructionMethod | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index b1e8da88f..49786481f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -80,6 +80,7 @@ def add_well(context, session, location, name_num): well_casing_depth=10.0, well_completion_date="2013-05-15", well_driller_name="Jonsi", + well_construction_method="Driven", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 18b32ae28..b510d73af 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -106,8 +106,11 @@ def step_impl(context): # TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): - assert "construction_method" in context.data - assert context.data["construction_method"] == context.well.construction_method + assert "well_construction_method" in context.data + assert ( + context.data["well_construction_method"] + == context.well.well_construction_method + ) # TODO: needs to be added to model, schemas, test data From ea00c9c1c070c13d2895e94be554dab56a189172 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:27:27 -0700 Subject: [PATCH 007/119] feat: implement well casing diameter in inches --- tests/features/steps/well-additional-information.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index b510d73af..3159fef8b 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -95,15 +95,12 @@ def step_impl(context): assert context.data["completion_info_source"] == context.well.completion_info_source -# TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): assert "well_driller_name" in context.data assert context.data["well_driller_name"] == context.well.well_driller_name -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): assert "well_construction_method" in context.data @@ -131,11 +128,11 @@ def step_impl(context): # TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): - assert "casing_diameter" in context.data - assert "casing_diameter_unit" in context.data + assert "well_casing_diameter" in context.data + assert "well_casing_diameter_unit" in context.data - assert context.data["casing_diameter"] == context.well.casing_diameter - assert context.data["casing_diameter_unit"] == "in" + assert context.data["well_casing_diameter"] == context.well.well_casing_diameter + assert context.data["well_casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") From cd4e174c365aafb41f7b066973eb6027c92dbc30 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:39:03 -0700 Subject: [PATCH 008/119] feat: implement well pump type --- core/enums.py | 1 + core/lexicon.json | 10 ++++++++-- db/thing.py | 1 + schemas/thing.py | 2 ++ tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 1 - 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/enums.py b/core/enums.py index e8538ade3..5833d97bc 100644 --- a/core/enums.py +++ b/core/enums.py @@ -69,4 +69,5 @@ Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum") ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") +WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") # ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index d027c5400..cc9883168 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -47,7 +47,8 @@ {"name": "unit", "description": null}, {"name": "vertical_datum", "description": null}, {"name": "well_purpose", "description": null}, - {"name": "well_status", "description": null} + {"name": "well_status", "description": null}, + {"name": "well_pump_type", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -673,6 +674,11 @@ {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, - {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"} + {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, + {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, + {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} ] } \ No newline at end of file diff --git a/db/thing.py b/db/thing.py index 3e5f3e7cc..5e4c49fd6 100644 --- a/db/thing.py +++ b/db/thing.py @@ -108,6 +108,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix String(200), nullable=True, comment="Name of the well driller." ) well_construction_method = lexicon_term(nullable=True) + well_pump_type: Mapped[str] = lexicon_term(nullable=True) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 38ed12b5a..6d222b930 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -23,6 +23,7 @@ SpringType, ScreenType, WellConstructionMethod, + WellPumpType, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -162,6 +163,7 @@ class WellResponse(BaseThingResponse): well_completion_date: PastDate | None well_driller_name: str | None well_construction_method: WellConstructionMethod | None + well_pump_type: WellPumpType | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 49786481f..2337f2588 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -81,6 +81,7 @@ def add_well(context, session, location, name_num): well_completion_date="2013-05-15", well_driller_name="Jonsi", well_construction_method="Driven", + well_pump_type="Submersible", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 3159fef8b..a23924488 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -125,7 +125,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): assert "well_casing_diameter" in context.data From ca6b6c46d4a3eac88f84e48c7263ca7625778439 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:46:56 -0700 Subject: [PATCH 009/119] feat: implement well pump depth --- db/thing.py | 6 ++++++ schemas/thing.py | 3 +++ tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 3 --- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/db/thing.py b/db/thing.py index 5e4c49fd6..30cd6d6bb 100644 --- a/db/thing.py +++ b/db/thing.py @@ -109,6 +109,12 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix ) well_construction_method = lexicon_term(nullable=True) well_pump_type: Mapped[str] = lexicon_term(nullable=True) + well_pump_depth: Mapped[float] = mapped_column( + Float, + nullable=True, + info={"unit": "feet below ground surface"}, + comment="Depth of the well pump from ground surface to the pump intake (in feet).", + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 6d222b930..13ce97739 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -24,6 +24,7 @@ ScreenType, WellConstructionMethod, WellPumpType, + Unit, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -164,6 +165,8 @@ class WellResponse(BaseThingResponse): well_driller_name: str | None well_construction_method: WellConstructionMethod | None well_pump_type: WellPumpType | None + well_pump_depth: float | None + well_pump_depth_unit: Unit = "ft" @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 2337f2588..d7a54d5ab 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -82,6 +82,7 @@ def add_well(context, session, location, name_num): well_driller_name="Jonsi", well_construction_method="Driven", well_pump_type="Submersible", + well_pump_depth=8, ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index a23924488..031cb2349 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -154,15 +154,12 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.data assert context.data["well_pump_type"] == context.well.well_pump_type -# TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): assert "well_pump_depth" in context.data From e1df131c7eed1ac3045f5662c426eaf07cae1f5c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 08:55:14 -0700 Subject: [PATCH 010/119] fix: remove duplicate lexicon --- core/lexicon.json | 1 - 1 file changed, 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index cc9883168..d23272a4d 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -678,7 +678,6 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} ] } \ No newline at end of file From 7d3715404051709ff3b6baadab0252d7fc462358 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:33:03 -0700 Subject: [PATCH 011/119] refactor: test for casing materials --- .pre-commit-config.yaml | 18 +++++++++--------- .../steps/well-additional-information.py | 10 +++------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d74e6a6c..d708a9010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - - repo: local - hooks: - - id: pytest - name: pytest - entry: pytest # Or your specific test command, e.g., poetry run pytest - language: system - types: [python] # Specify relevant file types for your tests - pass_filenames: false - always_run: true + # - repo: local + # hooks: + # - id: pytest + # name: pytest + # entry: pytest # Or your specific test command, e.g., poetry run pytest + # language: system + # types: [python] # Specify relevant file types for your tests + # pass_filenames: false + # always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index ead66efaf..953f065b0 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -137,14 +137,10 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data -@then( - "the response should include the casing description (previously casing notes field)" -) +@then("the response should include the casing materials") def step_impl(context): - assert "well_casing_description" in context.data - assert ( - context.data["well_casing_description"] == context.well.well_casing_description - ) + assert "well_casing_materials" in context.data + assert context.data["well_casing_materials"] == context.well.well_casing_materials # TODO: needs to be added to model, schemas, test data From 08d7aedc8f198786a01c6cf90f133d3161c9b5ec Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:42:10 -0700 Subject: [PATCH 012/119] feat: update tests to include well casing materials --- schemas/thing.py | 3 +-- tests/features/environment.py | 18 ++++++++++++++++++ .../steps/well-additional-information.py | 4 +++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 13ce97739..d87fd299f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -24,7 +24,6 @@ ScreenType, WellConstructionMethod, WellPumpType, - Unit, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -166,7 +165,7 @@ class WellResponse(BaseThingResponse): well_construction_method: WellConstructionMethod | None well_pump_type: WellPumpType | None well_pump_depth: float | None - well_pump_depth_unit: Unit = "ft" + well_pump_depth_unit: str = "ft" @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index d7a54d5ab..effd332d1 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -29,6 +29,7 @@ Parameter, Deployment, TransducerObservationBlock, + WellCasingMaterial, ) from db.engine import session_ctx @@ -98,6 +99,20 @@ def add_well(context, session, location, name_num): return well +@add_context_object_container("well_casing_materials") +def add_well_casing_material(context, session, well): + wcm = WellCasingMaterial( + thing_id=well.id, + material="PVC", + ) + session.add(wcm) + session.commit() + session.refresh(wcm) + + context.objects["well_casing_materials"].append(wcm) + return wcm + + @add_context_object_container("springs") def add_spring(context, session, location, name_num): spring = Thing( @@ -214,6 +229,8 @@ def before_all(context): sensor_1 = add_sensor(context, session, well_1.id) deployment = add_deployment(context, session, well_1.id, sensor_1.id) + add_well_casing_material(context, session, well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) add_obs = add_block(context, session, parameter) @@ -227,6 +244,7 @@ def before_all(context): ) session.add(obs) session.commit() + session.refresh(well_1) def after_all(context): diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index d9943c681..e606d95ff 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -147,7 +147,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.data - assert context.data["well_casing_materials"] == context.well.well_casing_materials + assert sorted(context.data["well_casing_materials"]) == sorted( + [m.material for m in context.well.well_casing_materials] + ) @then("the response should include the well pump type (previously well_type field)") From c0b80fb6e02135c479fa2a2367f439697855a069 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:42:42 -0700 Subject: [PATCH 013/119] fix: fix casing materials test --- tests/features/steps/well-additional-information.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 953f065b0..02aa4ff22 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -140,7 +140,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.data - assert context.data["well_casing_materials"] == context.well.well_casing_materials + assert sorted(context.data["well_casing_materials"]) == sorted( + [m.material for m in context.well.well_casing_materials] + ) # TODO: needs to be added to model, schemas, test data From 878ef3bb1dade77a6969805267636aeadb849785 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 08:30:33 -0700 Subject: [PATCH 014/119] fix: uncomment pytest from pre commit --- .pre-commit-config.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d708a9010..5d74e6a6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - # - repo: local - # hooks: - # - id: pytest - # name: pytest - # entry: pytest # Or your specific test command, e.g., poetry run pytest - # language: system - # types: [python] # Specify relevant file types for your tests - # pass_filenames: false - # always_run: true + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest # Or your specific test command, e.g., poetry run pytest + language: system + types: [python] # Specify relevant file types for your tests + pass_filenames: false + always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference From 13f2a45f8cdb2297c4eacb5bd33016a363fbb6f8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:25:38 -0700 Subject: [PATCH 015/119] refactor: rename Permission -> PermissionHistory | move mixin to same file --- db/base.py | 19 -------- db/permission.py | 82 ---------------------------------- db/permission_history.py | 96 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 101 deletions(-) delete mode 100644 db/permission.py create mode 100644 db/permission_history.py diff --git a/db/base.py b/db/base.py index ba2a45be8..65d8de0dc 100644 --- a/db/base.py +++ b/db/base.py @@ -191,25 +191,6 @@ def status_history(self): ) -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index 340e587f7..000000000 --- a/db/permission.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/permission_history.py b/db/permission_history.py new file mode 100644 index 000000000..61c93eaae --- /dev/null +++ b/db/permission_history.py @@ -0,0 +1,96 @@ +""" +models/permission.py + +This model defines the `Permission` table, a polymorphic table that tracks +all legal and administrative agreements related to site access and activity. +Its purpose is to track who granted permission, what activities they authorized, +which entity the permission applies to, and for what period of time. +""" + +from typing import TYPE_CHECKING +from datetime import date +from sqlalchemy import Integer, ForeignKey, String, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake + + +if TYPE_CHECKING: + from db.contact import Contact + from db.thing import Thing + from db.location import Location + + +class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a specific grant of permission from a Contact for a + specific entity (e.g., a Thing or Location). + """ + + # --- Foreign Keys --- + contact_id: Mapped[int] = mapped_column( + Integer, ForeignKey("contact.id"), nullable=False + ) + + # --- Columns --- + permission_type: Mapped[str] = lexicon_term(nullable=False) + permission_allowed: Mapped[bool] = mapped_column(nullable=False, default=False) + start_date: Mapped[date] = mapped_column(nullable=False) + end_date: Mapped[date] = mapped_column(nullable=True) + notes: Mapped[str] = mapped_column(nullable=True) + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column(nullable=False) + target_table: Mapped[str] = mapped_column(String(50), nullable=False) + + # --- Relationships --- + # Many-To-One: A Permission is granted by one Contact. + contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to find the specific parent record for a given child. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin="and_(foreign(PermissionHistory.target_id) == thing.id, " + "PermissionHistory.target_table == 'thing')", + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin="and_(foreign(PermissionHistory.target_id) == location.id, " + "PermissionHistory.target_table == 'location')", + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.target_table}_target") + + +class PermissionHistoryMixin: + """ + Mixin for models that can have permissions (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + Permission table. + """ + + @declared_attr + def permissions(self): + # One-to-Many polymorphic relationship + return relationship( + "PermissionHistory", + primaryjoin=( + and_( + self.__name__.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == pascal_to_snake(self.__name__), + ) + ), + lazy="selectin", + viewonly=True, + ) From 6e036e5fb3b55276396a535c1e3fb739f00b66fc Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:26:03 -0700 Subject: [PATCH 016/119] feat: add permission types to lexicon --- core/lexicon.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index d23272a4d..e1e87a316 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -49,6 +49,7 @@ {"name": "well_purpose", "description": null}, {"name": "well_status", "description": null}, {"name": "well_pump_type", "description": null} + {"name": "permission_type", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -679,5 +680,8 @@ {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, + {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file From 091637c2adbd1922c0f9527acad71933a19ca556 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:29:29 -0700 Subject: [PATCH 017/119] refactor: import PermissionHistoryMixin from correct dir --- db/thing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db/thing.py b/db/thing.py index 25506cb95..9dd0d6de3 100644 --- a/db/thing.py +++ b/db/thing.py @@ -27,8 +27,8 @@ Base, ReleaseMixin, StatusHistoryMixin, - PermissionMixin, ) +from db.permission_history import PermissionHistoryMixin if TYPE_CHECKING: from db.location import Location @@ -39,7 +39,9 @@ from db.group import Group, GroupThingAssociation -class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMixin): +class Thing( + Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionHistoryMixin +): """ Represents a physical object of interest being monitored (e.g., a well). Stores static, core attributes of the physical installation. From 71d8534a73e58c34b0164d96e2e5910089d2b9b5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:32:05 -0700 Subject: [PATCH 018/119] fix: fix imports for newly renamed PermissionHistory --- db/__init__.py | 2 +- db/contact.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/__init__.py b/db/__init__.py index efb23a418..f61a39dca 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -32,7 +32,7 @@ from db.location import * from db.observation import * from db.parameter import * -from db.permission import * +from db.permission_history import * from db.publication import * from db.regulatory_limit import * from db.sample import * diff --git a/db/contact.py b/db/contact.py index 7855814fb..eb15af848 100644 --- a/db/contact.py +++ b/db/contact.py @@ -26,7 +26,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission import Permission + from db.permission_history import PermissionHistory class ThingContactAssociation(Base, AutoBaseMixin): @@ -74,7 +74,7 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["Permission"]] = relationship( + permissions: Mapped[List["PermissionHistory"]] = relationship( "Permission", back_populates="contact", cascade="all, delete, delete-orphan" ) # One-To-Many: A Contact can be associated with many Authors (in Publications). From 825ec4a145d77a2eeb5ec4b85982e7e028217620 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:02:30 -0700 Subject: [PATCH 019/119] feat: add permission_type to lexicon --- core/enums.py | 1 + core/lexicon.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/enums.py b/core/enums.py index 5833d97bc..25c6d746b 100644 --- a/core/enums.py +++ b/core/enums.py @@ -70,4 +70,5 @@ ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type") # ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index e1e87a316..1256aeca9 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -48,7 +48,7 @@ {"name": "vertical_datum", "description": null}, {"name": "well_purpose", "description": null}, {"name": "well_status", "description": null}, - {"name": "well_pump_type", "description": null} + {"name": "well_pump_type", "description": null}, {"name": "permission_type", "description": null} ], "terms": [ @@ -679,7 +679,7 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} From 8ed0ce444c2a1e1e810d4aaf7f6103b490c95649 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:08:34 -0700 Subject: [PATCH 020/119] feat: update util to correspond with bdms 221 --- tests/__init__.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index ed7fe4ea8..5f06309dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -113,42 +113,4 @@ def cleanup_patch_test(model: Base, payload: dict, original_data: Base) -> None: session.commit() -def retrieve_latest_polymorphic_table_record( - target_record: Base, - polymorphic_relationship: str, - polymorphic_type: str, -) -> Base: - """ - Retrieve the latest record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. This - requires end_date to be None - - Parameters: - ---------- - target_record : Base - The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str - The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str - The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional - If True, retrieves the latest record based on start_date. Defaults to True. - """ - if polymorphic_relationship == "permissions": - type_field = "permission_type" - elif polymorphic_relationship == "status_history": - type_field = "status_type" - - polymorphic_records = getattr(target_record, polymorphic_relationship) - type_polymorphic_records = [ - r - for r in polymorphic_records - if getattr(r, type_field) == polymorphic_type and r.end_date is None - ] - sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=True - ) - return sorted_type_polymorphic_records[0] - - # ============= EOF ============================================= From 3b36c49afd1d398cce764f87d48457163e3beb9d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:35:49 -0700 Subject: [PATCH 021/119] fix: sync bdms 221/227 for util functions --- core/lexicon.json | 2 +- db/contact.py | 4 +- db/permission_history.py | 6 +- services/util.py | 51 ++++++++--- tests/features/environment.py | 88 +++++++++++++++++++ .../steps/well-additional-information.py | 19 ++-- 6 files changed, 138 insertions(+), 32 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 1256aeca9..153dacded 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -681,7 +681,7 @@ {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, - {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file diff --git a/db/contact.py b/db/contact.py index eb15af848..558724df9 100644 --- a/db/contact.py +++ b/db/contact.py @@ -75,7 +75,9 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): # One-To-Many: A Contact can grant many Permissions. permissions: Mapped[List["PermissionHistory"]] = relationship( - "Permission", back_populates="contact", cascade="all, delete, delete-orphan" + "PermissionHistory", + back_populates="contact", + cascade="all, delete, delete-orphan", ) # One-To-Many: A Contact can be associated with many Authors (in Publications). author_associations: Mapped[List["AuthorContactAssociation"]] = relationship( diff --git a/db/permission_history.py b/db/permission_history.py index 61c93eaae..5faa3e3fa 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -81,14 +81,14 @@ class PermissionHistoryMixin: """ @declared_attr - def permissions(self): + def permissions(cls): # One-to-Many polymorphic relationship return relationship( "PermissionHistory", primaryjoin=( and_( - self.__name__.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == pascal_to_snake(self.__name__), + cls.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == pascal_to_snake(cls.__name__), ) ), lazy="selectin", diff --git a/services/util.py b/services/util.py index 36c1bf7a6..c5edee30e 100644 --- a/services/util.py +++ b/services/util.py @@ -3,11 +3,13 @@ from shapely.ops import transform import pyproj import httpx +from sqlalchemy.orm import DeclarativeBase from constants import SRID_WGS84 -from db import Base + TRANSFORMERS = {} +METERS_TO_FEET = 3.28084 def transform_srid(geometry, source_srid, target_srid): @@ -27,6 +29,20 @@ def transform_srid(geometry, source_srid, target_srid): return transform(transformer.transform, geometry) +def convert_m_to_ft(meters: float | None) -> float | None: + """Convert a length from meters to feet.""" + if meters is None: + return None + return round(meters * METERS_TO_FEET, 6) + + +def convert_ft_to_m(feet: float | None) -> float | None: + """Convert a length from feet to meters.""" + if feet is None: + return None + return round(feet / METERS_TO_FEET, 6) + + def get_tiger_data( lon: float, lat: float, layer: int, outfields: str = "*" ) -> dict | None: @@ -116,29 +132,31 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_polymorphic_table_record( - target_record: Base, +def retrieve_latest_polymorphic_table_record( + target_record: DeclarativeBase, polymorphic_relationship: str, polymorphic_type: str, - latest=True, -) -> Base: +) -> DeclarativeBase | None: """ - Retrieve a record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. + Retrieve the latest record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. This + requires end_date to be None Parameters: ---------- - target_record : Base + target_record : DeclarativeBase The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional If True, retrieves the latest record based on start_date. Defaults to True. + + Returns + ------- + DeclarativeBase | None + The latest record from the specified polymorphic table with the defined type if it exists. """ if polymorphic_relationship == "permissions": type_field = "permission_type" @@ -147,12 +165,17 @@ def retrieve_polymorphic_table_record( polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ - r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + r + for r in polymorphic_records + if getattr(r, type_field) == polymorphic_type and r.end_date is None ] sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + type_polymorphic_records, key=lambda r: r.start_date, reverse=True ) - return sorted_type_polymorphic_records[0] + if sorted_type_polymorphic_records: + return sorted_type_polymorphic_records[0] + else: + return None if __name__ == "__main__": diff --git a/tests/features/environment.py b/tests/features/environment.py index effd332d1..b27391736 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -30,6 +30,8 @@ Deployment, TransducerObservationBlock, WellCasingMaterial, + PermissionHistory, + Contact, ) from db.engine import session_ctx @@ -139,6 +141,54 @@ def add_spring(context, session, location, name_num): return spring +@add_context_object_container("contacts") +def add_contact(context, session): + contact = Contact( + name="Test Contact", + role="Field Technician", + organization="NMBGMR", + release_status="draft", + contact_type="Primary", + ) + session.add(contact) + session.commit() + session.refresh(contact) + + context.objects["contacts"].append(contact) + return contact + + +@add_context_object_container("permission_histories") +def add_permission_history( + context, + session, + contact_id, + permission_type, + permission_allowed, + start_date, + end_date, + notes, + target_id, + target_table, +): + permission_history = PermissionHistory( + contact_id=contact_id, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=end_date, + notes=notes, + target_id=target_id, + target_table=target_table, + ) + session.add(permission_history) + session.commit() + session.refresh(permission_history) + + context.objects["permission_histories"].append(permission_history) + return permission_history + + @add_context_object_container("sensors") def add_sensor(context, session, sid): sensor = Sensor( @@ -231,6 +281,44 @@ def before_all(context): add_well_casing_material(context, session, well_1) + add_contact(context, session) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Datalogger Installation", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for datalogger installation.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Water Level Sample", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for water level sampling.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Chemistry Sample", + permission_allowed=False, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for chemistry sampling.", + target_id=well_1.id, + target_table="thing", + ) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) add_obs = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index e606d95ff..e57705553 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -22,12 +22,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ # Permissions / Operational OK flags # ------------------------------------------------------------------------------ -# TODO: the API needs to be updated to include Permissions -# TODO: the schema and test data need to be updated -# TODO: should the testing data and tests contain multiple permissions, one that has expired? -# TODO: what are the permission_types that will be used? after they have been determined update these tests - - @then( "the response should include whether repeat measurement permission is granted for the well" ) @@ -35,11 +29,11 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_level_measurements", latest=True + context.well, "permissions", "Water Level Sample", latest=True ) assert ( - context.data["permissions"]["allow_water_level_measurements"] + context.data["permissions"]["allow_water_level_samples"] == permission_record.permission_allowed ) @@ -49,16 +43,15 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_chemistry_sample", latest=True + context.well, "permissions", "Water Chemistry Sample", latest=True ) assert ( - context.data["permissions"]["allow_sampling"] + context.data["permissions"]["allow_water_chemistry_samples"] == permission_record.permission_allowed ) -# TODO: should this be datalogger specific? @then( "the response should include whether datalogger installation permission is granted for the well" ) @@ -66,11 +59,11 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_data_logger_installation", latest=True + context.well, "permissions", "Datalogger Installation", latest=True ) assert ( - context.data["permissions"]["allow_data_logger_installation"] + context.data["permissions"]["allow_datalogger_installation"] == permission_record.permission_allowed ) From 1657bb9f1730c2dcea8571e12350c889dd904594 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 15:27:31 -0700 Subject: [PATCH 022/119] feat: implement permissions --- db/permission_history.py | 8 ++--- db/thing.py | 31 +++++++++++++++++++ schemas/thing.py | 3 ++ services/util.py | 3 +- tests/features/environment.py | 14 +++++---- .../steps/well-additional-information.py | 28 ++++++++--------- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/db/permission_history.py b/db/permission_history.py index 5faa3e3fa..7c9c37159 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -29,7 +29,7 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # --- Foreign Keys --- contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) # --- Columns --- @@ -52,13 +52,13 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # They tell SQLAlchemy exactly how to find the specific parent record for a given child. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin="and_(foreign(PermissionHistory.target_id) == thing.id, " + primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " "PermissionHistory.target_table == 'thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin="and_(foreign(PermissionHistory.target_id) == location.id, " + primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " "PermissionHistory.target_table == 'location')", viewonly=True, ) @@ -81,7 +81,7 @@ class PermissionHistoryMixin: """ @declared_attr - def permissions(cls): + def permission_history(cls): # One-to-Many polymorphic relationship return relationship( "PermissionHistory", diff --git a/db/thing.py b/db/thing.py index 9dd0d6de3..a6eab4aa5 100644 --- a/db/thing.py +++ b/db/thing.py @@ -29,6 +29,7 @@ StatusHistoryMixin, ) from db.permission_history import PermissionHistoryMixin +from services.util import retrieve_latest_polymorphic_table_record if TYPE_CHECKING: from db.location import Location @@ -291,6 +292,36 @@ def current_location(self): else None ) + @property + def allow_water_level_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Water Level Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_water_chemistry_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Water Chemistry Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_datalogger_installation(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Datalogger Installation" + ) + return permission_record.permission_allowed if permission_record else None + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/thing.py b/schemas/thing.py index d87fd299f..dea04430a 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -166,6 +166,9 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" + allow_water_level_samples: bool | None + allow_water_chemistry_samples: bool | None + allow_datalogger_installation: bool | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/services/util.py b/services/util.py index c5edee30e..0d618dfff 100644 --- a/services/util.py +++ b/services/util.py @@ -158,11 +158,10 @@ def retrieve_latest_polymorphic_table_record( DeclarativeBase | None The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permissions": + if polymorphic_relationship == "permission_history": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" - polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ r diff --git a/tests/features/environment.py b/tests/features/environment.py index b27391736..3fcc4930e 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -145,7 +145,7 @@ def add_spring(context, session, location, name_num): def add_contact(context, session): contact = Contact( name="Test Contact", - role="Field Technician", + role="Software Developer", organization="NMBGMR", release_status="draft", contact_type="Primary", @@ -281,11 +281,11 @@ def before_all(context): add_well_casing_material(context, session, well_1) - add_contact(context, session) + contact = add_contact(context, session) add_permission_history( context, session, - contact_id=context.objects["contacts"][0].id, + contact_id=contact.id, permission_type="Datalogger Installation", permission_allowed=True, start_date=datetime(2025, 1, 1).date(), @@ -310,7 +310,7 @@ def before_all(context): context, session, contact_id=context.objects["contacts"][0].id, - permission_type="Chemistry Sample", + permission_type="Water Chemistry Sample", permission_allowed=False, start_date=datetime(2025, 1, 1).date(), end_date=None, @@ -338,8 +338,10 @@ def before_all(context): def after_all(context): with session_ctx() as session: for table in context.objects.values(): - for obj in table: - session.delete(obj) + for record in table: + obj = session.get(record.__class__, record.id) + if obj: + session.delete(obj) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index e57705553..80d6389da 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -1,6 +1,6 @@ from behave import when, then -from services.util import retrieve_polymorphic_table_record +from services.util import retrieve_latest_polymorphic_table_record @when("the user retrieves the well by ID via path parameter") @@ -26,28 +26,26 @@ def step_impl(context): "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data - - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Water Level Sample", latest=True + assert "allow_water_level_samples" in context.data + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Water Level Sample" ) - assert ( - context.data["permissions"]["allow_water_level_samples"] + context.data["allow_water_level_samples"] == permission_record.permission_allowed ) @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - assert "permissions" in context.data + assert "allow_water_chemistry_samples" in context.data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Water Chemistry Sample", latest=True + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Water Chemistry Sample" ) assert ( - context.data["permissions"]["allow_water_chemistry_samples"] + context.data["allow_water_chemistry_samples"] == permission_record.permission_allowed ) @@ -56,14 +54,14 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data + assert "allow_datalogger_installation" in context.data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Datalogger Installation", latest=True + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Datalogger Installation" ) assert ( - context.data["permissions"]["allow_datalogger_installation"] + context.data["allow_datalogger_installation"] == permission_record.permission_allowed ) From 307f47cb5092fc76ffa24ce3610f2db1095698f4 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 16:18:53 -0700 Subject: [PATCH 023/119] feat: add is_suitable_for_datalogger to thing --- db/thing.py | 5 +++++ schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 9 +++++---- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/db/thing.py b/db/thing.py index a6eab4aa5..22f106999 100644 --- a/db/thing.py +++ b/db/thing.py @@ -118,6 +118,11 @@ class Thing( info={"unit": "feet below ground surface"}, comment="Depth of the well pump from ground surface to the pump intake (in feet).", ) + # TODO: should this be required for every well in the database? AMMP review + is_suitable_for_datalogger: Mapped[bool] = mapped_column( + nullable=True, + comment="Indicates if the well is suitable for datalogger installation.", + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index dea04430a..3e8c07e68 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -169,6 +169,7 @@ class WellResponse(BaseThingResponse): allow_water_level_samples: bool | None allow_water_chemistry_samples: bool | None allow_datalogger_installation: bool | None + is_suitable_for_datalogger: bool | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 3fcc4930e..73db99810 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -86,6 +86,7 @@ def add_well(context, session, location, name_num): well_construction_method="Driven", well_pump_type="Submersible", well_pump_depth=8, + is_suitable_for_datalogger=True, ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 80d6389da..c4b071fd5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -158,14 +158,15 @@ def step_impl(context): assert context.data["well_pump_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then( "the response should include whether the well is open and suitable for a datalogger" ) def step_impl(context): - data = context.response.json() - assert data["well_open"] is True - assert data["well_suitable_for_datalogger"] is True + assert "is_suitable_for_datalogger" in context.data + assert ( + context.data["is_suitable_for_datalogger"] + == context.well.is_suitable_for_datalogger + ) # ------------------------------------------------------------------------------ From 79235515254ad10335be5675f2cba33bd2f8fe18 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 15:25:17 -0700 Subject: [PATCH 024/119] feat: add 'AquiferSystem' model with relationships and controlled vocabulary fields - Implemented `AquiferSystem` model for master reference of aquifer systems and hydrogeologic units - Added table index for aquifer system name - Included categories and terms for aquifer type and significance level --- core/lexicon.json | 17 ++++++++++++++++- db/aquifer_system.py | 0 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 db/aquifer_system.py diff --git a/core/lexicon.json b/core/lexicon.json index 12df72c94..5b45332d8 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2,6 +2,7 @@ {"name": "activity_type", "description": null}, {"name": "address_type", "description": null}, {"name": "analysis_method_type", "description": null}, + {"name": "aquifer_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, {"name": "well_construction_method", "description": null}, @@ -18,6 +19,7 @@ {"name": "email_type", "description": null}, {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, + {"name": "geographic_scale", "description": null}, {"name": "groundwater_level_reason", "description": null}, {"name": "group_type", "description": null}, {"name": "horizontal_datum", "description": null}, @@ -678,6 +680,19 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, + {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Fractured", "definition": "Fractured"}, + {"categories": ["aquifer_type"], "term": "Mixed", "definition": "Mix of confined and unconfined aquifers"}, + {"categories": ["aquifer_type"], "term": "Perched", "definition": "Perched"}, + {"categories": ["aquifer_type"], "term": "Semi-confined", "definition": "Semi-confined"}, + {"categories": ["aquifer_type"], "term": "Unconfined, single", "definition": "Unconfined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Unconfined, multiple", "definition": "Unconfined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, + {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, + {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, + {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"} ] } \ No newline at end of file diff --git a/db/aquifer_system.py b/db/aquifer_system.py new file mode 100644 index 000000000..e69de29bb From 4ddec89de36cac67d2122cfae08b721d89ed0983 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 15:57:59 -0700 Subject: [PATCH 025/119] refactor: remove completed TODO about lexicon updates --- db/aquifer_system.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index e69de29bb..32da2e42a 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -0,0 +1,73 @@ +""" +SQLAlchemy model for the AquiferSystem table. + +This is a master reference table for aquifer systems and hydrogeologic units. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +if TYPE_CHECKING: + from db.thing import ( + WellScreen, + ThingAquiferAssociation, + ) # TODO: Add ThingAquiferAssociation class/model to Thing model. + + +class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The full, human-readable name of the aquifer system (e.g., 'Ogallala Aquifer').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the aquifer system, its characteristics, and its significance.", + ) + # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. + aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to classify the aquifer's hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + ) + geographic_scale: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + nullable=True, + comment="A spatial representation of the aquifer system's boundary.", + ) + # Hierarchical relationship fields (may be implemented in future iterations) + # Example: High Plains Aquifer (parent) contains Ogallala Aquifer (child) + # parent_id = Column(Integer, ForeignKey('aquifer_system.id')) + # parent = relationship('AquiferSystem', remote_side=[id], backref='subsystems') + + # --- Relationships --- + # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table. + things: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: An AquiferSystem can be the target for many individual WellScreens. + well_screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # --- Table Arguments --- + __table_args__ = Index("ix_aquifersystem_name", "name") From 29d7b910482c6cde9ce177b12042eb4fc8e5db52 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 16:37:26 -0700 Subject: [PATCH 026/119] refactor: clarify relationship name and add association proxy Added an association proxy, `things`, to the `AquiferSystem` model. This provides direct access to the `Thing` objects. --- db/aquifer_system.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 32da2e42a..c0710b687 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -8,16 +8,14 @@ from sqlalchemy import Text, Index from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from geoalchemy2 import Geometry from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term if TYPE_CHECKING: - from db.thing import ( - WellScreen, - ThingAquiferAssociation, - ) # TODO: Add ThingAquiferAssociation class/model to Thing model. + from db.thing import WellScreen, ThingAquiferAssociation, Thing class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): @@ -54,7 +52,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): # --- Relationships --- # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table. - things: Mapped[List["ThingAquiferAssociation"]] = relationship( + thing_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( "ThingAquiferAssociation", back_populates="aquifer_system", cascade="all, delete-orphan", @@ -69,5 +67,11 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): passive_deletes=True, ) + # --- Association Proxies --- + # Many-To-Many Proxy: Provides direct access to the Thing objects. + things: AssociationProxy[List["Thing"]] = association_proxy( + "thing_associations", "thing" + ) + # --- Table Arguments --- __table_args__ = Index("ix_aquifersystem_name", "name") From 0d3f807c131aa902e62facc475818ce97b241a80 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 16:41:50 -0700 Subject: [PATCH 027/119] refactor: clarify proxy purpose doc statement --- db/aquifer_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index c0710b687..fd121cd7b 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -68,7 +68,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): ) # --- Association Proxies --- - # Many-To-Many Proxy: Provides direct access to the Thing objects. + # Proxy to directly access Things (wells) associated with this AquiferSystem. things: AssociationProxy[List["Thing"]] = association_proxy( "thing_associations", "thing" ) From c3393d9abc41b8568fc53b936140dbc7aa6b94f3 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:10:37 -0700 Subject: [PATCH 028/119] feat: add ThingAquiferAssociation model and update Thing relationships - Created ThingAquiferAssociation model to manage many-to-many relationships between Thing and AquiferSystem - Updated Thing model to include aquifer_associations relationship for linking to aquifer systems --- db/thing.py | 9 +++++++ db/thing_aquifer_association.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 db/thing_aquifer_association.py diff --git a/db/thing.py b/db/thing.py index df0ef8611..03902c866 100644 --- a/db/thing.py +++ b/db/thing.py @@ -39,6 +39,7 @@ from db.sensor import Sensor from db.contact import Contact from db.group import Group, GroupThingAssociation + from db.thing_aquifer_association import ThingAquiferAssociation class Thing( @@ -262,6 +263,14 @@ class Thing( passive_deletes=True, ) + # One-To-Many: A Thing can be associated with many AquiferSystems via the ThingAquiferAssociation join table. + aquifer_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py new file mode 100644 index 000000000..5f14696ca --- /dev/null +++ b/db/thing_aquifer_association.py @@ -0,0 +1,42 @@ +""" +SQLAlchemy model for the ThingAquiferAssociation table. + +This table is a join table (or "association object") whose purpose is to manage +the many-to-many relationship between a Thing and an AquiferSystem. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey + +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.aquifer_system import AquiferSystem + + +class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the association of a Thing to an AquiferSystem. This is an Association Object. + """ + + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="CASCADE"), nullable=False + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="aquifer_associations" + ) + + # Many-To-One: This association links to one AquiferSystem. + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="thing_associations" + ) From af092ddac74090a56caceb5995e832f705a6d3a6 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:15:55 -0700 Subject: [PATCH 029/119] feat: add `aqufiers` proxy to Thing model Added an `aquifers` proxy to the Thing model to directly access the AqufierSystems associated with a particular Thing. --- db/thing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/db/thing.py b/db/thing.py index 03902c866..246d9536b 100644 --- a/db/thing.py +++ b/db/thing.py @@ -39,6 +39,7 @@ from db.sensor import Sensor from db.contact import Contact from db.group import Group, GroupThingAssociation + from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation @@ -296,6 +297,11 @@ class Thing( "group_associations", "group" ) + # Proxy to directly access AquiferSystems associated with this Thing + aquifers: AssociationProxy[List["AquiferSystem"]] = association_proxy( + "aquifer_associations", "aquifer_system" + ) + # Full-text search vector search_vector = Column(TSVectorType("name", "well_construction_notes")) From e330df46125468efa93fcff0d7f578520ce90a04 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:55:39 -0700 Subject: [PATCH 030/119] feat: Link WellScreen to AquiferSystem This commit addresses the need to handle scenarios where multiple well screens may target different aquifers. Added the aquifer_system_id foreign key to the WellScreen model. This creates a direct, physical link between a screened interval and the specific hydrogeologic unit it monitors. --- db/thing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/thing.py b/db/thing.py index 246d9536b..c43064fe0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -346,6 +346,9 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): thing_id: Mapped[int] = mapped_column( ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="SET NULL"), nullable=True + ) screen_depth_top: Mapped[float] = mapped_column( info={"unit": "feet below ground surface"}, nullable=True ) @@ -363,6 +366,10 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): # Many-To-One: A WellScreen belongs to one Thing. thing: Mapped["Thing"] = relationship("Thing", back_populates="screens") + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="well_screens", passive_deletes=True + ) + class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): """ From 04a28be8849cdb4e77591816595cbb4fae53f2a5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 11:36:55 -0700 Subject: [PATCH 031/119] feat: add GeologicFormation and ThingFormationAssociation models with Thing relationship updates - Introduced GeologicFormation model as a master reference for geologic formations - Created ThingFormationAssociation model to manage many-to-many relationships between Thing and GeologicFormation, including depth interval fields - Updated Thing model to include formation_associations relationship and association proxy for direct access to related geologic formations --- db/geologic_formation.py | 73 +++++++++++++++++++++++++++++++ db/thing.py | 15 +++++++ db/thing_formation_association.py | 60 +++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 db/geologic_formation.py create mode 100644 db/thing_formation_association.py diff --git a/db/geologic_formation.py b/db/geologic_formation.py new file mode 100644 index 000000000..a0119d50c --- /dev/null +++ b/db/geologic_formation.py @@ -0,0 +1,73 @@ +""" +SQLAlchemy model for the GeologicFormation table. + +This table is a master reference table for geologic formations. Its purpose is to store definitions and descriptions +of various geologic formations that can be referenced by other tables in the database. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +if TYPE_CHECKING: + from db.thing import Thing + from db.thing_formation_association import ThingFormationAssociation + + +class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + # TODO: Should `name` use a controlled vocabulary? + name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", + ) + # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. + code: Mapped[str] = lexicon_term( + nullable=True, + unique=True, + comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the geologic formation, its characteristics, and its significance.", + ) + # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. + lithology: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary for the primary, dominant rock type" + "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + nullable=True, + comment="A spatial representation of the geologic formation's extent.", + ) + + # --- Relationships --- + # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the + # ThingFormationAssociation join table. + thing_associations: Mapped[List["ThingFormationAssociation"]] = relationship( + "ThingFormationAssociation", + back_populates="geologic_formation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # --- Association Proxies --- + # Provides direct access to Things (wells) that penetrate this formation. + things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing") + + # --- Table Arguments --- + __table_args__ = ( + Index("ix_geologicformation_name", "name"), + Index("ix_geologicformation_code", "code"), + ) diff --git a/db/thing.py b/db/thing.py index c43064fe0..b5a2a284a 100644 --- a/db/thing.py +++ b/db/thing.py @@ -41,6 +41,8 @@ from db.group import Group, GroupThingAssociation from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation + from db.geologic_formation import GeologicFormation + from db.thing_formation_association import ThingFormationAssociation class Thing( @@ -272,6 +274,14 @@ class Thing( passive_deletes=True, ) + # Many-To-Many: A Thing can penetrate many GeologicFormations. + formation_associations: Mapped[List["ThingFormationAssociation"]] = relationship( + "ThingFormationAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" @@ -302,6 +312,11 @@ class Thing( "aquifer_associations", "aquifer_system" ) + # Proxy to directly access the GeologicFormations penetrated by this Thing. + formations: AssociationProxy[List["GeologicFormation"]] = association_proxy( + "formation_associations", "geologic_formation" + ) + # Full-text search vector search_vector = Column(TSVectorType("name", "well_construction_notes")) diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py new file mode 100644 index 000000000..c5f0f9592 --- /dev/null +++ b/db/thing_formation_association.py @@ -0,0 +1,60 @@ +""" +SQLAlchemy model for the ThingFormationAssociation table. + +This table is an association object that creates a many-to-many relationship between a Thing (well) and a +GeologicFormation. It stores the lithology for a well, detailing the depth intervals for each formation it penetrates. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.geologic_formation import GeologicFormation + + +class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + This is a= join table (Association Object). It represents the association of a Thing to a + GeologicFormation at a specific depth interval. + """ + + # --- Foreign Keys --- + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), + nullable=False, + comment="The foreign key linking this record to the `Thing` table." + "Deleting a `Thing` will cascade and delete its formation log.", + ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), + nullable=True, + comment="The foreign key linking this record to the `GeologicFormation` table." + "This is set to `SET NULL` on delete, as deleting a formation definition (a rare admin action)" + "should not delete the historical fact that a well had a pick at this depth.", + ) + + # Depth interval fields + top_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the top of the geologic formation, as measured from ground surface.", + ) + bottom_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the bottom of the geologic formation, as measured from ground surface.", + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="formation_associations" + ) + + # Many-To-One: This association links to one GeologicFormation. + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="thing_associations" + ) From 477247a66165e9987b8709806d0d7326d4e65926 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 12:35:55 -0700 Subject: [PATCH 032/119] feat(lexicon): add "formation_code" category and associated terms to lexicon --- core/lexicon.json | 302 ++++++++++++++++++++++++++++++++++++++- db/geologic_formation.py | 5 +- 2 files changed, 302 insertions(+), 5 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 5b45332d8..6d980ab43 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -52,7 +52,8 @@ {"name": "well_purpose", "description": null}, {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, - {"name": "well_pump_type", "description": null} + {"name": "well_pump_type", "description": null}, + {"name": "formation_code", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -675,7 +676,7 @@ {"categories": ["monitoring_frequency"], "term": "Biannual", "definition": "Location is monitored twice a year."}, {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, - {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."} + {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, @@ -693,6 +694,301 @@ {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, - {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"} + {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"}, + {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, +{"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, +{"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, +{"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, +{"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, +{"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, +{"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, +{"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, +{"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, +{"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, +{"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, +{"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, +{"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, +{"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, +{"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, +{"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, +{"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, +{"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, +{"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, +{"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, +{"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, +{"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, +{"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, +{"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, +{"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, +{"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, +{"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, +{"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, +{"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, +{"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, +{"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, +{"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, +{"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, +{"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, +{"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, +{"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, +{"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, +{"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, +{"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, +{"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, +{"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, +{"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, +{"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, +{"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, +{"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, +{"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, +{"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, +{"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, +{"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, +{"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, +{"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, +{"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, +{"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, +{"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, +{"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, +{"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, +{"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, +{"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, +{"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, +{"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, +{"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, +{"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, +{"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, +{"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, +{"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, +{"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, +{"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, +{"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, +{"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, +{"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, +{"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, +{"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, +{"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, +{"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, +{"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, +{"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, +{"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, +{"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, +{"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, +{"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, +{"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, +{"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, +{"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, +{"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, +{"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, +{"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, +{"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, +{"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, +{"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, +{"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, +{"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, +{"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, +{"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, +{"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, +{"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, +{"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, +{"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, +{"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, +{"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, +{"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, +{"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, +{"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, +{"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, +{"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, +{"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, +{"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, +{"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, +{"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, +{"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, +{"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, +{"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, +{"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, +{"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, +{"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, +{"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, +{"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, +{"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, +{"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, +{"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, +{"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, +{"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, +{"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, +{"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, +{"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, +{"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, +{"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, +{"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, +{"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, +{"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, +{"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, +{"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, +{"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, +{"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, +{"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, +{"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, +{"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, +{"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, +{"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, +{"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, +{"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, +{"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, +{"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, +{"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, +{"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, +{"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, +{"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, +{"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, +{"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, +{"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, +{"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, +{"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, +{"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, +{"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, +{"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, +{"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, +{"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, +{"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, +{"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, +{"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, +{"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, +{"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, +{"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, +{"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, +{"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, +{"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, +{"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, +{"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, +{"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, +{"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, +{"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, +{"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, +{"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, +{"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, +{"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, +{"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, +{"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, +{"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, +{"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, +{"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, +{"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, +{"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, +{"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, +{"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, +{"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, +{"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, +{"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, +{"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, +{"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, +{"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, +{"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, +{"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, +{"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, +{"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, +{"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, +{"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, +{"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, +{"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, +{"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, +{"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, +{"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, +{"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, +{"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, +{"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, +{"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, +{"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, +{"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, +{"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, +{"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, +{"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, +{"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, +{"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, +{"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, +{"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, +{"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, +{"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, +{"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, +{"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, +{"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, +{"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, +{"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, +{"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, +{"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, +{"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, +{"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, +{"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, +{"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, +{"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, +{"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, +{"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, +{"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, +{"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, +{"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, +{"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, +{"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, +{"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, +{"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, +{"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, +{"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, +{"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, +{"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, +{"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, +{"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, +{"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, +{"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, +{"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, +{"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, +{"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, +{"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, +{"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, +{"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, +{"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, +{"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, +{"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, +{"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, +{"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, +{"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, +{"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, +{"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, +{"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, +{"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, +{"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, +{"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, +{"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, +{"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, +{"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"} ] } \ No newline at end of file diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a0119d50c..ff6ce0a85 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -30,7 +30,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", ) # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. - code: Mapped[str] = lexicon_term( + formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').", @@ -41,7 +41,8 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): comment="A detailed description of the geologic formation, its characteristics, and its significance.", ) # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. - lithology: Mapped[str] = lexicon_term( + # This should be implemented after AMMP reviews and cleans up their formation terms and codes. + lithology: Mapped[str] = mapped_column( nullable=True, comment="A controlled vocabulary for the primary, dominant rock type" "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", From d359fa1639879f241d5536236434993ba69f99fa Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 12:40:23 -0700 Subject: [PATCH 033/119] rafactor(model): Remove TODO about adding formation_code values to lexicon --- db/geologic_formation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index ff6ce0a85..35332aac3 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -29,7 +29,6 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): unique=True, comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", ) - # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, From fc8132768a6cec563034795f89099e59305679ff Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 16:53:15 -0700 Subject: [PATCH 034/119] refactor(lexicon): refine terms associated with "geographic_scale". --- core/lexicon.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 6d980ab43..0461f530d 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -692,9 +692,10 @@ {"categories": ["aquifer_type"], "term": "Unconfined, single", "definition": "Unconfined single aquifer"}, {"categories": ["aquifer_type"], "term": "Unconfined, multiple", "definition": "Unconfined multiple aquifers"}, {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, - {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, - {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, - {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"}, + {"categories": ["geographic_scale"], "term": "Major", "definition": "Major aquifers of national significance"}, + {"categories": ["geographic_scale"], "term": "Regional", "definition": "Important aquifers serving regions"}, + {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, + {"categories": ["geographic_scale"], "term": "Minor", "definition": "Limited extent or yield"}, {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, {"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, {"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, From b01d172179ed75e1f0a389323c19d52ef09af2f1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 17:00:29 -0700 Subject: [PATCH 035/119] refactor(model): Replace hardcoded srid with SRID_WGS84 from constants Replace the hardcoded `srid` in the `aquifer_system` and `geologic_formation` models with the SRID_WGS84 from constants. --- db/aquifer_system.py | 4 +++- db/geologic_formation.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index fd121cd7b..95066c2e8 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -14,6 +14,8 @@ from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term +from constants import SRID_WGS84 + if TYPE_CHECKING: from db.thing import WellScreen, ThingAquiferAssociation, Thing @@ -41,7 +43,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", ) boundary: Mapped[Geometry] = mapped_column( - Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), nullable=True, comment="A spatial representation of the aquifer system's boundary.", ) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index 35332aac3..d916f402a 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -15,6 +15,8 @@ from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term +from constants import SRID_WGS84 + if TYPE_CHECKING: from db.thing import Thing from db.thing_formation_association import ThingFormationAssociation @@ -47,7 +49,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", ) boundary: Mapped[Geometry] = mapped_column( - Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), nullable=True, comment="A spatial representation of the geologic formation's extent.", ) From f35e61d61faf664d756f6505e86410dbb7c28463 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 19:42:54 -0700 Subject: [PATCH 036/119] refactor(model): Add geologic_formation foreign key to WellScreen model. --- db/geologic_formation.py | 6 +++++- db/thing.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index d916f402a..a29c20010 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -18,7 +18,7 @@ from constants import SRID_WGS84 if TYPE_CHECKING: - from db.thing import Thing + from db.thing import Thing, WellScreen from db.thing_formation_association import ThingFormationAssociation @@ -63,6 +63,10 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): cascade="all, delete-orphan", passive_deletes=True, ) + # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. + screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", back_populates="geologic_formation", passive_deletes=True + ) # --- Association Proxies --- # Provides direct access to Things (wells) that penetrate this formation. diff --git a/db/thing.py b/db/thing.py index b5a2a284a..25dcb0969 100644 --- a/db/thing.py +++ b/db/thing.py @@ -364,6 +364,9 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): aquifer_system_id: Mapped[int] = mapped_column( ForeignKey("aquifer_system.id", ondelete="SET NULL"), nullable=True ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), nullable=True + ) screen_depth_top: Mapped[float] = mapped_column( info={"unit": "feet below ground surface"}, nullable=True ) From 390d6f39aacc4e92fc1064111daa3b8d1a56025b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 09:10:01 -0700 Subject: [PATCH 037/119] refactor(model): enhance relationships for `ThingAquiferAssociation` and `ThingFormationAssociation` models Add `lazy="joined"` to `ThingAquiferAssociation` and `ThingFormationAssociation` models to eagerly load relationships. --- db/thing_aquifer_association.py | 4 ++-- db/thing_formation_association.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index 5f14696ca..d5fb2881a 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -33,10 +33,10 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationship Definitions --- # Many-To-One: This association links to one Thing. thing: Mapped["Thing"] = relationship( - "Thing", back_populates="aquifer_associations" + "Thing", back_populates="aquifer_associations", lazy="joined" ) # Many-To-One: This association links to one AquiferSystem. aquifer_system: Mapped["AquiferSystem"] = relationship( - "AquiferSystem", back_populates="thing_associations" + "AquiferSystem", back_populates="thing_associations", lazy="joined" ) diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py index c5f0f9592..8904fa089 100644 --- a/db/thing_formation_association.py +++ b/db/thing_formation_association.py @@ -51,10 +51,10 @@ class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationship Definitions --- # Many-To-One: This association links to one Thing. thing: Mapped["Thing"] = relationship( - "Thing", back_populates="formation_associations" + "Thing", back_populates="formation_associations", lazy="joined" ) # Many-To-One: This association links to one GeologicFormation. geologic_formation: Mapped["GeologicFormation"] = relationship( - "GeologicFormation", back_populates="thing_associations" + "GeologicFormation", back_populates="thing_associations", lazy="joined" ) From c738afba94d576c945d35ac1cec9807497996762 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 15:51:39 -0700 Subject: [PATCH 038/119] refactor(model): refactor `lithology` field to a lexicon_term - Refactor `lithology` field to a lexicon_term - Add lithology values to lexicon --- core/lexicon.json | 667 ++++++++++++++++++++++----------------- db/geologic_formation.py | 16 +- 2 files changed, 381 insertions(+), 302 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 0461f530d..15479d6b4 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -53,7 +53,8 @@ {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, {"name": "well_pump_type", "description": null}, - {"name": "formation_code", "description": null} + {"name": "formation_code", "description": null}, + {"name": "lithology", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -697,299 +698,375 @@ {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, {"categories": ["geographic_scale"], "term": "Minor", "definition": "Limited extent or yield"}, {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, -{"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, -{"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, -{"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, -{"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, -{"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, -{"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, -{"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, -{"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, -{"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, -{"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, -{"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, -{"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, -{"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, -{"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, -{"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, -{"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, -{"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, -{"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, -{"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, -{"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, -{"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, -{"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, -{"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, -{"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, -{"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, -{"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, -{"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, -{"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, -{"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, -{"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, -{"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, -{"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, -{"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, -{"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, -{"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, -{"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, -{"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, -{"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, -{"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, -{"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, -{"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, -{"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, -{"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, -{"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, -{"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, -{"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, -{"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, -{"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, -{"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, -{"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, -{"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, -{"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, -{"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, -{"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, -{"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, -{"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, -{"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, -{"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, -{"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, -{"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, -{"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, -{"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, -{"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, -{"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, -{"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, -{"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, -{"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, -{"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, -{"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, -{"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, -{"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, -{"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, -{"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, -{"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, -{"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, -{"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, -{"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, -{"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, -{"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, -{"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, -{"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, -{"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, -{"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, -{"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, -{"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, -{"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, -{"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, -{"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, -{"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, -{"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, -{"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, -{"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, -{"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, -{"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, -{"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, -{"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, -{"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, -{"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, -{"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, -{"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, -{"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, -{"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, -{"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, -{"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, -{"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, -{"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, -{"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, -{"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, -{"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, -{"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, -{"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, -{"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, -{"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, -{"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, -{"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, -{"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, -{"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, -{"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, -{"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, -{"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, -{"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, -{"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, -{"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, -{"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, -{"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, -{"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, -{"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, -{"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, -{"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, -{"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, -{"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, -{"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, -{"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, -{"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, -{"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, -{"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, -{"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, -{"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, -{"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, -{"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, -{"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, -{"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, -{"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, -{"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, -{"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, -{"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, -{"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, -{"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, -{"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, -{"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, -{"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, -{"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, -{"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, -{"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, -{"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, -{"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, -{"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, -{"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, -{"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, -{"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, -{"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, -{"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, -{"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, -{"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, -{"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, -{"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, -{"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, -{"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, -{"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, -{"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, -{"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, -{"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, -{"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, -{"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, -{"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, -{"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, -{"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, -{"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, -{"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, -{"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, -{"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, -{"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, -{"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, -{"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, -{"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, -{"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, -{"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, -{"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, -{"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, -{"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, -{"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, -{"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, -{"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, -{"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, -{"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, -{"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, -{"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, -{"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, -{"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, -{"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, -{"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, -{"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, -{"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, -{"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, -{"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, -{"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, -{"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, -{"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, -{"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, -{"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, -{"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, -{"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, -{"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, -{"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, -{"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, -{"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, -{"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, -{"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, -{"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, -{"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, -{"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, -{"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, -{"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, -{"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, -{"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, -{"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, -{"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, -{"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, -{"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, -{"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, -{"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, -{"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, -{"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, -{"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, -{"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, -{"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, -{"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, -{"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, -{"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, -{"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, -{"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, -{"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, -{"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, -{"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, -{"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, -{"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, -{"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, -{"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, -{"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, -{"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, -{"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, -{"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, -{"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, -{"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, -{"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, -{"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, -{"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, -{"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, -{"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, -{"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, -{"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, -{"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, -{"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, -{"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"} + {"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, + {"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, + {"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, + {"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, + {"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, + {"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, + {"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, + {"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, + {"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, + {"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, + {"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, + {"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, + {"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, + {"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, + {"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, + {"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, + {"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, + {"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, + {"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, + {"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, + {"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, + {"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, + {"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, + {"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, + {"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, + {"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, + {"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, + {"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, + {"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, + {"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, + {"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, + {"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, + {"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, + {"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, + {"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, + {"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, + {"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, + {"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, + {"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, + {"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, + {"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, + {"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, + {"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, + {"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, + {"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, + {"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, + {"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, + {"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, + {"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, + {"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, + {"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, + {"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, + {"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, + {"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, + {"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, + {"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, + {"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, + {"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, + {"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, + {"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, + {"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, + {"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, + {"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, + {"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, + {"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, + {"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, + {"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, + {"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, + {"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, + {"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, + {"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, + {"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, + {"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, + {"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, + {"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, + {"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, + {"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, + {"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, + {"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, + {"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, + {"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, + {"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, + {"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, + {"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, + {"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, + {"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, + {"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, + {"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, + {"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, + {"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, + {"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, + {"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, + {"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, + {"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, + {"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, + {"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, + {"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, + {"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, + {"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, + {"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, + {"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, + {"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, + {"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, + {"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, + {"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, + {"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, + {"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, + {"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, + {"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, + {"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, + {"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, + {"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, + {"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, + {"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, + {"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, + {"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, + {"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, + {"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, + {"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, + {"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, + {"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, + {"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, + {"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, + {"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, + {"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, + {"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, + {"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, + {"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, + {"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, + {"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, + {"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, + {"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, + {"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, + {"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, + {"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, + {"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, + {"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, + {"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, + {"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, + {"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, + {"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, + {"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, + {"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, + {"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, + {"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, + {"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, + {"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, + {"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, + {"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, + {"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, + {"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, + {"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, + {"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, + {"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, + {"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, + {"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, + {"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, + {"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, + {"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, + {"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, + {"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, + {"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, + {"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, + {"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, + {"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, + {"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, + {"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, + {"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, + {"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, + {"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, + {"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, + {"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, + {"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, + {"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, + {"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, + {"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, + {"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, + {"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, + {"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, + {"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, + {"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, + {"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, + {"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, + {"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, + {"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, + {"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, + {"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, + {"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, + {"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, + {"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, + {"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, + {"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, + {"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, + {"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, + {"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, + {"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, + {"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, + {"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, + {"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, + {"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, + {"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, + {"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, + {"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, + {"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, + {"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, + {"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, + {"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, + {"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, + {"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, + {"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, + {"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, + {"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, + {"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, + {"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, + {"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, + {"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, + {"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, + {"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, + {"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, + {"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, + {"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, + {"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, + {"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, + {"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, + {"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, + {"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, + {"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, + {"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, + {"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, + {"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, + {"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, + {"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, + {"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"}, + {"categories": ["lithology"],"term": "Alluvium","definition": "Alluvium"}, + {"categories": ["lithology"],"term": "Anhydrite","definition": "Anhydrite"}, + {"categories": ["lithology"],"term": "Arkose","definition": "Arkose"}, + {"categories": ["lithology"],"term": "Boulders","definition": "Boulders"}, + {"categories": ["lithology"],"term": "Boulders, silt and clay","definition": "Boulders, silt and clay"}, + {"categories": ["lithology"],"term": "Boulders and sand","definition": "Boulders and sand"}, + {"categories": ["lithology"],"term": "Bentonite","definition": "Bentonite"}, + {"categories": ["lithology"],"term": "Breccia","definition": "Breccia"}, + {"categories": ["lithology"],"term": "Basalt","definition": "Basalt"}, + {"categories": ["lithology"],"term": "Conglomerate","definition": "Conglomerate"}, + {"categories": ["lithology"],"term": "Chalk","definition": "Chalk"}, + {"categories": ["lithology"],"term": "Chert","definition": "Chert"}, + {"categories": ["lithology"],"term": "Clay","definition": "Clay"}, + {"categories": ["lithology"],"term": "Caliche","definition": "Caliche"}, + {"categories": ["lithology"],"term": "Calcite","definition": "Calcite"}, + {"categories": ["lithology"],"term": "Clay, some sand","definition": "Clay, some sand"}, + {"categories": ["lithology"],"term": "Claystone","definition": "Claystone"}, + {"categories": ["lithology"],"term": "Coal","definition": "Coal"}, + {"categories": ["lithology"],"term": "Cobbles","definition": "Cobbles"}, + {"categories": ["lithology"],"term": "Cobbles, silt and clay","definition": "Cobbles, silt and clay"}, + {"categories": ["lithology"],"term": "Cobbles and sand","definition": "Cobbles and sand"}, + {"categories": ["lithology"],"term": "Dolomite","definition": "Dolomite"}, + {"categories": ["lithology"],"term": "Dolomite and shale","definition": "Dolomite and shale"}, + {"categories": ["lithology"],"term": "Evaporite","definition": "Evaporite"}, + {"categories": ["lithology"],"term": "Gneiss","definition": "Gneiss"}, + {"categories": ["lithology"],"term": "Gypsum","definition": "Gypsum"}, + {"categories": ["lithology"],"term": "Graywacke","definition": "Graywacke"}, + {"categories": ["lithology"],"term": "Gravel and clay","definition": "Gravel and clay"}, + {"categories": ["lithology"],"term": "Gravel, cemented","definition": "Gravel, cemented"}, + {"categories": ["lithology"],"term": "Gravel, sand and silt","definition": "Gravel, sand and silt"}, + {"categories": ["lithology"],"term": "Granite, gneiss","definition": "Granite, gneiss"}, + {"categories": ["lithology"],"term": "Granite","definition": "Granite"}, + {"categories": ["lithology"],"term": "Gravel, silt and clay","definition": "Gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Gravel","definition": "Gravel"}, + {"categories": ["lithology"],"term": "Igneous undifferentiated","definition": "Igneous undifferentiated"}, + {"categories": ["lithology"],"term": "Lignite","definition": "Lignite"}, + {"categories": ["lithology"],"term": "Limestone and dolomite","definition": "Limestone and dolomite"}, + {"categories": ["lithology"],"term": "Limestone and shale","definition": "Limestone and shale"}, + {"categories": ["lithology"],"term": "Limestone","definition": "Limestone"}, + {"categories": ["lithology"],"term": "Marl","definition": "Marl"}, + {"categories": ["lithology"],"term": "Mudstone","definition": "Mudstone"}, + {"categories": ["lithology"],"term": "Metamorphic undifferentiated","definition": "Metamorphic undifferentiated"}, + {"categories": ["lithology"],"term": "Marlstone","definition": "Marlstone"}, + {"categories": ["lithology"],"term": "No Recovery","definition": "No Recovery"}, + {"categories": ["lithology"],"term": "Peat","definition": "Peat"}, + {"categories": ["lithology"],"term": "Quartzite","definition": "Quartzite"}, + {"categories": ["lithology"],"term": "Rhyolite","definition": "Rhyolite"}, + {"categories": ["lithology"],"term": "Sand","definition": "Sand"}, + {"categories": ["lithology"],"term": "Schist","definition": "Schist"}, + {"categories": ["lithology"],"term": "Sand and clay","definition": "Sand and clay"}, + {"categories": ["lithology"],"term": "Sand and gravel","definition": "Sand and gravel"}, + {"categories": ["lithology"],"term": "Sandstone and shale","definition": "Sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand and silt","definition": "Sand and silt"}, + {"categories": ["lithology"],"term": "Sand, gravel and clay","definition": "Sand, gravel and clay"}, + {"categories": ["lithology"],"term": "Shale","definition": "Shale"}, + {"categories": ["lithology"],"term": "Silt","definition": "Silt"}, + {"categories": ["lithology"],"term": "Siltstone and shale","definition": "Siltstone and shale"}, + {"categories": ["lithology"],"term": "Siltstone","definition": "Siltstone"}, + {"categories": ["lithology"],"term": "Slate","definition": "Slate"}, + {"categories": ["lithology"],"term": "Sand, some clay","definition": "Sand, some clay"}, + {"categories": ["lithology"],"term": "Sandstone","definition": "Sandstone"}, + {"categories": ["lithology"],"term": "Silt and clay","definition": "Silt and clay"}, + {"categories": ["lithology"],"term": "Travertine","definition": "Travertine"}, + {"categories": ["lithology"],"term": "Tuff","definition": "Tuff"}, + {"categories": ["lithology"],"term": "Volcanic undifferentiated","definition": "Volcanic undifferentiated"}, + {"categories": ["lithology"],"term": "Clay, yellow","definition": "Clay, yellow"}, + {"categories": ["lithology"],"term": "Clay, red","definition": "Clay, red"}, + {"categories": ["lithology"],"term": "Surficial sediment","definition": "Surficial sediment"}, + {"categories": ["lithology"],"term": "Limestone and sandstone, interbedded","definition": "Limestone and sandstone, interbedded"}, + {"categories": ["lithology"],"term": "Gravel and boulders","definition": "Gravel and boulders"}, + {"categories": ["lithology"],"term": "Sand, silt and gravel","definition": "Sand, silt and gravel"}, + {"categories": ["lithology"],"term": "Sand, gravel, silt and clay","definition": "Sand, gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Andesite","definition": "Andesite"}, + {"categories": ["lithology"],"term": "Ignesous, intrusive, undifferentiated","definition": "Ignesous, intrusive, undifferentiated"}, + {"categories": ["lithology"],"term": "Limestone, sandstone and shale","definition": "Limestone, sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand, silt and clay","definition": "Sand, silt and clay"} ] } \ No newline at end of file diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a29c20010..d8a7c7515 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -25,12 +25,14 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} - # TODO: Should `name` use a controlled vocabulary? - name: Mapped[str] = mapped_column( - nullable=False, - unique=True, - comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", - ) + # TODO: Let the API map formation codes to names using a formations.json file that can be periodically updated + # from the authoritative source (.e.g USGS). A placeholder `formations.json` file had been added to the `core` + # directory. + # name: Mapped[str] = mapped_column( + # nullable=False, + # unique=True, + # comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", + # ) formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, @@ -43,7 +45,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): ) # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. # This should be implemented after AMMP reviews and cleans up their formation terms and codes. - lithology: Mapped[str] = mapped_column( + lithology: Mapped[str] = lexicon_term( nullable=True, comment="A controlled vocabulary for the primary, dominant rock type" "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", From 4d55177c3ee2e65fbffc3f5f80e890ee8e7aae56 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 15:58:30 -0700 Subject: [PATCH 039/119] feat(core): create placeholder `formations.json` This is a placeholder file to allow the API to map formation names to codes. It can be periodically updated from the authoritative source (.e.g USGS). --- core/formations.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 core/formations.json diff --git a/core/formations.json b/core/formations.json new file mode 100644 index 000000000..e69de29bb From 8362f9c5a3f695380f453cf543663b94b2c8fa92 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 17:07:12 -0700 Subject: [PATCH 040/119] refactor: add aquifer and geology related models to `db/__init__.py` --- db/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/__init__.py b/db/__init__.py index efb23a418..68c94d598 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -40,6 +40,10 @@ from db.status_history import * from db.thing import * from db.transducer import * +from db.aquifer_system import * +from db.geologic_formation import * +from db.thing_aquifer_association import * +from db.thing_formation_association import * from sqlalchemy import ( func, From 39aeefa597bfd91ecedf72ac58ea899f0f857d7b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 17:12:15 -0700 Subject: [PATCH 041/119] feat: add aquifer and geology related enums. --- core/enums.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/enums.py b/core/enums.py index 5833d97bc..0da1d2eef 100644 --- a/core/enums.py +++ b/core/enums.py @@ -70,4 +70,7 @@ ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +AquiferType: type[Enum] = build_enum_from_lexicon_category("aquifer_type") +GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale") +Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") # ============= EOF ============================================= From 47b8415f05b7a4f37dfb3d5d4f3deeb383b81148 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 19:59:14 -0700 Subject: [PATCH 042/119] feat(schemas): add response schemas for aquifer systems and geologic formations Add comprehensive response schemas for AquiferSystem and GeologicFormation models. Integrate these schemas into Thing/Well response models to provide complete geological context. Changes: - Add AquiferSystemResponse with name, type, scale, and boundary fields - Add GeologicFormationResponse with formation code, lithology, and boundary - Add ThingFormationAssociationResponse with depth interval data - Update WellResponse to include aquifers and formations lists - Add field validators to convert ORM association objects to response schemas - Update WellScreenResponse to include aquifer_system and geologic_formation - Update CreateWellScreen and UpdateWellScreen with geological associations --- schemas/aquifer_system.py | 32 ++++++++++++++++++++++++ schemas/geologic_formation.py | 46 +++++++++++++++++++++++++++++++++++ schemas/thing.py | 32 ++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 schemas/aquifer_system.py create mode 100644 schemas/geologic_formation.py diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py new file mode 100644 index 000000000..5f5b3ed4d --- /dev/null +++ b/schemas/aquifer_system.py @@ -0,0 +1,32 @@ +from typing import List + +from pydantic import BaseModel + +from schemas import BaseResponseModel + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str + coordinates: ( + List[float] + | List[List[float]] + | List[List[List[float]]] + | List[List[List[List[float]]]] + ) + + +class AquiferSystemResponse(BaseResponseModel): + """ + Response schema for aquifer system details. + """ + + name: str + description: str | None = None + aquifer_type: str + geographic_scale: str + boundary: GeoJSONGeometry | None = None diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py new file mode 100644 index 000000000..2f5f0432d --- /dev/null +++ b/schemas/geologic_formation.py @@ -0,0 +1,46 @@ +from typing import List + +from pydantic import BaseModel + +from schemas import BaseResponseModel + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str + coordinates: ( + List[float] + | List[List[float]] + | List[List[List[float]]] + | List[List[List[List[float]]]] + ) + + +class GeologicFormationResponse(BaseResponseModel): + """ + Response schema for geologic formation details. + """ + + formation_code: str | None = None + description: str | None = None + lithology: str | None = None + boundary: GeoJSONGeometry | None = None + + +class ThingFormationAssociationResponse(BaseResponseModel): + """ + Response schema for the association between a Thing and a GeologicFormation. + Includes depth interval information. + """ + + thing_id: int + geologic_formation_id: int | None = None + geologic_formation: GeologicFormationResponse | None = None + top_depth: float + top_depth_unit: str = "ft" + bottom_depth: float + bottom_depth_unit: str = "ft" diff --git a/schemas/thing.py b/schemas/thing.py index d87fd299f..36f3eb757 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -27,6 +27,11 @@ ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse +from schemas.aquifer_system import AquiferSystemResponse +from schemas.geologic_formation import ( + GeologicFormationResponse, + ThingFormationAssociationResponse, +) # -------- VALIDATE ---------- @@ -121,6 +126,8 @@ class CreateWellScreen(BaseCreateModel): """ thing_id: int + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float = Field(gt=0, description="Screen depth bottom in feet") screen_depth_top: float = Field(gt=0, description="Screen depth top in feet") screen_type: ScreenType | None = None @@ -166,6 +173,8 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" + aquifers: list[AquiferSystemResponse] = [] + formations: list[ThingFormationAssociationResponse] = [] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -186,6 +195,23 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("aquifers", mode="before") + def populate_aquifers(cls, aquifers): + """Convert aquifer association objects to aquifer system objects.""" + if aquifers is not None: + # Handle if aquifers are already AquiferSystem objects + if hasattr(aquifers[0] if aquifers else None, "aquifer_system"): + return [assoc.aquifer_system for assoc in aquifers] + return aquifers or [] + + @field_validator("formations", mode="before") + def populate_formations(cls, formations): + """Convert formation association objects to response objects.""" + if formations is not None: + # formations should already be ThingFormationAssociation objects + return formations + return [] + class SpringResponse(BaseThingResponse): """ @@ -222,6 +248,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse + aquifer_system_id: int | None = None + aquifer_system: AquiferSystemResponse | None = None + geologic_formation_id: int | None = None + geologic_formation: GeologicFormationResponse | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -295,6 +325,8 @@ class UpdateThingIdLink(BaseUpdateModel): class UpdateWellScreen(BaseUpdateModel): + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float | None = None screen_depth_top: float | None = None screen_description: str | None = None From b4c8beb604bf53c36a69da80c315932544fa3f8b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 11:50:19 -0700 Subject: [PATCH 043/119] fix: fix artifacts from merge conflicts --- db/thing.py | 8 +- tests/features/environment.py | 3 +- .../steps/well-additional-information.py | 74 +++++++++++-------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/db/thing.py b/db/thing.py index 53cb88df0..7ee3c9cba 100644 --- a/db/thing.py +++ b/db/thing.py @@ -28,7 +28,7 @@ ReleaseMixin, ) from db.permission_history import PermissionHistoryMixin -from services.util import retrieve_latest_polymorphic_table_record +from services.util import retrieve_latest_polymorphic_history_table_record from db.status_history import StatusHistoryMixin from db.measuring_point_history import MeasuringPointHistory from db.data_provenance import DataProvenanceMixin @@ -403,7 +403,7 @@ def allow_water_level_samples(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Water Level Sample" ) return permission_record.permission_allowed if permission_record else None @@ -413,7 +413,7 @@ def allow_water_chemistry_samples(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Water Chemistry Sample" ) return permission_record.permission_allowed if permission_record else None @@ -423,7 +423,7 @@ def allow_datalogger_installation(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Datalogger Installation" ) return permission_record.permission_allowed if permission_record else None diff --git a/tests/features/environment.py b/tests/features/environment.py index c56bb0e86..289d0b0f6 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -448,7 +448,7 @@ def before_all(context): add_permission_history( context, session, - contact_id=contact.id, + contact_id=context.objects["contacts"][0].id, permission_type="Datalogger Installation", permission_allowed=True, start_date=datetime(2025, 1, 1).date(), @@ -618,7 +618,6 @@ def before_all(context): ) session.commit() - session.refresh(well_1) # the following needs to be refreshed to get all the new relationships session.refresh(well_1) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 2baa32db6..1dc5c3518 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -12,7 +12,7 @@ def step_impl(context): assert "allow_water_level_samples" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Water Level Sample" + context.objects["wells"][0], "permission_history", "Water Level Sample" ) assert ( context.water_well_data["allow_water_level_samples"] @@ -25,7 +25,7 @@ def step_impl(context): assert "allow_water_chemistry_samples" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Water Chemistry Sample" + context.objects["wells"][0], "permission_history", "Water Chemistry Sample" ) assert ( @@ -41,11 +41,11 @@ def step_impl(context): assert "allow_datalogger_installation" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Datalogger Installation" + context.objects["wells"][0], "permission_history", "Datalogger Installation" ) assert ( - context.water_well_data["permissions"]["allow_data_logger_installation"] + context.water_well_data["allow_datalogger_installation"] == permission_record.permission_allowed ) @@ -57,10 +57,10 @@ def step_impl(context): @then("the response should include the completion date of the well") def step_impl(context): - assert "completion_date" in context.water_well_data - assert context.water_well_data[ - "completion_date" - ] == context.well.completion_date.strftime("%Y-%m-%d") + assert "well_completion_date" in context.water_well_data + assert context.water_well_data["well_completion_date"] == context.objects["wells"][ + 0 + ].well_completion_date.strftime("%Y-%m-%d") # TODO: needs to be added to model, schemas, test data @@ -69,22 +69,25 @@ def step_impl(context): assert "completion_info_source" in context.water_well_data assert ( context.water_well_data["completion_info_source"] - == context.well.completion_info_source + == context.objects["wells"][0].completion_info_source ) @then("the response should include the driller name") def step_impl(context): - assert "driller_name" in context.water_well_data - assert context.water_well_data["driller_name"] == context.well.driller_name + assert "well_driller_name" in context.water_well_data + assert ( + context.water_well_data["well_driller_name"] + == context.objects["wells"][0].well_driller_name + ) @then("the response should include the construction method") def step_impl(context): - assert "construction_method" in context.water_well_data + assert "well_construction_method" in context.water_well_data assert ( - context.water_well_data["construction_method"] - == context.well.construction_method + context.water_well_data["well_construction_method"] + == context.objects["wells"][0].well_construction_method ) @@ -94,7 +97,7 @@ def step_impl(context): assert "construction_info_source" in context.water_well_data assert ( context.water_well_data["construction_info_source"] - == context.well.construction_info_source + == context.objects["wells"][0].construction_info_source ) @@ -105,11 +108,14 @@ def step_impl(context): @then("the response should include the casing diameter in inches") def step_impl(context): - assert "casing_diameter" in context.water_well_data - assert "casing_diameter_unit" in context.water_well_data + assert "well_casing_diameter" in context.water_well_data + assert "well_casing_diameter_unit" in context.water_well_data - assert context.water_well_data["casing_diameter"] == context.well.casing_diameter - assert context.water_well_data["casing_diameter_unit"] == "in" + assert ( + context.water_well_data["well_casing_diameter"] + == context.objects["wells"][0].well_casing_diameter + ) + assert context.water_well_data["well_casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") @@ -118,7 +124,8 @@ def step_impl(context): assert "well_casing_depth_unit" in context.water_well_data assert ( - context.water_well_data["well_casing_depth"] == context.well.well_casing_depth + context.water_well_data["well_casing_depth"] + == context.objects["wells"][0].well_casing_depth ) assert context.water_well_data["well_casing_depth_unit"] == "ft" @@ -127,16 +134,18 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data - assert ( - context.water_well_data["well_casing_materials"] - == context.well.well_casing_materials + assert sorted(context.water_well_data["well_casing_materials"]) == sorted( + [m.material for m in context.objects["wells"][0].well_casing_materials] ) @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.water_well_data - assert context.water_well_data["well_pump_type"] == context.well.well_pump_type + assert ( + context.water_well_data["well_pump_type"] + == context.objects["wells"][0].well_pump_type + ) @then("the response should include the well pump depth in feet (new field)") @@ -144,7 +153,10 @@ def step_impl(context): assert "well_pump_depth" in context.water_well_data assert "well_pump_depth_unit" in context.water_well_data - assert context.water_well_data["well_pump_depth"] == context.well.well_pump_depth + assert ( + context.water_well_data["well_pump_depth"] + == context.objects["wells"][0].well_pump_depth + ) assert context.water_well_data["well_pump_depth_unit"] == "ft" @@ -155,7 +167,7 @@ def step_impl(context): assert "is_suitable_for_datalogger" in context.water_well_data assert ( context.water_well_data["is_suitable_for_datalogger"] - == context.well.is_suitable_for_datalogger + == context.objects["wells"][0].is_suitable_for_datalogger ) @@ -170,7 +182,7 @@ def step_impl(context): ) def step_impl(context): assert "formation" in context.water_well_data - assert context.water_well_data["formation"] == context.well.formation + assert context.water_well_data["formation"] == context.objects["wells"][0].formation # TODO: needs to be added to model, schemas, test data, lexicon @@ -180,7 +192,8 @@ def step_impl(context): def step_impl(context): assert "aquifer_class_code" in context.water_well_data assert ( - context.water_well_data["aquifer_class_code"] == context.well.aquifer_class_code + context.water_well_data["aquifer_class_code"] + == context.objects["wells"][0].aquifer_class_code ) @@ -191,4 +204,7 @@ def step_impl(context): ) def step_impl(context): assert "aquifer_type" in context.water_well_data - assert context.water_well_data["aquifer_type"] == context.well.aquifer_type + assert ( + context.water_well_data["aquifer_type"] + == context.objects["wells"][0].aquifer_type + ) From d1149f500e7fb2de6618b94970c8b9393824bc42 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:20:22 -0700 Subject: [PATCH 044/119] refactor: rename `ThingFormationAssociation` usages to `ThingGeologicFormationAssociation` --- db/geologic_formation.py | 16 +++++++++------- db/thing.py | 14 ++++++++------ db/thing_formation_association.py | 4 ++-- schemas/geologic_formation.py | 2 +- schemas/thing.py | 6 +++--- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index d8a7c7515..af12d7a8a 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from db.thing import Thing, WellScreen - from db.thing_formation_association import ThingFormationAssociation + from db.thing_formation_association import ThingGeologicFormationAssociation class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): @@ -58,12 +58,14 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationships --- # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the - # ThingFormationAssociation join table. - thing_associations: Mapped[List["ThingFormationAssociation"]] = relationship( - "ThingFormationAssociation", - back_populates="geologic_formation", - cascade="all, delete-orphan", - passive_deletes=True, + # ThingGeologicFormationAssociation join table. + thing_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="geologic_formation", + cascade="all, delete-orphan", + passive_deletes=True, + ) ) # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. screens: Mapped[List["WellScreen"]] = relationship( diff --git a/db/thing.py b/db/thing.py index 25dcb0969..106af773e 100644 --- a/db/thing.py +++ b/db/thing.py @@ -42,7 +42,7 @@ from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation from db.geologic_formation import GeologicFormation - from db.thing_formation_association import ThingFormationAssociation + from db.thing_formation_association import ThingGeologicFormationAssociation class Thing( @@ -275,11 +275,13 @@ class Thing( ) # Many-To-Many: A Thing can penetrate many GeologicFormations. - formation_associations: Mapped[List["ThingFormationAssociation"]] = relationship( - "ThingFormationAssociation", - back_populates="thing", - cascade="all, delete-orphan", - passive_deletes=True, + formation_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) ) # --- Association Proxies --- diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py index 8904fa089..0707df269 100644 --- a/db/thing_formation_association.py +++ b/db/thing_formation_association.py @@ -1,5 +1,5 @@ """ -SQLAlchemy model for the ThingFormationAssociation table. +SQLAlchemy model for the ThingGeologicFormationAssociation table. This table is an association object that creates a many-to-many relationship between a Thing (well) and a GeologicFormation. It stores the lithology for a well, detailing the depth intervals for each formation it penetrates. @@ -17,7 +17,7 @@ from db.geologic_formation import GeologicFormation -class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): +class ThingGeologicFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): """ This is a= join table (Association Object). It represents the association of a Thing to a GeologicFormation at a specific depth interval. diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 2f5f0432d..f6b3083d3 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -31,7 +31,7 @@ class GeologicFormationResponse(BaseResponseModel): boundary: GeoJSONGeometry | None = None -class ThingFormationAssociationResponse(BaseResponseModel): +class ThingGeologicFormationAssociationResponse(BaseResponseModel): """ Response schema for the association between a Thing and a GeologicFormation. Includes depth interval information. diff --git a/schemas/thing.py b/schemas/thing.py index 36f3eb757..398630e66 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -30,7 +30,7 @@ from schemas.aquifer_system import AquiferSystemResponse from schemas.geologic_formation import ( GeologicFormationResponse, - ThingFormationAssociationResponse, + ThingGeologicFormationAssociationResponse, ) @@ -174,7 +174,7 @@ class WellResponse(BaseThingResponse): well_pump_depth: float | None well_pump_depth_unit: str = "ft" aquifers: list[AquiferSystemResponse] = [] - formations: list[ThingFormationAssociationResponse] = [] + formations: list[ThingGeologicFormationAssociationResponse] = [] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -208,7 +208,7 @@ def populate_aquifers(cls, aquifers): def populate_formations(cls, formations): """Convert formation association objects to response objects.""" if formations is not None: - # formations should already be ThingFormationAssociation objects + # formations should already be ThingGeologicFormationAssociation objects return formations return [] From c85f9714d2619db92b2cfdc84e28eddb9199c7a5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:22:39 -0700 Subject: [PATCH 045/119] refactor: rename `ThingFormationAssociation` file`ThingGeologicFormationAssociation` --- db/__init__.py | 2 +- db/geologic_formation.py | 4 +++- db/thing.py | 4 +++- ...association.py => thing_geologic_formation_association.py} | 0 4 files changed, 7 insertions(+), 3 deletions(-) rename db/{thing_formation_association.py => thing_geologic_formation_association.py} (100%) diff --git a/db/__init__.py b/db/__init__.py index 68c94d598..ef14f7c06 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -43,7 +43,7 @@ from db.aquifer_system import * from db.geologic_formation import * from db.thing_aquifer_association import * -from db.thing_formation_association import * +from db.thing_geologic_formation_association import * from sqlalchemy import ( func, diff --git a/db/geologic_formation.py b/db/geologic_formation.py index af12d7a8a..130ed8d45 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -19,7 +19,9 @@ if TYPE_CHECKING: from db.thing import Thing, WellScreen - from db.thing_formation_association import ThingGeologicFormationAssociation + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): diff --git a/db/thing.py b/db/thing.py index 106af773e..12a6ae5ed 100644 --- a/db/thing.py +++ b/db/thing.py @@ -42,7 +42,9 @@ from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation from db.geologic_formation import GeologicFormation - from db.thing_formation_association import ThingGeologicFormationAssociation + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) class Thing( diff --git a/db/thing_formation_association.py b/db/thing_geologic_formation_association.py similarity index 100% rename from db/thing_formation_association.py rename to db/thing_geologic_formation_association.py From 853e4507743fe9ed25a14b9fa757894e2c323eb5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:49:14 -0700 Subject: [PATCH 046/119] refactor(schema): remove aquifer and formation field validators from `schemas/thing.py` --- schemas/thing.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 398630e66..8a63c431f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -195,23 +195,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("aquifers", mode="before") - def populate_aquifers(cls, aquifers): - """Convert aquifer association objects to aquifer system objects.""" - if aquifers is not None: - # Handle if aquifers are already AquiferSystem objects - if hasattr(aquifers[0] if aquifers else None, "aquifer_system"): - return [assoc.aquifer_system for assoc in aquifers] - return aquifers or [] - - @field_validator("formations", mode="before") - def populate_formations(cls, formations): - """Convert formation association objects to response objects.""" - if formations is not None: - # formations should already be ThingGeologicFormationAssociation objects - return formations - return [] - class SpringResponse(BaseThingResponse): """ From 2c33a7900cacab2c01910f2ecc4fd5e9a288be4f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 13:28:16 -0700 Subject: [PATCH 047/119] feat: implement data source tests for well additional information --- db/thing.py | 12 ++++++++++++ run_bdd.sh | 4 ++-- schemas/thing.py | 2 ++ tests/features/environment.py | 18 ++++++++++++++++++ .../steps/well-additional-information.py | 13 +++++++------ 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/db/thing.py b/db/thing.py index 7ee3c9cba..1ed1cbbc3 100644 --- a/db/thing.py +++ b/db/thing.py @@ -398,6 +398,18 @@ def measuring_point_description(self) -> str | None: def well_depth_source(self) -> str | None: return self._get_data_provenance_attribute("well_depth", "origin_source") + @property + def well_completion_date_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_completion_date", "origin_source" + ) + + @property + def well_construction_method_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_construction_method", "origin_source" + ) + @property def allow_water_level_samples(self): """ diff --git a/run_bdd.sh b/run_bdd.sh index 29d0be47d..cd05769e4 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -66,7 +66,7 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} # tests/features/thing-query-parameters.feature #uv run behave tests/features/well-inventory-csv.feature -uv run behave tests/features/well-additional-information.feature --capture -# uv run behave tests/features --tags="@backend and @production" --capture +# uv run behave tests/features/well-additional-information.feature --capture +uv run behave tests/features --tags="@backend and @production" --capture echo "✅ BDD test run complete." diff --git a/schemas/thing.py b/schemas/thing.py index e980e4e76..50f56e7c4 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -223,8 +223,10 @@ class WellResponse(BaseThingResponse): well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None well_completion_date: PastOrTodayDate | None + well_completion_date_source: str | None well_driller_name: str | None well_construction_method: WellConstructionMethod | None + well_construction_method_source: str | None well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" diff --git a/tests/features/environment.py b/tests/features/environment.py index 289d0b0f6..2dc410517 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -606,6 +606,24 @@ def before_all(context): origin_source="Other", ) + well_completion_date_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_completion_date", + origin_source="Data Portal", + ) + + well_construction_method_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_construction_method", + origin_source="Data Portal", + ) + for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 1dc5c3518..eaf83e1ce 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -66,10 +66,11 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): - assert "completion_info_source" in context.water_well_data + assert "well_completion_date_source" in context.water_well_data + assert ( - context.water_well_data["completion_info_source"] - == context.objects["wells"][0].completion_info_source + context.water_well_data["well_completion_date_source"] + == context.objects["wells"][0].well_completion_date_source ) @@ -94,10 +95,10 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): - assert "construction_info_source" in context.water_well_data + assert "well_construction_method_source" in context.water_well_data assert ( - context.water_well_data["construction_info_source"] - == context.objects["wells"][0].construction_info_source + context.water_well_data["well_construction_method_source"] + == context.objects["wells"][0].well_construction_method_source ) From 8ef2592b0994fe5598840e610752bde4b4ff0d47 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 13:29:22 -0700 Subject: [PATCH 048/119] fix: remove outdated notes --- tests/features/steps/well-additional-information.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index eaf83e1ce..d7d3b768c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -63,7 +63,6 @@ def step_impl(context): ].well_completion_date.strftime("%Y-%m-%d") -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): assert "well_completion_date_source" in context.water_well_data @@ -92,7 +91,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): assert "well_construction_method_source" in context.water_well_data @@ -131,7 +129,6 @@ def step_impl(context): assert context.water_well_data["well_casing_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data From ced10b947629847e94cbd12c8b45225eeae94041 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 16:56:06 -0700 Subject: [PATCH 049/119] feat: transfer water level sample permissions --- schemas/thing.py | 2 +- transfers/permissions_transfer.py | 52 +++++++++++++++++++++++++++++++ transfers/transfer.py | 4 +++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 transfers/permissions_transfer.py diff --git a/schemas/thing.py b/schemas/thing.py index 50f56e7c4..a8cac2523 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -130,7 +130,7 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_height: float = Field( ge=0, description="Measuring point height in feet" ) - measuring_point_description: str | None + measuring_point_description: str | None = None notes: list[CreateNote] | None = None diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py new file mode 100644 index 000000000..e1f7bd069 --- /dev/null +++ b/transfers/permissions_transfer.py @@ -0,0 +1,52 @@ +from sqlalchemy.orm import Session +from datetime import datetime + +from db import Thing, PermissionHistory +from transfers.util import read_csv, logger, replace_nans + + +def transfer_permissions(session: Session): + """ + The transferred wells and contacts need to be queried to know who gave + permission to which well since contact_id is required for PermissionHistory + """ + wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) + wdf = replace_nans(wdf) + + transferred_wells = ( + session.query(Thing).filter(Thing.thing_type == "water well").all() + ) + + for well in transferred_wells: + if len(well.contacts) == 0: + logger.critical( + f"Well {well.name} has no associated contacts; skipping permission transfer." + ) + continue + else: + # Assuming the first contact is the relevant one + contact_id = well.contacts[0].id + + allow_water_level_samples = wdf.loc[ + wdf["PointID"] == well.name, "MonitorOK" + ].values + if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + try: + permission_allowed = bool(allow_water_level_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Level Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Level Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + continue + session.commit() diff --git a/transfers/transfer.py b/transfers/transfer.py index 77275ed35..2b576a4b2 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -36,6 +36,7 @@ transfer_wells, transfer_wellscreens, ) +from transfers.permissions_transfer import transfer_permissions from transfers.asset_transfer import transfer_assets from transfers.util import timeit, timeit_direct @@ -124,6 +125,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING ASSETS") timeit_direct(transfer_assets, sess) + message("TRANSFERRING PERMISSIONS") + results = timeit_direct(transfer_permissions, sess) + def transfer_debugging(sess, limit=100): message("STARTING TRANSFER DEBUG", new_line_at_top=False) From e48b71eea39c1ccb83bdf0d2581feb1296fc49b0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 08:13:43 -0700 Subject: [PATCH 050/119] feat: transfer permissions from legacy db --- transfers/permissions_transfer.py | 55 ++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index e1f7bd069..d07c2cbe2 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -48,5 +48,58 @@ def transfer_permissions(session: Session): except Exception as e: logger.error(f"Error transferring permission for well {well.name}: {e}") session.rollback() - continue + pass + + allow_water_chemistry_samples = wdf.loc[ + wdf["PointID"] == well.name, "SampleOK" + ].values + if ( + len(allow_water_chemistry_samples) > 0 + and allow_water_chemistry_samples is not None + ): + try: + permission_allowed = bool(allow_water_chemistry_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Chemistry Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Chemistry Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + allow_datalogger_installation = wdf.loc[ + wdf["PointID"] == well.name, "OpenWellLoggerOK" + ].values + if ( + len(allow_datalogger_installation) > 0 + and allow_datalogger_installation is not None + ): + try: + permission_allowed = bool(allow_datalogger_installation[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Datalogger Installation", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Datalogger Installation permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + session.commit() From 171b41b6da02d35b850aa9deec50e9daa4977040 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 10:44:20 -0700 Subject: [PATCH 051/119] feat: transfer well construction information --- schemas/thing.py | 3 +++ transfers/well_transfer.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/schemas/thing.py b/schemas/thing.py index a8cac2523..2438c50c4 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -132,6 +132,9 @@ class CreateWell(CreateBaseThing, ValidateWell): ) measuring_point_description: str | None = None notes: list[CreateNote] | None = None + well_completion_date: PastOrTodayDate | None = None + well_driller_name: str | None = None + well_construction_method: WellConstructionMethod | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ee54d0216..d2cb4b186 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -257,6 +257,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None notes=( [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=row.ConstructionMethod, ) CreateWell.model_validate(data) From a9cec8ad1c9359351517ccf3820b23f4f88a7468 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 12:32:13 -0700 Subject: [PATCH 052/119] WIP: transfers for well additional info --- schemas/thing.py | 2 ++ transfers/well_transfer.py | 53 ++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 2438c50c4..b23ae7d4f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -133,8 +133,10 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_description: str | None = None notes: list[CreateNote] | None = None well_completion_date: PastOrTodayDate | None = None + well_completion_date_source: str | None = None well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None + well_construction_method_source: str | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index d2cb4b186..e41cf789a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -36,6 +36,7 @@ StatusHistory, MonitoringFrequencyHistory, MeasuringPointHistory, + DataProvenance, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -280,6 +281,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "well_casing_materials", "measuring_point_height", "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" @@ -288,17 +291,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_data.pop("notes") well = Thing(**well_data) session.add(well) - # logger.info(f"Created well for {row.PointID}") - - # flush well to access its ID for status_history - # session.flush() - - # session.commit() - # session.refresh(well) - # if notes: - # for ni in notes: - # nn = well.add_note(ni['content'], ni['note_type']) - # session.add(nn) if well_purposes: for wp in well_purposes: @@ -350,11 +342,44 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None for dp in data_provenances: session.add(dp) + if not isna(row.CompletionSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_completion_date", + origin_source=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.CompletionSource}" + ), + ) + session.add(dp) + + if not isna(row.DataSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_construction_method", + origin_source=lexicon_mapper.map_value( + f"LU_DataSource:{row.DataSource}" + ), + ) + session.add(dp) + + if not isna(row.DepthSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_depth", + origin_source=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.DepthSource}" + ), + ) + session.add(dp) + """ - Developer's note + Developer's note - It's not clear when the measuring point from NM_Aquifer was - determined, so I'm setting start_date to the day of the transfer + It's not clear when the measuring point from NM_Aquifer was + determined, so I'm setting start_date to the day of the transfer """ measuring_point_history = MeasuringPointHistory( thing_id=well.id, From 722425fae83c6adce0cca7eb9f9651d0a867d4c1 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 13:07:31 -0700 Subject: [PATCH 053/119] feat: implement origin_type --- db/data_provenance.py | 8 ++++++-- db/thing.py | 4 ++-- tests/features/environment.py | 10 ++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/db/data_provenance.py b/db/data_provenance.py index 06c468c8d..14cfdc5aa 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -53,9 +53,13 @@ class DataProvenance(AutoBaseMixin, ReleaseMixin, Base): ) # Values from the following NMAquifer tables are included as `origin_source` terms in the lexicon: # 'LU_DataSource', 'LU_Depth_CompletionSource'. - origin_source: Mapped[str] = lexicon_term( + origin_type: Mapped[str] = lexicon_term( nullable=True, - comment="Indicates the origin source of the data (e.g'Driller's Log', 'Well Report'.", + comment="Indicates the type of origin the data (e.g'Driller's Log', 'Well Report'.", + ) + origin_source: Mapped[str] = mapped_column( + nullable=True, + comment="The specific source of the data (e.g., 'J. Brown Thesis, \"I like APIs\", Pomona College, 1994').", ) # Values from the following NMAquifer tables are included as `collection_method` terms in the lexicon: # 'LU_AltitudeMethod','LU_CoordinateMethod'. diff --git a/db/thing.py b/db/thing.py index 1ed1cbbc3..b9c2c54a0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -396,12 +396,12 @@ def measuring_point_description(self) -> str | None: @property def well_depth_source(self) -> str | None: - return self._get_data_provenance_attribute("well_depth", "origin_source") + return self._get_data_provenance_attribute("well_depth", "origin_type") @property def well_completion_date_source(self) -> str | None: return self._get_data_provenance_attribute( - "well_completion_date", "origin_source" + "well_completion_date", "origin_type" ) @property diff --git a/tests/features/environment.py b/tests/features/environment.py index 2dc410517..ac223b530 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -385,7 +385,8 @@ def add_data_provenance( target_id, target_table, field_name, - origin_source, + origin_type=None, + origin_source=None, collection_method=None, accuracy_value=None, accuracy_unit=None, @@ -395,6 +396,7 @@ def add_data_provenance( collection_method=collection_method, target_id=target_id, target_table=target_table, + origin_type=origin_type, origin_source=origin_source, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, @@ -603,7 +605,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_depth", - origin_source="Other", + origin_type="Other", ) well_completion_date_source = add_data_provenance( @@ -612,7 +614,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_completion_date", - origin_source="Data Portal", + origin_type="Data Portal", ) well_construction_method_source = add_data_provenance( @@ -621,7 +623,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_construction_method", - origin_source="Data Portal", + origin_source="Jacob's 2013 Thesis", ) for purpose in ["Domestic", "Irrigation"]: From 75265f335802dd267342d07387049b030ca9c18c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 13:56:23 -0700 Subject: [PATCH 054/119] feat: transfer well source information --- transfers/util.py | 1 - transfers/well_transfer.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index cbf0f2b17..c4b423250 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -369,7 +369,6 @@ def make_location_data_provenance( target_id=location.id, target_table="location", field_name="point", - origin_source=None, collection_method=coordinate_method, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index e41cf789a..2c0073f03 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -260,7 +260,13 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ), well_completion_date=row.CompletionDate, well_driller_name=row.DrillerName, - well_construction_method=row.ConstructionMethod, + well_construction_method=( + lexicon_mapper.map_value( + f"LU_ConstructionMethod:{row.ConstructionMethod}" + ) + if not isna(row.ConstructionMethod) + else None + ), ) CreateWell.model_validate(data) @@ -347,7 +353,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_completion_date", - origin_source=lexicon_mapper.map_value( + origin_type=lexicon_mapper.map_value( f"LU_Depth_CompletionSource:{row.CompletionSource}" ), ) @@ -358,9 +364,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_construction_method", - origin_source=lexicon_mapper.map_value( - f"LU_DataSource:{row.DataSource}" - ), + origin_source=row.DataSource, ) session.add(dp) @@ -369,7 +373,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_depth", - origin_source=lexicon_mapper.map_value( + origin_type=lexicon_mapper.map_value( f"LU_Depth_CompletionSource:{row.DepthSource}" ), ) From 2b90f869e6cdb633750f9f41cee9011a8471e9df Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 14:00:30 -0700 Subject: [PATCH 055/119] feat: add Quemado Mutual Water and Sewage Works Association to organizations for contacts --- core/lexicon.json | 1 + transfers/data/owners_organization_mapper.json | 1 + 2 files changed, 2 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index ba4fd8f7e..46033eee5 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -574,6 +574,7 @@ {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, + {"categories": ["organization"], "term": "Quemado Municipal Water & SWA", "definition": "Quemado Municipal Water & SWA"}, {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, diff --git a/transfers/data/owners_organization_mapper.json b/transfers/data/owners_organization_mapper.json index 5ce45a8bf..b4f29bd7b 100644 --- a/transfers/data/owners_organization_mapper.json +++ b/transfers/data/owners_organization_mapper.json @@ -89,6 +89,7 @@ "Pecos Trail Inn": "Pecos Trail Inn", "Pelican Spa": "Pelican Spa", "Pistachio Tree Ranch": "Pistachio Tree Ranch", + "Quemado Mutual Water and Sewage Works Association": "Quemado Municipal Water & SWA", "Rancho Encantado": "Rancho Encantado", "Rancho San Lucas": "Rancho San Lucas", "Rancho San Marcos": "Rancho San Marcos", From cf72461be26acf99f7b682f8b5d30cd709fa6169 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 14:20:26 -0700 Subject: [PATCH 056/119] fix: remove erroneous or check --- transfers/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/util.py b/transfers/util.py index c4b423250..9b2afc84c 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -275,7 +275,7 @@ def make_location_data_provenance( ) -> list[DataProvenance]: provenance_records = [] - if row.AltitudeAccuracy or row.CoordinateAccuracy: + if row.AltitudeAccuracy: provenance = DataProvenance( target_id=location.id, target_table="location", From 2991591d6539901946e18c9e4aad1f3dd84b1e07 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Thu, 20 Nov 2025 15:30:58 -0700 Subject: [PATCH 057/119] feat(transfers): add `LU_AquiferClass` and `LU_AquiferType` lookup tables to `util.py`. Maps the `LU_AquiferClass` and `LU_AquiferType` lookup tables to the lexicon. --- transfers/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transfers/util.py b/transfers/util.py index 8b9524ad5..a276708db 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -366,6 +366,8 @@ def _make_lu_to_lexicon_mapper(self): # Lookup tables where CODE maps to MEANING lu_tables = [ "LU_AltitudeMethod", + "LU_AquiferClass", + "LU_AquiferType", "LU_CollectionMethod", "LU_ConstructionMethod", "LU_CoordinateAccuracy", From 00e7225f0086d19f7e15c25df7292b9c97e4448c Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Thu, 20 Nov 2025 19:05:27 -0700 Subject: [PATCH 058/119] feat(schemas): implement Create schemas for `aquifer_system` and `geologic_formation`. --- schemas/aquifer_system.py | 14 ++++++++++++++ schemas/geologic_formation.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 5f5b3ed4d..e7c8b9bd0 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -5,6 +5,20 @@ from schemas import BaseResponseModel +# ------ CREATE ---------- +class CreateAquiferSystem(BaseModel): + """ + Schema for creating an aquifer system. + Used during data transfer and API creation. + """ + + name: str + description: str | None = None + aquifer_type: str + geographic_scale: str + boundary: str | None = None + + # ------ RESPONSE ---------- class GeoJSONGeometry(BaseModel): """ diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index f6b3083d3..339188b5b 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -5,6 +5,19 @@ from schemas import BaseResponseModel +# ------ CREATE ---------- +class CreateGeologicFormation(BaseModel): + """ + Schema for creating a geologic formation. + Used during data transfer and API creation. + """ + + formation_code: str | None = None + description: str | None = None + lithology: str | None = None + boundary: str | None = None + + # ------ RESPONSE ---------- class GeoJSONGeometry(BaseModel): """ From 58b92f884bd7165c029dd15be7635911070f7175 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 21 Nov 2025 11:53:18 -0700 Subject: [PATCH 059/119] feat: transfer well pump type parse the construction notes field to get pump types. this can be more sophisticated, but that's a future problem. for now just check for some basic pump type string patterns --- core/lexicon.json | 4 ++-- schemas/thing.py | 1 + transfers/well_transfer.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 46033eee5..21666bccd 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -714,9 +714,9 @@ {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, - {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["well_pump_type"], "term": "Hand", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} diff --git a/schemas/thing.py b/schemas/thing.py index b23ae7d4f..48c055cd8 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -137,6 +137,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None + well_pump_type: WellPumpType | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 2c0073f03..346d91629 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -118,6 +118,21 @@ def _extract_casing_materials(row) -> list[str]: return materials +def _extract_well_pump_type(row) -> str | None: + construction_notes = row.ConstructionNotes.lower() + if "pump" in construction_notes: + if "submersible" in construction_notes: + return "Submersible" + elif "jet" in construction_notes: + return "Jet" + elif "line shaft" in construction_notes or "lineshaft" in construction_notes: + return "Line Shaft" + elif "hand" in construction_notes: + return "Hand" + else: + return None + + def get_wells_to_transfer( sess: Session, flags: dict = None ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -237,6 +252,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) + well_pump_type = ( + _extract_well_pump_type(row) if row.ConstructionNotes else None + ) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation @@ -267,6 +285,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None if not isna(row.ConstructionMethod) else None ), + well_pump_type=well_pump_type, ) CreateWell.model_validate(data) From 4e9876e1f23369283ba9588c834c6d54b5744ef6 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 21 Nov 2025 11:55:58 -0700 Subject: [PATCH 060/119] fix: change Air-rotary to Air-Rotary to correspond with lu table This can be updated later on but should be "Air-Rotary" for the transfer --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index 21666bccd..5821330b5 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -81,7 +81,7 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"}, {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, From e9639bff8de328c9cf9cd213b3c0366c6119a6f5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 23 Nov 2025 14:46:08 -0700 Subject: [PATCH 061/119] feat(model): Add `AquiferType` model and rename `aquifer_type` to `primary_aquifer_type` - Add AquiferType model to store multiple aquifer characteristics per association - Rename AquiferSystem.aquifer_type to primary_aquifer_type for clarity - Update ThingAquiferAssociation with aquifer_types relationship - Update related schemas to use primary_type This enables capturing wells that encounter multiple aquifer characteristics (e.g., both fractured and confined) without compound naming. --- db/__init__.py | 1 + db/aquifer_system.py | 9 +++-- db/aquifer_type.py | 58 +++++++++++++++++++++++++++++++++ db/thing_aquifer_association.py | 8 +++++ schemas/aquifer_system.py | 16 ++++++++- 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 db/aquifer_type.py diff --git a/db/__init__.py b/db/__init__.py index ef14f7c06..65ab4cd32 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -44,6 +44,7 @@ from db.geologic_formation import * from db.thing_aquifer_association import * from db.thing_geologic_formation_association import * +from db.aquifer_type import * from sqlalchemy import ( func, diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 95066c2e8..b72e91949 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from db.thing import WellScreen, ThingAquiferAssociation, Thing + from db.aquifer_type import AquiferType class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): @@ -34,9 +35,9 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): comment="A detailed description of the aquifer system, its characteristics, and its significance.", ) # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. - aquifer_type: Mapped[str] = lexicon_term( + primary_aquifer_type: Mapped[str] = lexicon_term( nullable=False, - comment="A controlled vocabulary field to classify the aquifer's hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + comment="A controlled vocabulary field to classify the aquifer's primary hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", ) geographic_scale: Mapped[str] = lexicon_term( nullable=False, @@ -74,6 +75,10 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): things: AssociationProxy[List["Thing"]] = association_proxy( "thing_associations", "thing" ) + # Proxy to directly access all AquiferTypes associated with this AquiferSystem. + aquifer_types: AssociationProxy[List["AquiferType"]] = association_proxy( + "thing_associations", "aquifer_types" + ) # --- Table Arguments --- __table_args__ = Index("ix_aquifersystem_name", "name") diff --git a/db/aquifer_type.py b/db/aquifer_type.py new file mode 100644 index 000000000..32900d801 --- /dev/null +++ b/db/aquifer_type.py @@ -0,0 +1,58 @@ +""" +SQLAlchemy model for the AquiferType table. + +This table stores the specific aquifer characteristics/types associated with +a Thing-AquiferSystem relationship. It allows capturing that a single aquifer +can have multiple characteristics simultaneously. + +Example: + A well in the "Ogallala" aquifer might tap portions that are both + "Fractured" AND "Confined". This would create: + - One AquiferSystem: "Ogallala" + - One ThingAquiferAssociation: linking well to Ogallala + - Two AquiferType records: "Fractured" and "Confined" +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.thing_aquifer_association import ThingAquiferAssociation + + +class AquiferType(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the specific aquifer types/characteristics for a + Thing-AquiferSystem association. + + This allows modeling the fact that: + - A single aquifer can have multiple characteristics + - Different wells may tap different characteristics of the same aquifer + - Characteristics are attributes of the relationship, not the aquifer itself + + Fields from WellData CSV: + - AquiferType: May contain multiple codes (e.g., "FC" = Fractured + Confined) + - Each code becomes a separate AquiferType record + """ + + # --- Columns --- + thing_aquifer_association_id: Mapped[int] = mapped_column( + ForeignKey("thing_aquifer_association.id", ondelete="CASCADE"), + nullable=False, + comment="Links to the Thing-Aquifer association this type describes.", + ) + aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="Controlled vocabulary for aquifer hydrologic properties. " + "Examples: 'Unconfined', 'Confined', 'Perched', 'Fractured', 'Unconsolidated'.", + ) + + # --- Relationships --- + # Many-to-One: Multiple aquifer types can belong to one association + thing_aquifer_association: Mapped["ThingAquiferAssociation"] = relationship( + "ThingAquiferAssociation", back_populates="aquifer_types" + ) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index d5fb2881a..414f09269 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from db.thing import Thing from db.aquifer_system import AquiferSystem + from db.aquifer_type import AquiferType class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): @@ -40,3 +41,10 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): aquifer_system: Mapped["AquiferSystem"] = relationship( "AquiferSystem", back_populates="thing_associations", lazy="joined" ) + # One-To-Many: An association can have multiple aquifer types. + aquifer_types: Mapped[list["AquiferType"]] = relationship( + "AquiferType", + back_populates="thing_aquifer_association", + cascade="all, delete-orphan", + passive_deletes=True, + ) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 5f5b3ed4d..4b4448211 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -5,6 +5,20 @@ from schemas import BaseResponseModel +# ------ CREATE ---------- +class CreateAquiferSystem(BaseModel): + """ + Schema for creating an aquifer system. + Used during data transfer and API creation. + """ + + name: str + description: str | None = None + primary_aquifer_type: str + geographic_scale: str + boundary: str | None = None + + # ------ RESPONSE ---------- class GeoJSONGeometry(BaseModel): """ @@ -27,6 +41,6 @@ class AquiferSystemResponse(BaseResponseModel): name: str description: str | None = None - aquifer_type: str + primary_aquifer_type: str geographic_scale: str boundary: GeoJSONGeometry | None = None From af07c3e20ddaf2c9cafbc2f2c52cc4c4c646fda1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 09:36:39 -0700 Subject: [PATCH 062/119] feat(transfer): WIP aquifer and geology transfers --- transfers/aquifer_system_transfer.py | 139 +++++++++++++++++++++++ transfers/geologic_formation_transfer.py | 85 ++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 transfers/aquifer_system_transfer.py create mode 100644 transfers/geologic_formation_transfer.py diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py new file mode 100644 index 000000000..0de7cc7ae --- /dev/null +++ b/transfers/aquifer_system_transfer.py @@ -0,0 +1,139 @@ +import time +from sqlalchemy.orm import Session +from pydantic import ValidationError + +from db import AquiferSystem +from schemas.aquifer_system import CreateAquiferSystem +from transfers.util import read_csv, replace_nans, logger + + +def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: + """ + Transfer aquifer system data from LU_AquiferClass CSV to the database. + + This creates the master list of named aquifer systems (e.g., Ogallala Aquifer). the primary_type field is set + to "Unknown" as a placeholder and will be updated during well transfer when we know what type each well encounters. + + This should be run BEFORE well_transfer.py so that aquifer records exist for wells to reference. + + Args: + session (Session): SQLAlchemy database session + limit (int, optional): Limit the number of records to transfer (for testing). + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read the CSV file + input_df = read_csv("LU_AquiferClass") + + # 2. Replace NaNs with NOne + cleaned_df = replace_nans(input_df) + + # 3. Initialize tracking variables for logging + n = len(input_df) + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + + logger.info(f"Starting transfer of {n} aquifer systems from LU_AquiferClass.") + + # 4. Process each row + for i, row in enumerate(cleaned_df.itertuples()): + # check if limit is reached + if limit and i >= limit: + logger.info(f"Reached limit of {limit} rows. Stopping migration.") + break + + # Log progress every 'step' rows + if i and not i % 25: + logger.info( + f"Processing row {i} of {n}. Avg rows per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Commit progress periodically + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing aquifer system {i}: {e}") + session.rollback() + continue + + # 5. Extract aquifer code and name + aquifer_code = row.CODE + aquifer_name = row.MEANING + + if not aquifer_name: + logger.warning( + f"Row {i} (code: {aquifer_code}) has no aquifer name (MEANING). Skipping." + ) + skipped_count += 1 + continue + + # 6. Check if aquifer system already exists + existing = ( + session.query(AquiferSystem) + .filter(AquiferSystem.name == aquifer_name) + .first() + ) + + if existing: + logger.info( + f"Aquifer '{aquifer_name}' (code: {aquifer_code}) already exists. Skipping." + ) + skipped_count += 1 + continue + + # 7. Prepare data dictionary + try: + data = CreateAquiferSystem( + name=aquifer_name, + description=None, # can be updated later + primary_aquifer_type="Unknown", # placeholder - will be updated during well transfer + ) + + # Validate data using Pydantic schema + CreateAquiferSystem.model_validate(data) + + except ValidationError as e: + errors.append({"code": aquifer_code, "name": aquifer_name, "error": str(e)}) + logger.critical( + f"Error creating aquifer system '{aquifer_name}' (code: {aquifer_code}) (row {i}): {e}" + ) + continue + + # 8. Create database record + aquifer_system = None + try: + aquifer_data = data.model_dump() + aquifer_system = AquiferSystem(**aquifer_data) + session.add(aquifer_system) + created_count += 1 + + logger.info( + f"Created aquifer system: {aquifer_system.name} (code: {aquifer_code})" + ) + + except Exception as e: + if aquifer_system is not None: + session.expunge(aquifer_system) + errors.append({"code": aquifer_code, "name": aquifer_name, "error": str(e)}) + logger.critical( + f"Error creating aquifer system record '{aquifer_name}': {e}" + ) + continue + + # 9. Final commit + try: + session.commit() + logger.info( + f"Successfully transferred {created_count} aquifer systems, skipped {skipped_count}. " + f"Note: primary_type set to 'Unknown' and will be updated during well transfer." + ) + except Exception as e: + logger.critical(f"Error in final commit: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/geologic_formation_transfer.py b/transfers/geologic_formation_transfer.py new file mode 100644 index 000000000..202c9d431 --- /dev/null +++ b/transfers/geologic_formation_transfer.py @@ -0,0 +1,85 @@ +import time +from sqlalchemy.orm import Session +from pydantic import ValidationError + +from db import GeologicFormation +from schemas.geologic_formation import CreateGeologicFormation +from transfers.util import read_csv, replace_nans, lexicon_mapper, logger + + +def transfer_geologic_formations(session: Session, limit: int = None) -> tuple: + """ + Transfer geologic formation data from LU_GeologicFormation CSV to the database. + + This should be run BEFORE well_transfer.py so that geologic formation records exist for wells to reference. + + Args: + session (Session): SQLAlchemy database session + limit (int, optional): Optional limit on number of records to transfer (for testing). + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read the CSV file + input_df = read_csv("LU_Formation") + + # 2. Replace NaNs with None + cleaned_df = replace_nans(input_df) + + # 3. Initialize tracking variables for logging + n = len(cleaned_df) + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + + logger.info(f"Starting transfer of {n} geologic formations") + + # 4. Process each row + for i, row in enumerate(cleaned_df.itertuples()): + # check if limit is reached + if limit and i >= limit: + logger.info(f"Reached limit of {limit} rows. Stopping migration.") + break + + # Log progress every 'step' rows + if i and not i % step: + logger.info( + f"Processing row {i} of {n}. Avg rows per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Commit progress periodically + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing geologic formation {i}: {e}") + session.rollback() + continue + + try: + payload = CreateGeologicFormation( + name=row.GeologicFormationName, + description=row.Description, + lithology=lexicon_mapper("Lithology", row.Lithology), + age=lexicon_mapper("GeologicAge", row.Age), + ) + formation = GeologicFormation(**payload.dict()) + session.add(formation) + created_count += 1 + except ValidationError as e: + error_msg = f"Validation error for row {i} with GeologicFormationName {row.GeologicFormationName}: {e.errors()}" + logger.critical(error_msg) + errors.append(error_msg) + except Exception as e: + error_msg = f"Error creating geologic formation for {row.GeologicFormationName}: {e}" + logger.critical(error_msg) + errors.append(error_msg) + continue + + # Final commit after all rows are processed + try: + session.commit() + except Exception as e: + logger.critical(f"Error during final commit of geologic formations: {e}") From ba2b2964550ffc7def0d12dd5cd7fa495f0b5401 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 11:13:21 -0700 Subject: [PATCH 063/119] feat(model): add eager loading Add eager loading to aquifer and geology relationships to the `thing` model. --- db/aquifer_system.py | 2 +- db/thing.py | 2 ++ db/thing_aquifer_association.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index b72e91949..53086f983 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -37,7 +37,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. primary_aquifer_type: Mapped[str] = lexicon_term( nullable=False, - comment="A controlled vocabulary field to classify the aquifer's primary hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + comment="A controlled vocabulary field to classify the aquifer system as a whole (e.g., 'Unconfined', 'Confined', 'Perched').", ) geographic_scale: Mapped[str] = lexicon_term( nullable=False, diff --git a/db/thing.py b/db/thing.py index 0bb361294..d2592f7ac 100644 --- a/db/thing.py +++ b/db/thing.py @@ -290,6 +290,7 @@ class Thing( back_populates="thing", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) # Many-To-Many: A Thing can penetrate many GeologicFormations. @@ -299,6 +300,7 @@ class Thing( back_populates="thing", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) ) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index 414f09269..cca5758a9 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -47,4 +47,5 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): back_populates="thing_aquifer_association", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) From 910d5b31bbb942894a66236a09a97bdd0b8e2a8f Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 11:45:51 -0700 Subject: [PATCH 064/119] feat(schema): refactor GeoJSON responses for `aquifer_system` and `geologic_formation` --- schemas/aquifer_system.py | 22 +++++++++++++--------- schemas/geologic_formation.py | 24 ++++++++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 4b4448211..6c077aa90 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -25,16 +25,11 @@ class GeoJSONGeometry(BaseModel): Geometry schema for GeoJSON response. """ - type: str - coordinates: ( - List[float] - | List[List[float]] - | List[List[List[float]]] - | List[List[List[List[float]]]] - ) + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] -class AquiferSystemResponse(BaseResponseModel): +class GeoJSONProperties(BaseResponseModel): """ Response schema for aquifer system details. """ @@ -43,4 +38,13 @@ class AquiferSystemResponse(BaseResponseModel): description: str | None = None primary_aquifer_type: str geographic_scale: str - boundary: GeoJSONGeometry | None = None + + +class AquiferSystemGeoJSONResponse(BaseModel): + """ + Response schema for aquifer system details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index f6b3083d3..d42e48389 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -11,16 +11,11 @@ class GeoJSONGeometry(BaseModel): Geometry schema for GeoJSON response. """ - type: str - coordinates: ( - List[float] - | List[List[float]] - | List[List[List[float]]] - | List[List[List[List[float]]]] - ) + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] -class GeologicFormationResponse(BaseResponseModel): +class GeoJSONProperties(BaseResponseModel): """ Response schema for geologic formation details. """ @@ -28,7 +23,16 @@ class GeologicFormationResponse(BaseResponseModel): formation_code: str | None = None description: str | None = None lithology: str | None = None - boundary: GeoJSONGeometry | None = None + + +class GeologicFormationGeoJSONResponse(BaseModel): + """ + Response schema for geologic formation details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties class ThingGeologicFormationAssociationResponse(BaseResponseModel): @@ -39,7 +43,7 @@ class ThingGeologicFormationAssociationResponse(BaseResponseModel): thing_id: int geologic_formation_id: int | None = None - geologic_formation: GeologicFormationResponse | None = None + geologic_formation: GeologicFormationGeoJSONResponse | None = None top_depth: float top_depth_unit: str = "ft" bottom_depth: float From 1342b8d30852d9cb47a845b29c55cd3179d403fc Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 12:54:53 -0700 Subject: [PATCH 065/119] fix: fix __table_args__ for aquifer and geology --- db/aquifer_system.py | 2 +- db/geologic_formation.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 53086f983..6a1681561 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -81,4 +81,4 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): ) # --- Table Arguments --- - __table_args__ = Index("ix_aquifersystem_name", "name") + __table_args__ = (Index("ix_aquifersystem_name", "name"),) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index 130ed8d45..a9483b501 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -79,7 +79,4 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing") # --- Table Arguments --- - __table_args__ = ( - Index("ix_geologicformation_name", "name"), - Index("ix_geologicformation_code", "code"), - ) + __table_args__ = (Index("ix_geologicformation_formation_code", "formation_code"),) From 88419b7911bf38070fb899c54dae25ddaba247ed Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:00:59 -0700 Subject: [PATCH 066/119] fix: add geologic_formation relation to WellScreen This was missing on the WellScreen side of the relationship --- db/geologic_formation.py | 2 +- db/thing.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a9483b501..2379f50f4 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -70,7 +70,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): ) ) # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. - screens: Mapped[List["WellScreen"]] = relationship( + well_screens: Mapped[List["WellScreen"]] = relationship( "WellScreen", back_populates="geologic_formation", passive_deletes=True ) diff --git a/db/thing.py b/db/thing.py index bf7e30db1..3c6ecf3db 100644 --- a/db/thing.py +++ b/db/thing.py @@ -528,6 +528,10 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): "AquiferSystem", back_populates="well_screens", passive_deletes=True ) + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="well_screens", passive_deletes=True + ) + class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): """ From 1a23dff0790e991edf6366db27379fc99788d9be Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:04:35 -0700 Subject: [PATCH 067/119] fix: add missing comma --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index ec195b6b1..44a89d2aa 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -55,7 +55,7 @@ {"name": "status_value", "description": null}, {"name": "origin_source", "description": null}, {"name": "well_pump_type", "description": null}, - {"name": "permission_type", "description": null} + {"name": "permission_type", "description": null}, {"name": "formation_code", "description": null}, {"name": "lithology", "description": null} ], From d82b16373700afac658f35ffb542645b273d58f8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:33:11 -0700 Subject: [PATCH 068/119] fix: use BaseResponseModel for GeoJSONProperties in location schema --- schemas/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/location.py b/schemas/location.py index e911e3359..81c1ddd2d 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -97,7 +97,7 @@ class GeoJSONUTMCoordinates(BaseModel): ) -class GeoJSONProperties(BaseModel): +class GeoJSONProperties(BaseResponseModel): elevation: float elevation_unit: str = "ft" vertical_datum: str = "NAVD88" From 20af430d00ceade9a6beecf63404058598c32be8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:35:25 -0700 Subject: [PATCH 069/119] fix: include baseresponseinfo in location geojson properties --- schemas/location.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/location.py b/schemas/location.py index 81c1ddd2d..218790496 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -147,6 +147,9 @@ def populate_fields(cls, data: Any) -> Any: data_dict["geometry"]["coordinates"] = coordinates # populate properties + data_dict["properties"]["id"] = data_dict.get("id") + data_dict["properties"]["created_at"] = data_dict.get("created_at") + data_dict["properties"]["release_status"] = data_dict.get("release_status") data_dict["properties"]["notes"] = data_dict.get("notes") data_dict["properties"]["elevation"] = convert_m_to_ft(elevation_m) data_dict["properties"]["elevation_method"] = data_dict.get("elevation_method") From 5778b741a607810b4133080e59f34e31718ed082 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:36:35 -0700 Subject: [PATCH 070/119] feat: implement aquifer_systems in thing response --- db/thing.py | 2 +- schemas/thing.py | 23 ++++++++++++++----- tests/features/environment.py | 23 ++++++++++++++++++- .../steps/well-additional-information.py | 12 +++------- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/db/thing.py b/db/thing.py index 3c6ecf3db..340d8b0e9 100644 --- a/db/thing.py +++ b/db/thing.py @@ -332,7 +332,7 @@ class Thing( ) # Proxy to directly access AquiferSystems associated with this Thing - aquifers: AssociationProxy[List["AquiferSystem"]] = association_proxy( + aquifer_systems: AssociationProxy[List["AquiferSystem"]] = association_proxy( "aquifer_associations", "aquifer_system" ) diff --git a/schemas/thing.py b/schemas/thing.py index aecccd3e0..47be1021f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,10 +33,11 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemResponse -from schemas.geologic_formation import ( - GeologicFormationResponse, -) +from schemas.aquifer_system import AquiferSystemGeoJSONResponse + +# from schemas.geologic_formation import ( +# GeologicFormationResponse, +# ) # -------- VALIDATE ---------- @@ -245,6 +246,7 @@ class WellResponse(BaseThingResponse): measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None + aquifer_systems: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None @@ -269,6 +271,14 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("aquifer_systems", mode="before") + def populate_aquifer_types_with_strings(cls, aquifer_systems): + if aquifer_systems is not None: + systems = [aquifer_system.name for aquifer_system in aquifer_systems] + else: + systems = [] + return systems + class SpringResponse(BaseThingResponse): """ @@ -291,9 +301,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemResponse | None = None + aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_type: str | None = None geologic_formation_id: int | None = None - geologic_formation: GeologicFormationResponse | None = None + # geologic_formation: GeologicFormationResponse | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float diff --git a/tests/features/environment.py b/tests/features/environment.py index ec5070bcc..0b2ca9d9f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -37,6 +37,7 @@ MeasuringPointHistory, MonitoringFrequencyHistory, DataProvenance, + AquiferSystem, ) from db.engine import session_ctx @@ -423,9 +424,26 @@ def add_transducer_observation(context, session, block, deployment_id, value): return obs +@add_context_object_container("aquifer_systems") +def add_aquifer_system(context, session, name, well): + aquifer_system = AquiferSystem( + name=name, + description="this is a test aquifer", + primary_aquifer_type="Artesian", + geographic_scale="Major", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(aquifer_system) + session.commit() + session.refresh(aquifer_system) + + context.objects["aquifer_systems"].append(aquifer_system) + return aquifer_system + + def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True if rebuild: erase_and_rebuild_db() @@ -631,6 +649,9 @@ def before_all(context): for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) + for name in ["Aquifer A", "Aquifer B"]: + add_aquifer_system(context, session, name, well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) block = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 0824a0246..a9d6833f5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -55,7 +55,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): assert "well_completion_date" in context.water_well_data @@ -74,7 +73,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): assert "well_driller_name" in context.water_well_data @@ -107,7 +105,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): assert "well_casing_diameter" in context.water_well_data @@ -140,8 +137,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.water_well_data @@ -151,7 +146,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): assert "well_pump_depth" in context.water_well_data @@ -194,10 +188,10 @@ def step_impl(context): "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - assert "aquifer_class_code" in context.water_well_data + assert "aquifer_systems" in context.water_well_data assert ( - context.water_well_data["aquifer_class_code"] - == context.objects["wells"][0].aquifer_class_code + context.water_well_data["aquifer_systems"] + == context.objects["wells"][0].aquifer_systems ) From c4f9de0dfc9f5a41811a0dbdda15319db86d95df Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:50:55 -0700 Subject: [PATCH 071/119] feat: fix aquifer systems test --- tests/features/environment.py | 5 +++++ tests/features/steps/well-additional-information.py | 6 ++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/features/environment.py b/tests/features/environment.py index 0b2ca9d9f..8c683fac8 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -38,6 +38,7 @@ MonitoringFrequencyHistory, DataProvenance, AquiferSystem, + ThingAquiferAssociation, ) from db.engine import session_ctx @@ -437,6 +438,10 @@ def add_aquifer_system(context, session, name, well): session.commit() session.refresh(aquifer_system) + association = ThingAquiferAssociation(thing=well, aquifer_system=aquifer_system) + session.add(association) + session.commit() + context.objects["aquifer_systems"].append(aquifer_system) return aquifer_system diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index a9d6833f5..dbaaba0ed 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -183,15 +183,13 @@ def step_impl(context): assert context.water_well_data["formation"] == context.objects["wells"][0].formation -# TODO: needs to be added to model, schemas, test data, lexicon @then( "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): assert "aquifer_systems" in context.water_well_data - assert ( - context.water_well_data["aquifer_systems"] - == context.objects["wells"][0].aquifer_systems + assert sorted(context.water_well_data["aquifer_systems"]) == sorted( + [system.name for system in context.objects["aquifer_systems"]] ) From d3abd833144c832800f48d69746c34c3e9adfd81 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 15:45:07 -0700 Subject: [PATCH 072/119] fix: return aquifer system and types in dict this allows us to couple types to a system incase a well is in multiple aquifers --- db/thing.py | 17 +++++++++ schemas/thing.py | 11 +----- tests/features/environment.py | 35 ++++++++++++++++--- .../steps/well-additional-information.py | 25 +++++++------ 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/db/thing.py b/db/thing.py index 340d8b0e9..35c61b631 100644 --- a/db/thing.py +++ b/db/thing.py @@ -476,6 +476,23 @@ def allow_datalogger_installation(self): ) return permission_record.permission_allowed if permission_record else None + @property + def aquifers(self) -> List[dict]: + """ + Returns a list of aquifer systems and their associated types for this Thing. + Each aquifer system is represented as a dictionary with its name and a list of types. + """ + aquifer_list = [] + for association in self.aquifer_associations: + aquifer_info = { + "aquifer_system": association.aquifer_system.name, + "aquifer_types": [ + atype.aquifer_type for atype in association.aquifer_types + ], + } + aquifer_list.append(aquifer_info) + return aquifer_list + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/thing.py b/schemas/thing.py index 47be1021f..0ff765d76 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -246,8 +246,7 @@ class WellResponse(BaseThingResponse): measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None - aquifer_systems: list[str] = [] - + aquifers: list[dict] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -271,14 +270,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("aquifer_systems", mode="before") - def populate_aquifer_types_with_strings(cls, aquifer_systems): - if aquifer_systems is not None: - systems = [aquifer_system.name for aquifer_system in aquifer_systems] - else: - systems = [] - return systems - class SpringResponse(BaseThingResponse): """ diff --git a/tests/features/environment.py b/tests/features/environment.py index 8c683fac8..d30f920ec 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -38,6 +38,7 @@ MonitoringFrequencyHistory, DataProvenance, AquiferSystem, + AquiferType, ThingAquiferAssociation, ) from db.engine import session_ctx @@ -438,17 +439,38 @@ def add_aquifer_system(context, session, name, well): session.commit() session.refresh(aquifer_system) + context.objects["aquifer_systems"].append(aquifer_system) + return aquifer_system + + +@add_context_object_container("thing_aquifer_associations") +def add_thing_aquifer_association(context, session, well, aquifer_system): association = ThingAquiferAssociation(thing=well, aquifer_system=aquifer_system) session.add(association) session.commit() + session.refresh(association) - context.objects["aquifer_systems"].append(aquifer_system) - return aquifer_system + context.objects["thing_aquifer_associations"].append(association) + return association + + +@add_context_object_container("aquifer_types") +def add_aquifer_type(context, session, aquifer_type_str, thing_aquifer_association): + aquifer_type = AquiferType( + aquifer_type=aquifer_type_str, + thing_aquifer_association=thing_aquifer_association, + ) + session.add(aquifer_type) + session.commit() + session.refresh(aquifer_type) + + context.objects["aquifer_types"].append(aquifer_type) + return aquifer_type def before_all(context): context.objects = {} - rebuild = True + rebuild = False # rebuild = True if rebuild: erase_and_rebuild_db() @@ -655,7 +677,12 @@ def before_all(context): add_well_purpose(context, session, well_1, purpose) for name in ["Aquifer A", "Aquifer B"]: - add_aquifer_system(context, session, name, well_1) + system = add_aquifer_system(context, session, name, well_1) + add_thing_aquifer_association(context, session, well_1, system) + + for t in ["Artesian", "Fractured"]: + taa = context.objects["thing_aquifer_associations"][0] + add_aquifer_type(context, session, t, taa) # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index dbaaba0ed..aba64d37c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -187,20 +187,23 @@ def step_impl(context): "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - assert "aquifer_systems" in context.water_well_data - assert sorted(context.water_well_data["aquifer_systems"]) == sorted( - [system.name for system in context.objects["aquifer_systems"]] - ) + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_system" in aquifer + assert sorted( + [a.get("aquifer_system") for a in context.water_well_data["aquifers"]] + ) == sorted([system.name for system in context.objects["aquifer_systems"]]) -# TODO: needs to be added to model, schemas, test data -# TODO: should this be plural? that is, a descriptor model of the well @then( "the response should include the aquifer type as the type of aquifers penetrated by the well" ) def step_impl(context): - assert "aquifer_type" in context.water_well_data - assert ( - context.water_well_data["aquifer_type"] - == context.objects["wells"][0].aquifer_type - ) + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_types" in aquifer + + if aquifer["aquifer_system"] == "Aquifer A": + assert sorted(aquifer["aquifer_types"]) == sorted( + [a.aquifer_type for a in context.objects["aquifer_types"]] + ) + else: + assert aquifer["aquifer_types"] == [] From b4cd84e001ff33267d7bc71075c413c4c7f2dae7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:01:02 -0700 Subject: [PATCH 073/119] feat: implement geologic_formations from feature files --- db/thing.py | 4 +-- schemas/thing.py | 9 +++++++ tests/features/environment.py | 27 +++++++++++++++++++ .../steps/well-additional-information.py | 7 ++--- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/db/thing.py b/db/thing.py index 35c61b631..b08a14747 100644 --- a/db/thing.py +++ b/db/thing.py @@ -337,8 +337,8 @@ class Thing( ) # Proxy to directly access the GeologicFormations penetrated by this Thing. - formations: AssociationProxy[List["GeologicFormation"]] = association_proxy( - "formation_associations", "geologic_formation" + geologic_formations: AssociationProxy[List["GeologicFormation"]] = ( + association_proxy("formation_associations", "geologic_formation") ) # Full-text search vector diff --git a/schemas/thing.py b/schemas/thing.py index 0ff765d76..bd7665f92 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -247,6 +247,7 @@ class WellResponse(BaseThingResponse): measuring_point_height_unit: str = "ft" measuring_point_description: str | None aquifers: list[dict] = [] + geologic_formations: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -270,6 +271,14 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("geologic_formations", mode="before") + def populate_geologic_formations_with_strings(cls, geologic_formations): + if geologic_formations is not None: + formations = [formation.formation_code for formation in geologic_formations] + else: + formations = [] + return formations + class SpringResponse(BaseThingResponse): """ diff --git a/tests/features/environment.py b/tests/features/environment.py index d30f920ec..e2959ac85 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -40,6 +40,8 @@ AquiferSystem, AquiferType, ThingAquiferAssociation, + GeologicFormation, + ThingGeologicFormationAssociation, ) from db.engine import session_ctx @@ -468,6 +470,29 @@ def add_aquifer_type(context, session, aquifer_type_str, thing_aquifer_associati return aquifer_type +@add_context_object_container("geologic_formations") +def add_geologic_formation(context, session, formation_code, well): + formation = GeologicFormation( + formation_code=formation_code, + description="This is a test geologic formation.", + lithology="Peat", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(formation) + session.commit() + session.refresh(formation) + + association = ThingGeologicFormationAssociation( + top_depth=1, bottom_depth=10, thing=well, geologic_formation=formation + ) + session.add(association) + session.commit() + session.refresh(association) + + context.objects["geologic_formations"].append(formation) + return formation + + def before_all(context): context.objects = {} rebuild = False @@ -684,6 +709,8 @@ def before_all(context): taa = context.objects["thing_aquifer_associations"][0] add_aquifer_type(context, session, t, taa) + add_geologic_formation(context, session, "000EXRV", well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) block = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index aba64d37c..638c6529f 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -174,13 +174,14 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then( "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "formation" in context.water_well_data - assert context.water_well_data["formation"] == context.objects["wells"][0].formation + assert "geologic_formations" in context.water_well_data + assert context.water_well_data["geologic_formations"] == [ + context.objects["geologic_formations"][0].formation_code + ] @then( From fbfa9089c29e9539025cec8cb8bf60195fb0520b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:20:35 -0700 Subject: [PATCH 074/119] fix: remove duplicate well pump types in lexicon this was an artifact from merge conflicts --- core/lexicon.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 990a5d657..8ef165e1a 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -699,10 +699,6 @@ {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, - {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, - {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, - {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, From 748a57b9baa542c84f6fce8b6160d520300d0558 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 16:21:48 -0700 Subject: [PATCH 075/119] feat(transfer): add stratigraphy transfer script and update geologic formation transfers. Add stratigraphy_transfer.py to handle detailed lithology log import, create well-formation associations, and update formation lithology fields from stratigraphy data. This script is essential for linking wells to geologic formations with depth intervals. --- transfers/geologic_formation_transfer.py | 90 ++++++-- transfers/stratigraphy_transfer.py | 251 +++++++++++++++++++++++ transfers/util.py | 1 + 3 files changed, 325 insertions(+), 17 deletions(-) create mode 100644 transfers/stratigraphy_transfer.py diff --git a/transfers/geologic_formation_transfer.py b/transfers/geologic_formation_transfer.py index 202c9d431..724eb79fe 100644 --- a/transfers/geologic_formation_transfer.py +++ b/transfers/geologic_formation_transfer.py @@ -4,7 +4,7 @@ from db import GeologicFormation from schemas.geologic_formation import CreateGeologicFormation -from transfers.util import read_csv, replace_nans, lexicon_mapper, logger +from transfers.util import read_csv, replace_nans, logger def transfer_geologic_formations(session: Session, limit: int = None) -> tuple: @@ -54,32 +54,88 @@ def transfer_geologic_formations(session: Session, limit: int = None) -> tuple: try: session.commit() except Exception as e: - logger.critical(f"Error committing geologic formation {i}: {e}") + logger.critical(f"Error committing geologic formations: {e}") session.rollback() continue + # 5. Extract formation code and description + formation_code = row.Code + + if not formation_code: + logger.warning(f"Skipping row {i}: Missing formation code") + skipped_count += 1 + continue + + # Check if this formation already exists + existing = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if existing: + logger.info( + f"Skipping row {i}: Formation code {formation_code} already exists" + ) + skipped_count += 1 + continue + + # 6. Prepare data for creation + # Note: We only store the formation_code. Formation names will be mapped by the API using a + # formations.json file from authoritative sources (e.g., USGS). + # The description field is left as None and can be populated later if needed. + # Note: lithology is set to None here and will be updated during stratigraphy transfer try: - payload = CreateGeologicFormation( - name=row.GeologicFormationName, - description=row.Description, - lithology=lexicon_mapper("Lithology", row.Lithology), - age=lexicon_mapper("GeologicAge", row.Age), + data = CreateGeologicFormation( + formation_code=formation_code, + description=None, # Not storing from legacy data + lithology=None, # Will be populated from Stratigraphy.csv ) - formation = GeologicFormation(**payload.dict()) - session.add(formation) - created_count += 1 + + # Validate the data using Pydantic schema + CreateGeologicFormation.model_validate(data) + except ValidationError as e: - error_msg = f"Validation error for row {i} with GeologicFormationName {row.GeologicFormationName}: {e.errors()}" - logger.critical(error_msg) - errors.append(error_msg) + errors.append({"code": formation_code, "errors": e.errors()}) + logger.critical( + f"Validation error for row {i} with Code {formation_code}: {e.errors()}" + ) + continue + except Exception as e: + errors.append({"code": formation_code, "errors": str(e)}) + logger.critical(f"Error preparing data for {formation_code}: {e}") + continue + + # 7. Create database object + geologic_formation = None + try: + formation_data = data.model_dump() + geologic_formation = GeologicFormation(**formation_data) + session.add(geologic_formation) + created_count += 1 + + logger.info( + f"Created geologic formation: {geologic_formation.formation_code}" + ) + except Exception as e: - error_msg = f"Error creating geologic formation for {row.GeologicFormationName}: {e}" - logger.critical(error_msg) - errors.append(error_msg) + if geologic_formation is not None: + session.expunge(geologic_formation) + errors.append({"code": formation_code, "error": str(e)}) + logger.critical( + f"Error creating geologic formation for {formation_code}: {e}" + ) continue - # Final commit after all rows are processed + # 8. Final commit try: session.commit() + logger.info( + f"Successfully transferred {created_count} geologic formations, skipped {skipped_count}. " + f"Note: lithology is None and will be updated during stratigraphy transfer." + ) except Exception as e: logger.critical(f"Error during final commit of geologic formations: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/stratigraphy_transfer.py b/transfers/stratigraphy_transfer.py new file mode 100644 index 000000000..ac74a2d9c --- /dev/null +++ b/transfers/stratigraphy_transfer.py @@ -0,0 +1,251 @@ +""" +Transfer script for stratigraphy (lithology log) data. + +This creates ThingGeologicFormationAssociation records from the Stratigraphy CSV, which contains depth-specific +formation information for wells. It also updates the GeologicFormation.lithology field based on the +Stratigraphy.Lithology data. +""" + +import time +from sqlalchemy.orm import Session + +from db import Thing, GeologicFormation, ThingGeologicFormationAssociation +from transfers.util import ( + read_csv, + replace_nans, + filter_to_valid_point_ids, + lexicon_mapper, + logger, +) + + +def transfer_stratigraphy(session: Session, limit: int = None) -> tuple: + """ + Transfer detailed stratigraphy (lithology log) data from Stratigraphy CSV. + + The Stratigraphy CSV contains multiple rows per well, each representing a + depth interval, the formation encountered, and its lithology. + + Fields used: + - PointID: Links to the well + - UnitIdentifier: Formation code (maps to LU_Formations) + - StratTop: Top depth of the layer (feet below ground surface) + - StratBottom: Bottom depth of the layer (feet below ground surface) + - Lithology: Lithology code (maps to LU_Lithology via ABBREVIATION field) + + This should be run AFTER: + 1. transfer_geologic_formations.py (so formations exist) + 2. transfer_wells.py (so wells exist) + + Args: + session: Database session + limit: Optional limit on number of WELLS to process (for testing) + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read and clean data + input_df = read_csv("Stratigraphy") + cleaned_df = replace_nans(input_df) + + # Step 2: Filter to only wells that exist in database + cleaned_df = filter_to_valid_point_ids(session, cleaned_df) + + n_records = len(cleaned_df) + n_wells = len(cleaned_df["PointID"].unique()) + + logger.info( + f"Starting transfer of {n_records} stratigraphy records for {n_wells} wells" + ) + + # 3. Initialize tracking variables for logging + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + lithology_updates = 0 + + # Step 4: Group by well for efficient processing + well_groups = cleaned_df.groupby("PointID") + + for well_index, (pointid, strat_group) in enumerate(well_groups): + # Check limit (on number of wells, not records) + if limit and well_index >= limit: + logger.info(f"Reached limit of {limit} wells. Stopping.") + break + + # Progress logging every 25 wells + if well_index and not well_index % step: + logger.info( + f"Processing well {well_index} of {n_wells}, " + f"avg wells per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Periodic commit + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing stratigraphy records: {e}") + session.rollback() + continue + + # 5. Get the well from database + thing = session.query(Thing).filter(Thing.name == pointid).first() + if not thing: + logger.warning( + f"Well {pointid} not found in database, skipping stratigraphy" + ) + skipped_count += len(strat_group) + continue + + logger.info( + f"Processing {len(strat_group)} stratigraphy layers for well {pointid}" + ) + + # 6. Process each stratigraphy record for this well + for layer_index, row in enumerate(strat_group.itertuples()): + # Validate required fields + # UnitIdentifier + if not hasattr(row, "UnitIdentifier") or not row.UnitIdentifier: + logger.warning( + f"Stratigraphy record {layer_index} for {pointid} has no UnitIdentifier, skipping" + ) + skipped_count += 1 + continue + # StratTop + if not hasattr(row, "StratTop") or row.StratTop is None: + logger.warning( + f"Stratigraphy record {layer_index} for {pointid} has no StratTop, skipping" + ) + skipped_count += 1 + continue + # StratBottom + if not hasattr(row, "StratBottom") or row.StratBottom is None: + logger.warning( + f"Stratigraphy record {layer_index} for {pointid} has no StratBottom, skipping" + ) + skipped_count += 1 + continue + + # Extract formation code + formation_code = row.UnitIdentifier.strip() + + # Validate depth values + try: + top_depth = float(row.StratTop) + bottom_depth = float(row.StratBottom) + except (ValueError, TypeError) as e: + logger.warning( + f"Invalid depth values for {pointid}: StratTop={row.StratTop}, " + f"StratBottom={row.StratBottom}, error: {e}" + ) + skipped_count += 1 + continue + + # Validate depth logic + if top_depth >= bottom_depth: + logger.warning( + f"Invalid depths for {pointid} layer {layer_index}: " + f"top={top_depth} >= bottom={bottom_depth}, skipping" + ) + skipped_count += 1 + continue + + if top_depth < 0: + logger.warning( + f"Negative top depth for {pointid} layer {layer_index}: {top_depth}, skipping" + ) + skipped_count += 1 + continue + + # 7. Get or create the formation + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if not formation: + # Create new formation if it doesn't exist + logger.info(f"Creating new geologic formation: {formation_code}") + formation = GeologicFormation( + formation_code=formation_code, + description=None, + lithology=None, # Will be set below + ) + session.add(formation) + session.flush() + + # 8. Update formation lithology if available and not already set + if hasattr(row, "Lithology") and row.Lithology: + try: + # Map lithology code to geologic_formation.lithology using ABBREVIATION field + lithology = lexicon_mapper.map_value( + f"LU_Lithology:{row.Lithology}" + ) + + # Update if formation does not have lithology yet + if not formation.lithology: + formation.lithology = lithology + lithology_updates += 1 + logger.info(f"Set lithology for {formation_code}: {lithology}") + elif formation.lithology != lithology: + # Log if there's a mismatch (different lithology for same formation) + logger.warning( + f"Formation {formation_code} has conflicting lithology: " + f"existing='{formation.lithology}', new='{lithology}'." + ) + except KeyError: + logger.warning( + f"Unknown lithology code '{row.Lithology}' for {pointid}, skipping lithology update" + ) + except Exception as e: + logger.warning(f"Error mapping lithology '{row.Lithology}': {e}") + + # 9. Create ThingGeologicFormationAssociation record + try: + formation_assoc = ThingGeologicFormationAssociation( + thing=thing, + geologic_formation=formation, + top_depth=top_depth, + bottom_depth=bottom_depth, + ) + session.add(formation_assoc) + created_count += 1 + + logger.info( + f" Layer {layer_index + 1}: {formation.formation_code} " + f"from {top_depth:.1f} to {bottom_depth:.1f} ft" + ) + + except Exception as e: + logger.critical( + f"Error creating stratigraphy association for {pointid}, " + f"formation {formation_code}: {e}" + ) + errors.append( + { + "pointid": pointid, + "formation": formation_code, + "layer": layer_index, + "error": str(e), + } + ) + skipped_count += 1 + continue + + # 10. Final commit + try: + session.commit() + logger.info( + f"Successfully transferred stratigraphy: " + f"{created_count} associations created, {skipped_count} skipped, " + f"{lithology_updates} lithology fields updated, {len(errors)} errors" + ) + except Exception as e: + logger.critical(f"Error in final commit: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/util.py b/transfers/util.py index 8dde3cd4d..4a753e719 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -421,6 +421,7 @@ def _make_lu_to_lexicon_mapper(self): "LU_Depth_CompletionSource", "LU_Discharge_ChemistrySource", "LU_LevelStatus", + "LU_Lithology", "LU_MajorAnalyte", "LU_MeasurementMethod", "LU_MinorTraceAnalyte", From e6af5e6aefe3452881fc078d6615a0b2caf66a35 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:42:50 -0700 Subject: [PATCH 076/119] fix: use OpenWellLoggerOK for is_suitable_for_datalogger field Laila indicated that this just describes if a datalogger can be physically installed at the well. It does not pertain to permissions --- schemas/thing.py | 1 + transfers/permissions_transfer.py | 34 ++++++++----------------------- transfers/well_transfer.py | 5 +++++ 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 977bf0c45..07e7e93ce 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -144,6 +144,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None + is_suitable_for_datalogger: bool | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index d07c2cbe2..1dcca2b8f 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -4,6 +4,14 @@ from db import Thing, PermissionHistory from transfers.util import read_csv, logger, replace_nans +""" +Developer's notes + +According to Laila the column WellData.OpenWellLoggerOK only pertains to the +physical properties of a well (that is, if a datalogger can be installed). It +does not pertain to permissions. +""" + def transfer_permissions(session: Session): """ @@ -76,30 +84,4 @@ def transfer_permissions(session: Session): session.rollback() pass - allow_datalogger_installation = wdf.loc[ - wdf["PointID"] == well.name, "OpenWellLoggerOK" - ].values - if ( - len(allow_datalogger_installation) > 0 - and allow_datalogger_installation is not None - ): - try: - permission_allowed = bool(allow_datalogger_installation[0]) - permission = PermissionHistory( - contact_id=contact_id, - permission_type="Datalogger Installation", - permission_allowed=permission_allowed, - start_date=datetime.today().date(), - target_id=well.id, - target_table="thing", - ) - session.add(permission) - logger.info( - f"Transferred Datalogger Installation permission for well {well.name}: {permission_allowed}." - ) - except Exception as e: - logger.error(f"Error transferring permission for well {well.name}: {e}") - session.rollback() - pass - session.commit() diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 346d91629..9e684cc8d 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -286,6 +286,11 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None else None ), well_pump_type=well_pump_type, + is_suitable_for_datalogger=( + bool(row.OpenWellLoggerOK) + if not isna(row.OpenWellLoggerOK) + else None + ), ) CreateWell.model_validate(data) From d3c8101cf156c7608752bbaa943881c5a8057755 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 17:00:14 -0700 Subject: [PATCH 077/119] feat: update CreateWell for transfers --- schemas/thing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/thing.py b/schemas/thing.py index 07e7e93ce..1baf324c3 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -144,7 +144,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None - is_suitable_for_datalogger: bool | None = None + is_suitable_for_datalogger: bool | None class CreateSpring(CreateBaseThing): From 2ea4cb16d9f8b896b23dc59eddc3ed916dedd434 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 07:57:26 -0700 Subject: [PATCH 078/119] fix: return aquifer/geology str instead of full response the well screen response includes the aquifer and geology information, but just as strings of names and codes instead of the full nested response objects. --- schemas/thing.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 1baf324c3..77df8c316 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,11 +33,6 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemGeoJSONResponse - -# from schemas.geologic_formation import ( -# GeologicFormationResponse, -# ) # -------- VALIDATE ---------- @@ -308,10 +303,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_system: str | None = None aquifer_type: str | None = None geologic_formation_id: int | None = None - # geologic_formation: GeologicFormationResponse | None = None + geologic_formation: str | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -319,6 +314,24 @@ class WellScreenResponse(BaseResponseModel): screen_type: str | None = None screen_description: str | None = None + @field_validator("aquifer_system", mode="before") + def populate_aquifer_system_with_name(cls, aquifer_system): + if aquifer_system is not None: + return aquifer_system.name + return None + + @field_validator("aquifer_type", mode="before") + def populate_aquifer_type_with_name(cls, aquifer_type): + if aquifer_type is not None: + return aquifer_type.name + return None + + @field_validator("geologic_formation_id", mode="before") + def populate_geologic_formation_with_code(cls, geologic_formation): + if geologic_formation is not None: + return geologic_formation.formation_code + return None + class GeoJSONGeometry(BaseModel): """ From 0994bec26f7198d638f1670f3f66f6fa982720b3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:16:31 -0700 Subject: [PATCH 079/119] refactor: clean up permission history testing data use a for loop for cleaner and more maintainable code --- tests/features/environment.py | 54 ++++++++++++----------------------- 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/tests/features/environment.py b/tests/features/environment.py index e2959ac85..6ba6caec3 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -517,42 +517,24 @@ def before_all(context): add_well_casing_material(context, session, well_1) contact = add_contact(context, session) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Datalogger Installation", - permission_allowed=True, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for datalogger installation.", - target_id=well_1.id, - target_table="thing", - ) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Water Level Sample", - permission_allowed=True, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for water level sampling.", - target_id=well_1.id, - target_table="thing", - ) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Water Chemistry Sample", - permission_allowed=False, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for chemistry sampling.", - target_id=well_1.id, - target_table="thing", - ) + + for permission in [ + "Datalogger Installation", + "Water Level Sample", + "Water Chemistry Sample", + ]: + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type=permission, + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes=f"Permission granted for {permission.lower()}.", + target_id=well_1.id, + target_table="thing", + ) measuring_point_history_1 = add_measuring_point_history( context, session, well=well_1 From 3041102e8b2c97d81b99971505cb2ae2736170ea Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:44:59 -0700 Subject: [PATCH 080/119] fix: use origin_type to retrieve well depth source --- tests/features/steps/well-core-information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index b0adc8346..1f56161f6 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -163,7 +163,7 @@ def step_impl(context): and r.target_table == "thing" and r.target_id == context.objects["wells"][0].id ] - well_depth_source = well_depth_source_records[0].origin_source + well_depth_source = well_depth_source_records[0].origin_type assert context.water_well_data["well_depth_source"] == well_depth_source From 5e8fdade7963a04308c7618cf039f9db51ed52c8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:46:21 -0700 Subject: [PATCH 081/119] refactor: use __tabename__ for polymorphic mixins this ensures fidelity in table naming across all database models --- db/data_provenance.py | 4 ++-- db/notes.py | 2 +- db/permission_history.py | 4 ++-- db/status_history.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db/data_provenance.py b/db/data_provenance.py index 14cfdc5aa..20505d94c 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -19,7 +19,7 @@ from sqlalchemy import Integer, Index, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin from db import lexicon_term @@ -120,7 +120,7 @@ def data_provenance(cls): "DataProvenance", primaryjoin=and_( cls.id == foreign(DataProvenance.target_id), - DataProvenance.target_table == pascal_to_snake(cls.__name__), + DataProvenance.target_table == cls.__tablename__, ), lazy="selectin", viewonly=True, diff --git a/db/notes.py b/db/notes.py index ab8384064..dfc99bbba 100644 --- a/db/notes.py +++ b/db/notes.py @@ -97,7 +97,7 @@ def notes(cls): "Notes", primaryjoin=and_( cls.id == foreign(Notes.target_id), - Notes.target_table == cls.__name__, + Notes.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", diff --git a/db/permission_history.py b/db/permission_history.py index 7c9c37159..591046bba 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -12,7 +12,7 @@ from sqlalchemy import Integer, ForeignKey, String, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: @@ -88,7 +88,7 @@ def permission_history(cls): primaryjoin=( and_( cls.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == pascal_to_snake(cls.__name__), + PermissionHistory.target_table == cls.__tablename__, ) ), lazy="selectin", diff --git a/db/status_history.py b/db/status_history.py index 8b3ee2321..15b5aec2f 100644 --- a/db/status_history.py +++ b/db/status_history.py @@ -19,7 +19,7 @@ ) from sqlalchemy.orm import Mapped, mapped_column, declared_attr, relationship, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term class StatusHistory(Base, AutoBaseMixin, ReleaseMixin): @@ -47,7 +47,7 @@ def status_history(cls): "StatusHistory", primaryjoin=and_( cls.id == foreign(StatusHistory.target_id), - StatusHistory.target_table == pascal_to_snake(cls.__name__), + StatusHistory.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", From 95ff8c0a63cc1395347a7b3284b98735e0a5e8bd Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 10:21:52 -0700 Subject: [PATCH 082/119] fix: use __tablename__ for NotesMixin add_note Now that __tablename__ is used throughout it should be used for the target_table in add_note --- db/notes.py | 2 +- tests/features/environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/notes.py b/db/notes.py index dfc99bbba..0e2e8ab8b 100644 --- a/db/notes.py +++ b/db/notes.py @@ -120,7 +120,7 @@ def add_note( content=content, note_type=note_type, target_id=self.id, - target_table=self.__class__.__name__, + target_table=self.__class__.__tablename__, release_status=release_status, ) diff --git a/tests/features/environment.py b/tests/features/environment.py index 6ba6caec3..5fca63bf0 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -495,7 +495,7 @@ def add_geologic_formation(context, session, formation_code, well): def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True if rebuild: erase_and_rebuild_db() From b14e6d05a7f166a6c57bd1745cb4dbe136c2ebda Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 10:41:54 -0700 Subject: [PATCH 083/119] refactor(transfer): update log levels and error tracking in stratigraphy transfer - Changed log level from 'warning' to 'critical' in the depth validation section - Added error tracking and clearer error messages. --- transfers/aquifer_system_transfer.py | 2 +- transfers/stratigraphy_transfer.py | 29 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py index 0de7cc7ae..7c1a42b79 100644 --- a/transfers/aquifer_system_transfer.py +++ b/transfers/aquifer_system_transfer.py @@ -47,7 +47,7 @@ def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: break # Log progress every 'step' rows - if i and not i % 25: + if i and not i % step: logger.info( f"Processing row {i} of {n}. Avg rows per second: {step / (time.time() - start_time):.2f}" ) diff --git a/transfers/stratigraphy_transfer.py b/transfers/stratigraphy_transfer.py index ac74a2d9c..dbbe3d0bb 100644 --- a/transfers/stratigraphy_transfer.py +++ b/transfers/stratigraphy_transfer.py @@ -137,25 +137,38 @@ def transfer_stratigraphy(session: Session, limit: int = None) -> tuple: top_depth = float(row.StratTop) bottom_depth = float(row.StratBottom) except (ValueError, TypeError) as e: - logger.warning( - f"Invalid depth values for {pointid}: StratTop={row.StratTop}, " - f"StratBottom={row.StratBottom}, error: {e}" + error_msg = f"Invalid depth values: StratTop={row.StratTop}, StratBottom={row.StratBottom}" + logger.critical( + f"{pointid} layer {layer_index}: {error_msg}, error: {e}" + ) + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": error_msg, + "details": str(e), # for conversion errors + } ) skipped_count += 1 continue # Validate depth logic if top_depth >= bottom_depth: - logger.warning( - f"Invalid depths for {pointid} layer {layer_index}: " - f"top={top_depth} >= bottom={bottom_depth}, skipping" + error_msg = ( + f"Invalid depth logic: top={top_depth} >= bottom={bottom_depth}" + ) + logger.critical(f"{pointid} layer {layer_index}: {error_msg}") + errors.append( + {"pointid": pointid, "layer": layer_index, "error": error_msg} ) skipped_count += 1 continue if top_depth < 0: - logger.warning( - f"Negative top depth for {pointid} layer {layer_index}: {top_depth}, skipping" + error_msg = f"Negative top depth: {top_depth}" + logger.critical(f"{pointid} layer {layer_index}: {error_msg}") + errors.append( + {"pointid": pointid, "layer": layer_index, "error": error_msg} ) skipped_count += 1 continue From 48830b92ecc743febb188cecc4fd8b4dd1f1c510 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 11:08:02 -0700 Subject: [PATCH 084/119] refactor(schemas): use enums in `geologic_formation` create schema. --- core/enums.py | 1 + schemas/geologic_formation.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/enums.py b/core/enums.py index 11fc2708e..a5c8be1de 100644 --- a/core/enums.py +++ b/core/enums.py @@ -78,4 +78,5 @@ AquiferType: type[Enum] = build_enum_from_lexicon_category("aquifer_type") GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale") Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") +FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code") # ============= EOF ============================================= diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 6cbc7357b..62e24ee8e 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -3,6 +3,7 @@ from pydantic import BaseModel from schemas import BaseResponseModel +from core.enums import FormationCode, Lithology # ------ CREATE ---------- @@ -12,9 +13,9 @@ class CreateGeologicFormation(BaseModel): Used during data transfer and API creation. """ - formation_code: str | None = None + formation_code: FormationCode | None = None description: str | None = None - lithology: str | None = None + lithology: Lithology | None = None boundary: str | None = None From dc9b1ef4d0acbf9750ab5aa0e4477a9463aee2ad Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 12:16:43 -0700 Subject: [PATCH 085/119] refactor(transfer): update log levels and error tracking in aquifer_system transfer - Changed log level from 'warning' to 'critical' in the depth validation section - Added error tracking --- transfers/aquifer_system_transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py index 7c1a42b79..137b3b00f 100644 --- a/transfers/aquifer_system_transfer.py +++ b/transfers/aquifer_system_transfer.py @@ -66,9 +66,9 @@ def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: aquifer_name = row.MEANING if not aquifer_name: - logger.warning( - f"Row {i} (code: {aquifer_code}) has no aquifer name (MEANING). Skipping." - ) + error_msg = f"Row {i} (code: {aquifer_code}) has no aquifer name (MEANING)." + logger.critical(error_msg) + errors.append({"row": i, "code": aquifer_code, "error": error_msg}) skipped_count += 1 continue From ee787d15288b60dff3d2a0cd2269032f15e20748 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 14:39:50 -0700 Subject: [PATCH 086/119] feat(transfer): Add aquifer system and geologic formation associations to well transfer - Implement _extract_aquifer_type_codes() to parse compound codes (e.g., "FC" -> Fractured + Confined) - Add get_or_create_aquifer() helper to manage unique aquifer system records - Add get_or_create_formation() helper to manage geologic formation records - Integrate aquifer association logic in `transfer_wells()` to create ThingAquiferAssociation and AquiferType records - Integrate formation association logic to create ThingGeologicFormationAssociation records with depth data - Support lexicon mapping for both AqClass (aquifer name) and AquiferType (characteristics) fields - Add comprehensive error handling and logging for aquifer/formation associations This enables proper tracking of wells' aquifer systems with multiple type characteristics and their associated geologic formations, preserving all source data from NM_Aquifer. --- transfers/well_transfer.py | 231 +++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ee54d0216..e713ef3e5 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -36,6 +36,11 @@ StatusHistory, MonitoringFrequencyHistory, MeasuringPointHistory, + AquiferSystem, + AquiferType, + GeologicFormation, + ThingAquiferAssociation, + ThingGeologicFormationAssociation, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -117,6 +122,107 @@ def _extract_casing_materials(row) -> list[str]: return materials +# Parse aquifer codes +def _extract_aquifer_type_codes(aquifer_code: str) -> list[str]: + """ + Parse aquifer type codes that may contain multiple values. + + Args: + aquifer_code: Raw code from AquiferType field + + Returns: + List of individual codes + """ + if not aquifer_code: + return [] + # clean the code + code = aquifer_code.strip().upper() + # split into individual characters. This handles cases like "FC" -> ["F", "C"] + individual_codes = list(code) + return individual_codes + + +# Get or create aquifer system +def get_or_create_aquifer_system( + session: Session, aquifer_name: str, primary_type: str +) -> AquiferSystem | None: + """ + Get existing aquifer or create new one if it doesn't exist. + + With the new AquiferType model, we create ONE aquifer record per named + aquifer (e.g., one "Santa Fe Group"), not multiple variants. + + Args: + session: Database session + aquifer_name: Name of the aquifer (from AqClass or type name) + primary_type: Primary aquifer type for the aquifer_type field + """ + # Try to find existing aquifer by name + aquifer = ( + session.query(AquiferSystem).filter(AquiferSystem.name == aquifer_name).first() + ) + + if aquifer: + return aquifer + + # Create new aquifer + try: + logger.info( + f"Creating new aquifer system: {aquifer_name} (primary type: {primary_type})" + ) + + aquifer = AquiferSystem( + name=aquifer_name, + aquifer_type=primary_type, # Primary type + geographic_scale=None, # Default + ) + session.add(aquifer) + session.flush() # Get the ID + return aquifer + except Exception as e: + logger.critical(f"Error creating aquifer {aquifer_name}: {e}") + return None + + +def get_or_create_geologic_formation( + session: Session, formation_code: str +) -> GeologicFormation | None: + """ + Get existing geologic formation or create new one if it doesn't exist. + + Args: + session: Database session + formation_code: The formation code from FormationZone field + + Returns: + GeologicFormation object or None if creation fails + """ + # Try to find existing formation + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if formation: + return formation + + # If not found, create new formation + try: + logger.info(f"Creating new geologic formation: {formation_code}") + formation = GeologicFormation( + formation_code=formation_code, + description=None, + lithology=None, + ) + session.add(formation) + session.flush() + return formation + except Exception as e: + logger.critical(f"Error creating formation {formation_code}: {e}") + return None + + def get_wells_to_transfer( sess: Session, flags: dict = None ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -330,6 +436,131 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None assoc.thing = well session.add(assoc) + # --- Create Aquifer Association with AquiferType records --- + if hasattr(row, "AquiferType") and not isna(row.AquiferType): + try: + # Parse codes (handles multi-character codes like "FC") + aquifer_codes = _extract_aquifer_type_codes(row.AquiferType) + + if not aquifer_codes: + logger.warning( + f"Well {row.PointID}: Empty aquifer codes after parsing '{row.AquiferType}'" + ) + else: + # Map AqClass code to aquifer name using lexicon mapper + if hasattr(row, "AqClass") and not isna(row.AqClass): + try: + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferClass:{row.AqClass}" + ) + except KeyError: + logger.warning( + f"Unknown AqClass code '{row.AqClass}' for well {row.PointID}, using first type as name" + ) + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + else: + # No AqClass - use first code's mapped name as aquifer name + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + + # Determine primary type + try: + primary_type = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + except KeyError: + logger.warning( + f"Unknown aquifer type code '{aquifer_codes[0]}' for well {row.PointID}" + ) + primary_type = None + + if primary_type: + # Get or create the aquifer + aquifer = get_or_create_aquifer_system( + session, aquifer_name, primary_type + ) + + if aquifer: + # Check if association already exists + existing_assoc = ( + session.query(ThingAquiferAssociation) + .filter( + ThingAquiferAssociation.thing_id == well.id, + ThingAquiferAssociation.aquifer_system_id + == aquifer.id, + ) + .first() + ) + + if not existing_assoc: + # Create the association + aquifer_assoc = ThingAquiferAssociation( + thing=well, aquifer_system=aquifer + ) + session.add(aquifer_assoc) + session.flush() + + # Create AquiferType records for EACH characteristic + aquifer_type_names = [] + for aquifer_code in aquifer_codes: + try: + type_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_code}" + ) + aquifer_type = AquiferType( + thing_aquifer_association=aquifer_assoc, + aquifer_type=type_name, + ) + session.add(aquifer_type) + aquifer_type_names.append(type_name) + except KeyError: + logger.warning( + f"Unknown aquifer code '{aquifer_code}' from AquiferType='{row.AquiferType}' " + f"for well {well.name}. Skipping this code." + ) + + logger.info( + f"Associated well {well.name} with aquifer {aquifer.name} " + f"(types: {', '.join(aquifer_type_names)})" + ) + + except Exception as e: + logger.critical( + f"Error creating aquifer associations for {well.name}: {e}" + ) + + # --- Create Formation Association (if FormationZone exists) --- + if hasattr(row, "FormationZone") and not isna(row.FormationZone): + try: + formation_code = row.FormationZone + formation = get_or_create_geologic_formation(session, formation_code) + + if formation: + top_depth = 0.0 + bottom_depth = ( + row.WellDepth + if row.WellDepth and not isna(row.WellDepth) + else 100.0 + ) + + formation_assoc = ThingGeologicFormationAssociation( + thing=well, + geologic_formation=formation, + top_depth=top_depth, + bottom_depth=bottom_depth, + ) + session.add(formation_assoc) + logger.info( + f"Associated well {well.name} with formation {formation.formation_code} (0-{bottom_depth} ft)" + ) + except Exception as e: + logger.critical( + f"Error creating formation association for {well.name}: {e}" + ) + session.commit() # add things thate need well id From 81fed8a0329bd29758046a506dcabd8786f46e28 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 20:42:17 -0700 Subject: [PATCH 087/119] refactor(transfer): Update logic related to creating an aquifer_system - Set primary_type placeholder to "Unknown" instead of None when creating an aquifer_system in `well_transfer.py` --- transfers/well_transfer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index e713ef3e5..6c7d4f6e2 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -473,9 +473,10 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ) except KeyError: logger.warning( - f"Unknown aquifer type code '{aquifer_codes[0]}' for well {row.PointID}" + f"Unknown aquifer type code '{aquifer_codes[0]}' for well {row.PointID}." + f"Setting primary_type to 'Unknown'" ) - primary_type = None + primary_type = "Unknown" # Creates aquifer with placeholder if primary_type: # Get or create the aquifer From cd1d170fcd4a0440f931b5f7b572f9c466f363ea Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 20:56:27 -0700 Subject: [PATCH 088/119] refactor(transfer): Update logic related to creating formation associations - Updated the logic so that a formation association is created only if valid well depth data exist. --- transfers/well_transfer.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 6c7d4f6e2..4f54b64dc 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -534,18 +534,28 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ) # --- Create Formation Association (if FormationZone exists) --- + # Note: This creates a single formation association from WellData. + # For detailed stratigraphy with depth intervals, see transfer_stratigraphy() if hasattr(row, "FormationZone") and not isna(row.FormationZone): try: formation_code = row.FormationZone formation = get_or_create_geologic_formation(session, formation_code) if formation: - top_depth = 0.0 - bottom_depth = ( - row.WellDepth - if row.WellDepth and not isna(row.WellDepth) - else 100.0 - ) + # Onlyl create association if valid well depth data exists + if ( + not hasattr(row, "WellDepth") + or isna(row.WellDepth) + or not row.WellDepth + ): + logger.warning( + f"Well {well.name} has FormationZone but no valid WellDepth. " + f"Skipping formation association. Use stratigraphy transfer for detailed depth data." + ) + else: + # Create association using actual well depth + top_depth = 0.0 + bottom_depth = float(row.WellDepth) formation_assoc = ThingGeologicFormationAssociation( thing=well, From e7f0aaa0e3411a64af74b2b2c741e68239385271 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 25 Nov 2025 21:06:17 -0700 Subject: [PATCH 089/119] refactor(transfer): add note to verify compound aquifer type codes with AMMP It is assumed that the first recorded type of a compound type is the primary type of the aquifer. --- transfers/well_transfer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 4f54b64dc..b4b206073 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -467,6 +467,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ) # Determine primary type + # This assumes the first recorded type of a compound type is the primary type of the aquifer. + # TODO: verify with AMMP try: primary_type = lexicon_mapper.map_value( f"LU_AquiferType:{aquifer_codes[0]}" From bdff12edaa20f743f8e5160add8473b162fe929d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 08:33:58 -0700 Subject: [PATCH 090/119] refactor: use sets in feature tests for comparison --- .../steps/well-additional-information.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 638c6529f..0b584d45c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -132,9 +132,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data - assert sorted(context.water_well_data["well_casing_materials"]) == sorted( - [m.material for m in context.objects["wells"][0].well_casing_materials] - ) + assert set(context.water_well_data["well_casing_materials"]) == { + m.material for m in context.objects["wells"][0].well_casing_materials + } @then("the response should include the well pump type (previously well_type field)") @@ -190,9 +190,9 @@ def step_impl(context): def step_impl(context): for aquifer in context.water_well_data["aquifers"]: assert "aquifer_system" in aquifer - assert sorted( - [a.get("aquifer_system") for a in context.water_well_data["aquifers"]] - ) == sorted([system.name for system in context.objects["aquifer_systems"]]) + assert {a.get("aquifer_system") for a in context.water_well_data["aquifers"]} == { + system.name for system in context.objects["aquifer_systems"] + } @then( @@ -203,8 +203,8 @@ def step_impl(context): assert "aquifer_types" in aquifer if aquifer["aquifer_system"] == "Aquifer A": - assert sorted(aquifer["aquifer_types"]) == sorted( - [a.aquifer_type for a in context.objects["aquifer_types"]] - ) + assert set(aquifer["aquifer_types"]) == { + a.aquifer_type for a in context.objects["aquifer_types"] + } else: assert aquifer["aquifer_types"] == [] From 19d736aba6d566e3946c24a8cf1cfb3ee59b4b82 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 09:55:21 -0700 Subject: [PATCH 091/119] refactor: return permission history records for a well instead of a specific permission type, return a list of permission records for all permission types associated with the well. --- db/thing.py | 42 +++------- schemas/permission_history.py | 18 +++++ schemas/thing.py | 67 +++++++++++++--- .../steps/well-additional-information.py | 77 ++++++++++++++++--- 4 files changed, 156 insertions(+), 48 deletions(-) create mode 100644 schemas/permission_history.py diff --git a/db/thing.py b/db/thing.py index b08a14747..cae9363e0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -446,36 +446,6 @@ def well_construction_method_source(self) -> str | None: "well_construction_method", "origin_source" ) - @property - def allow_water_level_samples(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Water Level Sample" - ) - return permission_record.permission_allowed if permission_record else None - - @property - def allow_water_chemistry_samples(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Water Chemistry Sample" - ) - return permission_record.permission_allowed if permission_record else None - - @property - def allow_datalogger_installation(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Datalogger Installation" - ) - return permission_record.permission_allowed if permission_record else None - @property def aquifers(self) -> List[dict]: """ @@ -493,6 +463,18 @@ def aquifers(self) -> List[dict]: aquifer_list.append(aquifer_info) return aquifer_list + @property + def permissions(self) -> list: + """ + Returns the associated permissions or an empty list. If there are no + associated permissions, an empty list is returned instead of None to + allow the API to serialize correctly (see schemas/thing.py). + """ + if self.permission_history: + return self.permission_history + else: + return [] + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/permission_history.py b/schemas/permission_history.py new file mode 100644 index 000000000..e0619d90e --- /dev/null +++ b/schemas/permission_history.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel +from schemas import PastOrTodayDate + +from core.enums import PermissionType + + +# ------ RESPONSE ---------- +class PermissionHistoryResponse(BaseModel): + """ + Even though permission_allowed and start_date are not-nullable in the + database, they are nullable here to accommodate cases where no permission + record exists for a given permission type. + """ + + permission_type: PermissionType + permission_allowed: bool | None + start_date: PastOrTodayDate | None + end_date: PastOrTodayDate | None diff --git a/schemas/thing.py b/schemas/thing.py index bd7665f92..2b8967553 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,11 +33,7 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemGeoJSONResponse - -# from schemas.geologic_formation import ( -# GeologicFormationResponse, -# ) +from schemas.permission_history import PermissionHistoryResponse # -------- VALIDATE ---------- @@ -238,9 +234,6 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" - allow_water_level_samples: bool | None - allow_water_chemistry_samples: bool | None - allow_datalogger_installation: bool | None is_suitable_for_datalogger: bool | None well_status: str | None measuring_point_height: float @@ -251,6 +244,7 @@ class WellResponse(BaseThingResponse): water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None + permissions: list[PermissionHistoryResponse] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -279,6 +273,43 @@ def populate_geologic_formations_with_strings(cls, geologic_formations): formations = [] return formations + @field_validator("permissions", mode="before") + def populate_permission_history_with_latest_records(cls, permissions): + """ + Populate the permission history with the latest records for each + type of permission. If multiple records exist for the same permission type + only the most recent one is included. If there are no records + the permission_allowed will be None + """ + permissions_to_return = [] + for permission_type in [ + "Water Level Sample", + "Water Chemistry Sample", + "Datalogger Installation", + ]: + # Filter records for the current permission type + filtered_records = [ + record + for record in permissions + if record.permission_type == permission_type and record.end_date is None + ] + if filtered_records: + # Get the most recent record based on start_date + latest_record = max( + filtered_records, key=lambda record: record.start_date + ) + permissions_to_return.append(latest_record) + else: + permissions_to_return.append( + PermissionHistoryResponse( + permission_type=permission_type, + permission_allowed=None, + start_date=None, + end_date=None, + ) + ) + return permissions_to_return + class SpringResponse(BaseThingResponse): """ @@ -301,7 +332,7 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_system: str | None = None aquifer_type: str | None = None geologic_formation_id: int | None = None # geologic_formation: GeologicFormationResponse | None = None @@ -312,6 +343,24 @@ class WellScreenResponse(BaseResponseModel): screen_type: str | None = None screen_description: str | None = None + @field_validator("aquifer_system", mode="before") + def populate_aquifer_system_with_name(cls, aquifer_system): + if aquifer_system is not None: + return aquifer_system.name + return None + + @field_validator("aquifer_type", mode="before") + def populate_aquifer_type_with_name(cls, aquifer_type): + if aquifer_type is not None: + return aquifer_type.name + return None + + @field_validator("geologic_formation_id", mode="before") + def populate_geologic_formation_with_code(cls, geologic_formation): + if geologic_formation is not None: + return geologic_formation.formation_code + return None + class GeoJSONGeometry(BaseModel): """ diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 0b584d45c..5658f2ccc 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -10,44 +10,103 @@ "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - assert "allow_water_level_samples" in context.water_well_data + permission_type = "Water Level Sample" + assert "permissions" in context.water_well_data + permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Water Level Sample" + context.objects["wells"][0], "permission_history", permission_type + ) + + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type ) assert ( - context.water_well_data["allow_water_level_samples"] + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - assert "allow_water_chemistry_samples" in context.water_well_data + permission_type = "Water Chemistry Sample" + assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Water Chemistry Sample" + context.objects["wells"][0], "permission_history", permission_type ) + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] assert ( - context.water_well_data["allow_water_chemistry_samples"] + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) + assert ( + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None @then( "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - assert "allow_datalogger_installation" in context.water_well_data + permission_type = "Datalogger Installation" + assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Datalogger Installation" + context.objects["wells"][0], "permission_history", permission_type ) + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) assert ( - context.water_well_data["allow_datalogger_installation"] + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None # ------------------------------------------------------------------------------ From 595d1d949a0e7a7db16eef5c79d7991043f2e13b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 26 Nov 2025 11:54:53 -0700 Subject: [PATCH 092/119] refactor(transfer): removed unnecessary if statement re: creating aquifers --- transfers/well_transfer.py | 90 +++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index b4b206073..036f84b80 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -480,55 +480,53 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ) primary_type = "Unknown" # Creates aquifer with placeholder - if primary_type: - # Get or create the aquifer - aquifer = get_or_create_aquifer_system( - session, aquifer_name, primary_type - ) + # Get or create the aquifer + aquifer = get_or_create_aquifer_system( + session, aquifer_name, primary_type + ) - if aquifer: - # Check if association already exists - existing_assoc = ( - session.query(ThingAquiferAssociation) - .filter( - ThingAquiferAssociation.thing_id == well.id, - ThingAquiferAssociation.aquifer_system_id - == aquifer.id, - ) - .first() + if aquifer: + # Check if association already exists + existing_assoc = ( + session.query(ThingAquiferAssociation) + .filter( + ThingAquiferAssociation.thing_id == well.id, + ThingAquiferAssociation.aquifer_system_id == aquifer.id, ) + .first() + ) - if not existing_assoc: - # Create the association - aquifer_assoc = ThingAquiferAssociation( - thing=well, aquifer_system=aquifer - ) - session.add(aquifer_assoc) - session.flush() - - # Create AquiferType records for EACH characteristic - aquifer_type_names = [] - for aquifer_code in aquifer_codes: - try: - type_name = lexicon_mapper.map_value( - f"LU_AquiferType:{aquifer_code}" - ) - aquifer_type = AquiferType( - thing_aquifer_association=aquifer_assoc, - aquifer_type=type_name, - ) - session.add(aquifer_type) - aquifer_type_names.append(type_name) - except KeyError: - logger.warning( - f"Unknown aquifer code '{aquifer_code}' from AquiferType='{row.AquiferType}' " - f"for well {well.name}. Skipping this code." - ) - - logger.info( - f"Associated well {well.name} with aquifer {aquifer.name} " - f"(types: {', '.join(aquifer_type_names)})" - ) + if not existing_assoc: + # Create the association + aquifer_assoc = ThingAquiferAssociation( + thing=well, aquifer_system=aquifer + ) + session.add(aquifer_assoc) + session.flush() + + # Create AquiferType records for EACH characteristic + aquifer_type_names = [] + for aquifer_code in aquifer_codes: + try: + type_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_code}" + ) + aquifer_type = AquiferType( + thing_aquifer_association=aquifer_assoc, + aquifer_type=type_name, + ) + session.add(aquifer_type) + aquifer_type_names.append(type_name) + except KeyError: + logger.warning( + f"Unknown aquifer code '{aquifer_code}' from AquiferType='{row.AquiferType}' " + f"for well {well.name}. Skipping this code." + ) + + logger.info( + f"Associated well {well.name} with aquifer {aquifer.name} " + f"(types: {', '.join(aquifer_type_names)})" + ) except Exception as e: logger.critical( From 656e159d4179a1d5ae826d5bd25266491166d6ce Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 13:27:45 -0700 Subject: [PATCH 093/119] fix: use regex to check for pump type in well construction notes regex is a more robust way to check for the presence of pump types in construction notes, handling variations in spacing and punctuation. --- transfers/well_transfer.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 9e684cc8d..22209741a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -16,7 +16,7 @@ import json import time from datetime import datetime, UTC - +import re import pandas as pd from pandas import isna from pydantic import ValidationError @@ -118,19 +118,26 @@ def _extract_casing_materials(row) -> list[str]: return materials +pattern = re.compile( + r"\b(?Pjet|hand|submersible)\b|\b(?Pline[-\s]+shaft)\b", re.IGNORECASE +) + + +def first_matched_term(text: str): + m = pattern.search(text) + if not m: + return None + return m.group("term") or m.group("phrase") + + +PUMP_MAPPING = {"jet": "Jet", "hand": "Hand", "submersible": "Submersible"} + + def _extract_well_pump_type(row) -> str | None: + if isna(row.ConstructionNotes): + return None construction_notes = row.ConstructionNotes.lower() - if "pump" in construction_notes: - if "submersible" in construction_notes: - return "Submersible" - elif "jet" in construction_notes: - return "Jet" - elif "line shaft" in construction_notes or "lineshaft" in construction_notes: - return "Line Shaft" - elif "hand" in construction_notes: - return "Hand" - else: - return None + return PUMP_MAPPING.get(first_matched_term(construction_notes), None) def get_wells_to_transfer( @@ -252,9 +259,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) - well_pump_type = ( - _extract_well_pump_type(row) if row.ConstructionNotes else None - ) + well_pump_type = _extract_well_pump_type(row) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation From 183d275525c5c323d90f46fbb6cf828d7136bb4b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 14:20:27 -0700 Subject: [PATCH 094/119] fix: ensure permissions are not null during transfer check to make sure that permissions are set before transferring them from one entity to another to avoid potential errors. --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 1dcca2b8f..fa9ecf8e7 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -1,5 +1,6 @@ from sqlalchemy.orm import Session from datetime import datetime +from pandas import isna from db import Thing, PermissionHistory from transfers.util import read_csv, logger, replace_nans @@ -15,8 +16,10 @@ def transfer_permissions(session: Session): """ - The transferred wells and contacts need to be queried to know who gave - permission to which well since contact_id is required for PermissionHistory + The transferred wells and contacts need to be transferred first + - to access the auto-generated well IDs + - to know who gave permission to which well since contact_id is required for + PermissionHistory """ wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) wdf = replace_nans(wdf) @@ -38,7 +41,9 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + if len(allow_water_level_samples) > 0 and not isna( + allow_water_level_samples[0] + ): try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -61,9 +66,8 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if ( - len(allow_water_chemistry_samples) > 0 - and allow_water_chemistry_samples is not None + if len(allow_water_chemistry_samples) > 0 and not isna( + allow_water_chemistry_samples[0] ): try: permission_allowed = bool(allow_water_chemistry_samples[0]) From 2d0f2b6177980836ca7ba895ea7df72a3f13f5eb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 15:10:15 -0700 Subject: [PATCH 095/119] refactor: check for existence of permissions before converting --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index fa9ecf8e7..18daa1040 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -41,9 +41,11 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and not isna( - allow_water_level_samples[0] - ): + if len(allow_water_level_samples) == 0: + pass + elif isna(allow_water_level_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -66,9 +68,11 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if len(allow_water_chemistry_samples) > 0 and not isna( - allow_water_chemistry_samples[0] - ): + if len(allow_water_chemistry_samples) == 0: + pass + elif isna(allow_water_chemistry_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_chemistry_samples[0]) permission = PermissionHistory( From 9a2cdbc1d9c02f74495ffa43d50440d691248e95 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 09:38:02 -0700 Subject: [PATCH 096/119] refactor(transfer): remove section that creates Formation Associations in `well_transfer.py` The current implementation for creating a Formation Association was problematic for the following reasons: 1. The FormationZone field from the WellData csv indicates completion zone, not the entire well stratigraphy 2. Setting top_depth = 0.0 incorrectly implies: * The formation starts at ground surface * The well only penetrates one formation * The entire well depth is within that single formation 3. ThingGeologicFormationAssociation = currently implies full stratigraphic column with depth intervals 4. Forcing FormationZone into a depth-based association creates misleading data Implementation - Added `formation_completion_code` Field to Thing Model - This provides a clear separation between `formation_completion_code` = "What formation is the well completed in" and `formation_associations` = "What formations does the borehole pass through?" - Updated well_transfer.py so that `ThingGeologicFormationAssociation` records are only being created from the Stratigraphy.csv (they were previously being created from WellData, too). --- db/thing.py | 6 ++++++ transfers/well_transfer.py | 44 +++++++++----------------------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/db/thing.py b/db/thing.py index cae9363e0..87c706a35 100644 --- a/db/thing.py +++ b/db/thing.py @@ -133,6 +133,12 @@ class Thing( info={"unit": "feet below ground surface"}, comment="Depth of the well pump from ground surface to the pump intake (in feet).", ) + formation_completion_code: Mapped[str] = lexicon_term( + nullable=True, + comment="The geologic formation in which the well was completed (from WellData.FormationZone). " + "This indicates the target formation for the well, not the full stratigraphic column. " + "For detailed depth-interval stratigraphy, see formation_associations.", + ) # TODO: should this be required for every well in the database? AMMP review is_suitable_for_datalogger: Mapped[bool] = mapped_column( nullable=True, diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 80e6a3ca6..5ccc3dde6 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -41,7 +41,6 @@ AquiferType, GeologicFormation, ThingAquiferAssociation, - ThingGeologicFormationAssociation, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -558,43 +557,20 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None f"Error creating aquifer associations for {well.name}: {e}" ) - # --- Create Formation Association (if FormationZone exists) --- - # Note: This creates a single formation association from WellData. - # For detailed stratigraphy with depth intervals, see transfer_stratigraphy() + # --- Set Formation Completion (NOT depth-based stratigraphy) --- + # This simply records which formation the well was completed in. + # For detailed depth-interval stratigraphy, see stratigraphy_transfer.py if hasattr(row, "FormationZone") and not isna(row.FormationZone): try: - formation_code = row.FormationZone - formation = get_or_create_geologic_formation(session, formation_code) - - if formation: - # Onlyl create association if valid well depth data exists - if ( - not hasattr(row, "WellDepth") - or isna(row.WellDepth) - or not row.WellDepth - ): - logger.warning( - f"Well {well.name} has FormationZone but no valid WellDepth. " - f"Skipping formation association. Use stratigraphy transfer for detailed depth data." - ) - else: - # Create association using actual well depth - top_depth = 0.0 - bottom_depth = float(row.WellDepth) - - formation_assoc = ThingGeologicFormationAssociation( - thing=well, - geologic_formation=formation, - top_depth=top_depth, - bottom_depth=bottom_depth, - ) - session.add(formation_assoc) - logger.info( - f"Associated well {well.name} with formation {formation.formation_code} (0-{bottom_depth} ft)" - ) + formation_code = row.FormationZone.strip() + # Set the formation_completion_code field directly on the well + well.formation_completion_code = formation_code + logger.info( + f"Set formation_completion_code for {well.name}: {formation_code}" + ) except Exception as e: logger.critical( - f"Error creating formation association for {well.name}: {e}" + f"Error setting formation completion for {well.name}: {e}" ) session.commit() From f6dec153fb6ef88bcc90a98231c5b1c991690520 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 10:38:51 -0700 Subject: [PATCH 097/119] refactor(transfer): add validation when assigning `formation_completion_code` to a well. --- transfers/well_transfer.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 5ccc3dde6..bb3cb7419 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -562,15 +562,36 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None # For detailed depth-interval stratigraphy, see stratigraphy_transfer.py if hasattr(row, "FormationZone") and not isna(row.FormationZone): try: - formation_code = row.FormationZone.strip() - # Set the formation_completion_code field directly on the well - well.formation_completion_code = formation_code - logger.info( - f"Set formation_completion_code for {well.name}: {formation_code}" + formation_code = row.FormationZone + + # Validate formation exists + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() ) + + if formation: + # Formation exists: Set association + well.formation_completion_code = formation_code + logger.info( + f"Set completion formation for {well.name}: {formation_code}" + ) + else: + # Formation does NOT exist: Do not create new formation. Flag and log for review + logger.warning( + f"MISSING FORMATION: Formation '{formation_code}' not found for well {well.name}. Flagged for review." + ) + errors.append( + { + "well": well.name, + "error": f"Unknown formation: {formation_code}", + } + ) + except Exception as e: logger.critical( - f"Error setting formation completion for {well.name}: {e}" + f"Error setting completion formation for {well.name}: {e}" ) session.commit() From 64ecb1831feda41dea78865a0fa48366b3449a2a Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 10:46:18 -0700 Subject: [PATCH 098/119] refactor(schema): add `formation_completion_code` field to CreateWell schema. --- schemas/thing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schemas/thing.py b/schemas/thing.py index 28b056c82..e7af7995e 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -141,6 +141,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None is_suitable_for_datalogger: bool | None + formation_completion_code: str | None = None class CreateSpring(CreateBaseThing): From 9a87a69d52a296018bb515cbbe3b99d305be69e2 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 11:22:35 -0700 Subject: [PATCH 099/119] refactor(schema): use `FormationCode` enum for `formation_completion_code` --- schemas/thing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/thing.py b/schemas/thing.py index e7af7995e..2ee35ad6c 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -28,6 +28,7 @@ MonitoringFrequency, WellConstructionMethod, WellPumpType, + FormationCode, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate from schemas.group import GroupResponse @@ -141,7 +142,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None is_suitable_for_datalogger: bool | None - formation_completion_code: str | None = None + formation_completion_code: FormationCode | None = None class CreateSpring(CreateBaseThing): From 9588627ed96f0132d51897d536777823043bef16 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 11:34:31 -0700 Subject: [PATCH 100/119] refactor(transfer): add missing transfers to `transfer.py` Added aquifer systems, geologic formations, and stratigraphy transfers. --- transfers/transfer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/transfers/transfer.py b/transfers/transfer.py index 2b576a4b2..29d76cec7 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -31,11 +31,14 @@ from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata from transfers.contact_transfer import transfer_contacts from transfers.sensor_transfer import transfer_sensors +from transfers.aquifer_system_transfer import transfer_aquifer_systems +from transfers.geologic_formation_transfer import transfer_geologic_formations from transfers.waterlevels_transfer import transfer_water_levels from transfers.well_transfer import ( transfer_wells, transfer_wellscreens, ) +from transfers.stratigraphy_transfer import transfer_stratigraphy from transfers.permissions_transfer import transfer_permissions from transfers.asset_transfer import transfer_assets @@ -58,6 +61,14 @@ def transfer_all(sess, limit=100): erase_and_rebuild_db() metrics = Metrics() + + # transfer aquifer systems and geologic formations first as well_transfer depend on them + message("TRANSFERRING AQUIFER SYSTEMS") + timeit_direct(transfer_aquifer_systems, sess) + + message("TRANSFERRING GEOLOGIC FORMATIONS") + timeit_direct(transfer_geologic_formations, sess) + message("TRANSFERRING WELLS") flags = { @@ -72,6 +83,9 @@ def transfer_all(sess, limit=100): results = timeit_direct(transfer_wellscreens, sess) metrics.well_screen_metrics(sess, *results) + message("TRANSFERRING STRATIGRAPHY") + timeit_direct(transfer_stratigraphy, sess) + message("TRANSFERRING SENSORS") results = timeit_direct(transfer_sensors, sess) metrics.sensor_metrics(sess, *results) From 8a78f6759ccbc8d6a4ef64ea8698f599c9e4510e Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 1 Dec 2025 13:03:18 -0700 Subject: [PATCH 101/119] feat(schemas): add enum, depth and geometry validations and association create model - Add `schemas/validators.py` with `DepthIntervalMixin` for logical depth checks, `GeometryMixin` for WKT validation, and `validate_enum_input` for validation. - Add `CreateThingGeologicFormationAssociation` to `schemas_geologic_formation.py` to support creation of formation picks/stratigraphy. - Update `CreateGeologicFormation` to enforce uppercase formation codes and validate WKT boundaries. - Update `CreateAquiferSystem` to validate WKT boundaries. --- schemas/aquifer_system.py | 18 +++++++-- schemas/geologic_formation.py | 35 ++++++++++++++-- schemas/validators.py | 76 +++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 schemas/validators.py diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 6c077aa90..3b0a58470 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -1,12 +1,14 @@ from typing import List -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from schemas import BaseResponseModel +from schemas.validators import GeometryMixin, validate_enum_input +from core.enums import AquiferType, GeographicScale # Import specific Enums # ------ CREATE ---------- -class CreateAquiferSystem(BaseModel): +class CreateAquiferSystem(BaseModel, GeometryMixin): """ Schema for creating an aquifer system. Used during data transfer and API creation. @@ -16,7 +18,17 @@ class CreateAquiferSystem(BaseModel): description: str | None = None primary_aquifer_type: str geographic_scale: str - boundary: str | None = None + # boundary field inherited from GeometryMixin + + @field_validator("primary_aquifer_type", mode="before") + @classmethod + def check_aquifer_type(cls, v): + return validate_enum_input(v, AquiferType) + + @field_validator("geographic_scale", mode="before") + @classmethod + def check_geographic_scale(cls, v): + return validate_enum_input(v, GeographicScale) # ------ RESPONSE ---------- diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 62e24ee8e..fbdaabb02 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -1,22 +1,51 @@ from typing import List -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from schemas import BaseResponseModel +from schemas.validators import DepthIntervalMixin, GeometryMixin, validate_enum_input from core.enums import FormationCode, Lithology # ------ CREATE ---------- -class CreateGeologicFormation(BaseModel): +class CreateGeologicFormation(BaseModel, GeometryMixin): """ Schema for creating a geologic formation. Used during data transfer and API creation. """ + # formation_code has its own custom uppercase validator formation_code: FormationCode | None = None description: str | None = None lithology: Lithology | None = None - boundary: str | None = None + # boundary: inherited from GeometryMixin + + @field_validator("formation_code", mode="before") + @classmethod + def upper_case_code(cls, v: str | None) -> str | None: + """ + Automatically uppercase the formation code. + """ + if isinstance(v, str): + return v.upper() + return v + + @field_validator("lithology", mode="before") + @classmethod + def check_lithology(cls, v): + return validate_enum_input(v, Lithology) + + +class CreateThingGeologicFormationAssociation(BaseModel, DepthIntervalMixin): + """ + Schema for linking a Thing (Well) to a GeologicFormation. + Uses DepthIntervalMixin to enforce bottom_depth > top_depth. + """ + + thing_id: int + geologic_formation_id: int | None = None + top_depth: float + bottom_depth: float # ------ RESPONSE ---------- diff --git a/schemas/validators.py b/schemas/validators.py new file mode 100644 index 000000000..77e3f12dd --- /dev/null +++ b/schemas/validators.py @@ -0,0 +1,76 @@ +""" +schemas/validators.py +Reusable Pydantic validators and mixins for aquifer and geology related schemas. +May consider expansion for other domain models in the future. +""" + +from typing import Any, Type +from pydantic import model_validator, field_validator, BaseModel, ValueError + +from enum import Enum + + +def validate_enum_input(v: Any, enum_cls: Type[Enum]) -> Any: + """ + Validates that the input matches an enum value, either exactly or case-insensitively. + Returns the actual Enum member value. + """ + if v is None: + return None + + # 1. Check if it's already a valid enum member or value + try: + return enum_cls(v).value + except ValueError: + pass + + # 2. Case-insensitive fallback (for string inputs) + if isinstance(v, str): + v_lower = v.lower() + for member in enum_cls: + if str(member.value).lower() == v_lower: + return member.value + + # 3. Fail if no match found + valid_options = [str(e.value) for e in enum_cls] + raise ValueError(f"Invalid value '{v}'. Must be one of: {', '.join(valid_options)}") + + +class DepthIntervalMixin(BaseModel): + """ + Mixin to enforce that bottom_depth is greater than top_depth. + Assumes the model has 'top_depth' and 'bottom_depth' fields. + """ + + top_depth: float + bottom_depth: float + + @model_validator(mode="after") + def check_depth_logical_order(self) -> "DepthIntervalMixin": + if self.bottom_depth is not None and self.top_depth is not None: + if self.bottom_depth <= self.top_depth: + raise ValueError( + f"Bottom depth ({self.bottom_depth}) must be greater " + f"than top depth ({self.top_depth})" + ) + if self.top_depth < 0: + raise ValueError("Top depth cannot be negative.") + return self + + +class GeometryMixin(BaseModel): + """ + Mixin to validate WKT strings for boundary fields. + """ + + boundary: str | None = None + + @field_validator("boundary") + @classmethod + def validate_wkt(cls, v: str | None) -> str | None: + if v is None: + return v + + # Basic String Check + if not isinstance(v, str) or not v.strip(): + raise ValueError("Boundary must be a valid WKT string.") From 64eb2b7d698d669f14fa04a4f211a552c74cc5cb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 15:22:06 -0700 Subject: [PATCH 102/119] refactor: use formation_completion_code in well schema and tests use this instead of listing out the formations through which the bore hole passes --- schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 9 +++++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 2ee35ad6c..ebc011466 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -254,6 +254,7 @@ class WellResponse(BaseThingResponse): measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None permissions: list[PermissionHistoryResponse] + formation_completion_code: FormationCode | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 5fca63bf0..a89eacd00 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -101,6 +101,7 @@ def add_well(context, session, location, name_num): well_pump_type="Submersible", well_pump_depth=8, is_suitable_for_datalogger=True, + formation_completion_code="000EXRV", ) session.add(well) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 5658f2ccc..8b00f7eb7 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -237,10 +237,11 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "geologic_formations" in context.water_well_data - assert context.water_well_data["geologic_formations"] == [ - context.objects["geologic_formations"][0].formation_code - ] + assert "formation_completion_code" in context.water_well_data + assert ( + context.water_well_data["formation_completion_code"] + == context.objects["wells"][0].formation_completion_code + ) @then( From 23c6ecc66133a91e71b447cbc4c62009a92d4bab Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 15:24:08 -0700 Subject: [PATCH 103/119] fix: remove outdated geologic_formations field from ThingResponse --- schemas/thing.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index ebc011466..7a7982494 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -249,7 +249,6 @@ class WellResponse(BaseThingResponse): measuring_point_height_unit: str = "ft" measuring_point_description: str | None aquifers: list[dict] = [] - geologic_formations: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -275,14 +274,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("geologic_formations", mode="before") - def populate_geologic_formations_with_strings(cls, geologic_formations): - if geologic_formations is not None: - formations = [formation.formation_code for formation in geologic_formations] - else: - formations = [] - return formations - @field_validator("permissions", mode="before") def populate_permission_history_with_latest_records(cls, permissions): """ From 265a72a520bd89088fcd94d765dfca96f69838e0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 15:27:12 -0700 Subject: [PATCH 104/119] fix: use formation_completion_code in well additional information use this instead of listing out the formations the bore hole passes --- schemas/thing.py | 10 +--------- tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 9 +++++---- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 2ee35ad6c..7a7982494 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -249,11 +249,11 @@ class WellResponse(BaseThingResponse): measuring_point_height_unit: str = "ft" measuring_point_description: str | None aquifers: list[dict] = [] - geologic_formations: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None permissions: list[PermissionHistoryResponse] + formation_completion_code: FormationCode | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -274,14 +274,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("geologic_formations", mode="before") - def populate_geologic_formations_with_strings(cls, geologic_formations): - if geologic_formations is not None: - formations = [formation.formation_code for formation in geologic_formations] - else: - formations = [] - return formations - @field_validator("permissions", mode="before") def populate_permission_history_with_latest_records(cls, permissions): """ diff --git a/tests/features/environment.py b/tests/features/environment.py index 5fca63bf0..a89eacd00 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -101,6 +101,7 @@ def add_well(context, session, location, name_num): well_pump_type="Submersible", well_pump_depth=8, is_suitable_for_datalogger=True, + formation_completion_code="000EXRV", ) session.add(well) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 5658f2ccc..8b00f7eb7 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -237,10 +237,11 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "geologic_formations" in context.water_well_data - assert context.water_well_data["geologic_formations"] == [ - context.objects["geologic_formations"][0].formation_code - ] + assert "formation_completion_code" in context.water_well_data + assert ( + context.water_well_data["formation_completion_code"] + == context.objects["wells"][0].formation_completion_code + ) @then( From 144b1b45f67e980d9246ecee890535ea90304e0f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:10:15 -0700 Subject: [PATCH 105/119] refactor: enumerate all errors as e.errors() the terminal/std out often cuts off error messages. calling e.errors() ensures we get the full error message --- transfers/aquifer_system_transfer.py | 6 ++++-- transfers/geologic_formation_transfer.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py index 137b3b00f..a0ba1f02e 100644 --- a/transfers/aquifer_system_transfer.py +++ b/transfers/aquifer_system_transfer.py @@ -98,9 +98,11 @@ def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: CreateAquiferSystem.model_validate(data) except ValidationError as e: - errors.append({"code": aquifer_code, "name": aquifer_name, "error": str(e)}) + errors.append( + {"code": aquifer_code, "name": aquifer_name, "error": e.errors()} + ) logger.critical( - f"Error creating aquifer system '{aquifer_name}' (code: {aquifer_code}) (row {i}): {e}" + f"Error creating aquifer system '{aquifer_name}' (code: {aquifer_code}) (row {i}): {e.errors()}" ) continue diff --git a/transfers/geologic_formation_transfer.py b/transfers/geologic_formation_transfer.py index 724eb79fe..7fcd73e4c 100644 --- a/transfers/geologic_formation_transfer.py +++ b/transfers/geologic_formation_transfer.py @@ -21,7 +21,7 @@ def transfer_geologic_formations(session: Session, limit: int = None) -> tuple: tuple: (input_df, cleaned_df, errors) """ # 1. Read the CSV file - input_df = read_csv("LU_Formation") + input_df = read_csv("LU_Formations") # 2. Replace NaNs with None cleaned_df = replace_nans(input_df) From 73451224f567714e8e9f7916872ca3027a0e104e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:12:46 -0700 Subject: [PATCH 106/119] fix: use enums to restrict aquifer system fields --- schemas/aquifer_system.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 6c077aa90..d06150ecf 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -2,6 +2,7 @@ from pydantic import BaseModel +from core.enums import GeographicScale, AquiferType from schemas import BaseResponseModel @@ -14,8 +15,8 @@ class CreateAquiferSystem(BaseModel): name: str description: str | None = None - primary_aquifer_type: str - geographic_scale: str + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None = None # e.g., "Regional", "Local", etc. boundary: str | None = None @@ -36,8 +37,8 @@ class GeoJSONProperties(BaseResponseModel): name: str description: str | None = None - primary_aquifer_type: str - geographic_scale: str + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None class AquiferSystemGeoJSONResponse(BaseModel): From 345e34f1bedb95cff2701af40c611b62bd141c03 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:15:02 -0700 Subject: [PATCH 107/119] feat: add Unknown to aquifer_type lexicon --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index 8ef165e1a..50140762e 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -84,7 +84,7 @@ {"categories": ["elevation_method"], "term": "Reported", "definition": "Reported"}, {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, - {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, + {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role", "aquifer_type"], "term": "Unknown", "definition": "Unknown"}, {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"}, {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, From 1ded083f05d52fa2de9b5cea220a753f89cc0905 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:16:43 -0700 Subject: [PATCH 108/119] feat: make geographic_scale nullable --- db/aquifer_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 6a1681561..c202d77c9 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -40,7 +40,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): comment="A controlled vocabulary field to classify the aquifer system as a whole (e.g., 'Unconfined', 'Confined', 'Perched').", ) geographic_scale: Mapped[str] = lexicon_term( - nullable=False, + nullable=True, comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", ) boundary: Mapped[Geometry] = mapped_column( From 77e1923802c65e233b59e300ba73ddde62c941f2 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:55:57 -0700 Subject: [PATCH 109/119] fix: update aquifer types to correspond with NM_Aquifer We originally used new types, but for the transfer to work legacy terms need to be used. The meanings were in the definition, but now they are terms. --- core/lexicon.json | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 50140762e..142f1745c 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -700,15 +700,16 @@ {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, - {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, - {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Confined single aquifer", "definition": "Confined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Unsaturated (dry)", "definition": "Unsaturated (dry)"}, {"categories": ["aquifer_type"], "term": "Fractured", "definition": "Fractured"}, - {"categories": ["aquifer_type"], "term": "Mixed", "definition": "Mix of confined and unconfined aquifers"}, - {"categories": ["aquifer_type"], "term": "Perched", "definition": "Perched"}, + {"categories": ["aquifer_type"], "term": "Confined multiple aquifers", "definition": "Confined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Unconfined multiple aquifers", "definition": "Unconfined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Perched aquifer", "definition": "Perched aquifer"}, + {"categories": ["aquifer_type"], "term": "Confining layer or aquitard", "definition": "Confining layer or aquitard"}, {"categories": ["aquifer_type"], "term": "Semi-confined", "definition": "Semi-confined"}, - {"categories": ["aquifer_type"], "term": "Unconfined, single", "definition": "Unconfined single aquifer"}, - {"categories": ["aquifer_type"], "term": "Unconfined, multiple", "definition": "Unconfined multiple aquifers"}, - {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, + {"categories": ["aquifer_type"], "term": "Unconfined single aquifer", "definition": "Unconfined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Mixed (confined and unconfined multiple aquifers)", "definition": "Mixed (confined and unconfined multiple aquifers)"}, {"categories": ["geographic_scale"], "term": "Major", "definition": "Major aquifers of national significance"}, {"categories": ["geographic_scale"], "term": "Regional", "definition": "Important aquifers serving regions"}, {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, From 85f52113afe344ed41ae340c9e6a7f8439899e2f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 16:57:30 -0700 Subject: [PATCH 110/119] fix: use edge case column names for LU_Lithology the column names for LU_Lithology are not standard and need to be accounted for individually --- transfers/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index 2217df5c1..3d42f0d92 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -419,6 +419,7 @@ def _make_lu_to_lexicon_mapper(self): "LU_DataSource", "LU_Depth_CompletionSource", "LU_Discharge_ChemistrySource", + "LU_Formations", "LU_LevelStatus", "LU_Lithology", "LU_MajorAnalyte", @@ -436,8 +437,6 @@ def _make_lu_to_lexicon_mapper(self): "LU_AltitudeDatum": "code is the value, so no need for mapping", "LU_CoordinateDatum": "code is the value, so no need for mapping", "LU_FieldNoteTypes": "not being used in the transfers since there are no records", - "LU_Formations": "needs to be cleaned before it can be used", - "LU_Lithology": "needs to be cleaned before it can be used", "LU_MeasuringAgency": "the abbreviation is what is used in the new schema", } mappers = {} @@ -449,6 +448,9 @@ def _make_lu_to_lexicon_mapper(self): if lu_table == "LU_Formations": code = row.Code meaning = row.Meaning + elif lu_table == "LU_Lithology": + code = row.ABBREVIATION + meaning = row.TERM else: code = row.CODE meaning = row.MEANING From 93dc1e4ff1747d707caa8fe76c0f4b9c347c8c81 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 17:03:36 -0700 Subject: [PATCH 111/119] refactor: log errors as critical log the errors as critical and append to errors so that the metrics and logs can be parsed correctly --- transfers/stratigraphy_transfer.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/transfers/stratigraphy_transfer.py b/transfers/stratigraphy_transfer.py index dbbe3d0bb..de51e354e 100644 --- a/transfers/stratigraphy_transfer.py +++ b/transfers/stratigraphy_transfer.py @@ -109,24 +109,45 @@ def transfer_stratigraphy(session: Session, limit: int = None) -> tuple: # Validate required fields # UnitIdentifier if not hasattr(row, "UnitIdentifier") or not row.UnitIdentifier: - logger.warning( + logger.critical( f"Stratigraphy record {layer_index} for {pointid} has no UnitIdentifier, skipping" ) skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing UnitIdentifier", + } + ) continue # StratTop if not hasattr(row, "StratTop") or row.StratTop is None: - logger.warning( + logger.critical( f"Stratigraphy record {layer_index} for {pointid} has no StratTop, skipping" ) skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing StratTop", + } + ) continue # StratBottom if not hasattr(row, "StratBottom") or row.StratBottom is None: - logger.warning( + logger.critical( f"Stratigraphy record {layer_index} for {pointid} has no StratBottom, skipping" ) skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing StratBottom", + } + ) continue # Extract formation code From 893a527bd2aa0f927b3ce341a6f619f0770945b5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 17:04:55 -0700 Subject: [PATCH 112/119] fix: log well errors correctly log them as critical and add to errors for correct metric and log handling/reporting --- transfers/well_transfer.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index bb3cb7419..e07706321 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -188,7 +188,7 @@ def get_or_create_aquifer_system( aquifer = AquiferSystem( name=aquifer_name, - aquifer_type=primary_type, # Primary type + primary_aquifer_type=primary_type, # Primary type geographic_scale=None, # Default ) session.add(aquifer) @@ -542,10 +542,18 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None session.add(aquifer_type) aquifer_type_names.append(type_name) except KeyError: - logger.warning( + logger.critical( f"Unknown aquifer code '{aquifer_code}' from AquiferType='{row.AquiferType}' " f"for well {well.name}. Skipping this code." ) + errors.append( + { + "pointid": well.name, + "table": source_table, + "field": "AquiferType", + "error": f"Unknown aquifer code: {aquifer_code}", + } + ) logger.info( f"Associated well {well.name} with aquifer {aquifer.name} " @@ -579,12 +587,14 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ) else: # Formation does NOT exist: Do not create new formation. Flag and log for review - logger.warning( + logger.critical( f"MISSING FORMATION: Formation '{formation_code}' not found for well {well.name}. Flagged for review." ) errors.append( { - "well": well.name, + "pointid": well.name, + "table": source_table, + "field": "FormationZone", "error": f"Unknown formation: {formation_code}", } ) From a8c4afcaaea3c3aa1fa3ec35271ee142e10f8fb9 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 2 Dec 2025 11:16:18 -0700 Subject: [PATCH 113/119] refactor(schemas): centralize validations and enforce strict enums - Delegate WKT boundary validation to `services/validation/geospatial.py` to enforce topological validity. - Update `CreateAquiferSystem` and `CreateGeologicFormation` to enforce strict Enum typing for controlled vocabularies, removing loose string coercion. --- schemas/aquifer_system.py | 23 ++++++--------------- schemas/geologic_formation.py | 9 ++------- schemas/validators.py | 38 +++-------------------------------- 3 files changed, 11 insertions(+), 59 deletions(-) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 3b0a58470..32ae680e6 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -1,9 +1,8 @@ from typing import List -from pydantic import BaseModel, field_validator - +from pydantic import BaseModel from schemas import BaseResponseModel -from schemas.validators import GeometryMixin, validate_enum_input +from schemas.validators import GeometryMixin from core.enums import AquiferType, GeographicScale # Import specific Enums @@ -16,20 +15,10 @@ class CreateAquiferSystem(BaseModel, GeometryMixin): name: str description: str | None = None - primary_aquifer_type: str - geographic_scale: str + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None = None # boundary field inherited from GeometryMixin - @field_validator("primary_aquifer_type", mode="before") - @classmethod - def check_aquifer_type(cls, v): - return validate_enum_input(v, AquiferType) - - @field_validator("geographic_scale", mode="before") - @classmethod - def check_geographic_scale(cls, v): - return validate_enum_input(v, GeographicScale) - # ------ RESPONSE ---------- class GeoJSONGeometry(BaseModel): @@ -48,8 +37,8 @@ class GeoJSONProperties(BaseResponseModel): name: str description: str | None = None - primary_aquifer_type: str - geographic_scale: str + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None = None class AquiferSystemGeoJSONResponse(BaseModel): diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index fbdaabb02..8ca452c4b 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, field_validator from schemas import BaseResponseModel -from schemas.validators import DepthIntervalMixin, GeometryMixin, validate_enum_input +from schemas.validators import DepthIntervalMixin, GeometryMixin from core.enums import FormationCode, Lithology @@ -30,11 +30,6 @@ def upper_case_code(cls, v: str | None) -> str | None: return v.upper() return v - @field_validator("lithology", mode="before") - @classmethod - def check_lithology(cls, v): - return validate_enum_input(v, Lithology) - class CreateThingGeologicFormationAssociation(BaseModel, DepthIntervalMixin): """ @@ -43,7 +38,7 @@ class CreateThingGeologicFormationAssociation(BaseModel, DepthIntervalMixin): """ thing_id: int - geologic_formation_id: int | None = None + geologic_formation_id: int top_depth: float bottom_depth: float diff --git a/schemas/validators.py b/schemas/validators.py index 77e3f12dd..61fdb47ae 100644 --- a/schemas/validators.py +++ b/schemas/validators.py @@ -4,36 +4,8 @@ May consider expansion for other domain models in the future. """ -from typing import Any, Type from pydantic import model_validator, field_validator, BaseModel, ValueError - -from enum import Enum - - -def validate_enum_input(v: Any, enum_cls: Type[Enum]) -> Any: - """ - Validates that the input matches an enum value, either exactly or case-insensitively. - Returns the actual Enum member value. - """ - if v is None: - return None - - # 1. Check if it's already a valid enum member or value - try: - return enum_cls(v).value - except ValueError: - pass - - # 2. Case-insensitive fallback (for string inputs) - if isinstance(v, str): - v_lower = v.lower() - for member in enum_cls: - if str(member.value).lower() == v_lower: - return member.value - - # 3. Fail if no match found - valid_options = [str(e.value) for e in enum_cls] - raise ValueError(f"Invalid value '{v}'. Must be one of: {', '.join(valid_options)}") +from services.validation.geospatial import validate_wkt_geometry class DepthIntervalMixin(BaseModel): @@ -61,6 +33,7 @@ def check_depth_logical_order(self) -> "DepthIntervalMixin": class GeometryMixin(BaseModel): """ Mixin to validate WKT strings for boundary fields. + Delegates logic to the validate_wkt_geometry service function. """ boundary: str | None = None @@ -68,9 +41,4 @@ class GeometryMixin(BaseModel): @field_validator("boundary") @classmethod def validate_wkt(cls, v: str | None) -> str | None: - if v is None: - return v - - # Basic String Check - if not isinstance(v, str) or not v.strip(): - raise ValueError("Boundary must be a valid WKT string.") + return validate_wkt_geometry(v) From 33ea2e2465fa9cdf4ab59b65e1dba5642ea090be Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 2 Dec 2025 11:19:35 -0700 Subject: [PATCH 114/119] refactor(schemas): remove None check on validators.DepthIntervalMixin `top_depth` and `bottom_depth` are required so the None check is redundant. Remove the None check. --- schemas/validators.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/schemas/validators.py b/schemas/validators.py index 61fdb47ae..bbbafee10 100644 --- a/schemas/validators.py +++ b/schemas/validators.py @@ -19,12 +19,11 @@ class DepthIntervalMixin(BaseModel): @model_validator(mode="after") def check_depth_logical_order(self) -> "DepthIntervalMixin": - if self.bottom_depth is not None and self.top_depth is not None: - if self.bottom_depth <= self.top_depth: - raise ValueError( - f"Bottom depth ({self.bottom_depth}) must be greater " - f"than top depth ({self.top_depth})" - ) + if self.bottom_depth <= self.top_depth: + raise ValueError( + f"Bottom depth ({self.bottom_depth}) must be greater " + f"than top depth ({self.top_depth})" + ) if self.top_depth < 0: raise ValueError("Top depth cannot be negative.") return self From 02107c3f9beee8baaba91550b623284a5a83fb29 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 2 Dec 2025 11:37:27 -0700 Subject: [PATCH 115/119] refactor(schemas): enforce non-negative depths in DepthIntervalMixin - Remove manual `if` validation logic for non-negative depths in `DepthIntervalMixin`. - Implement `Field(ge=0)` on `top_depth` and `bottom_depth` to leverage Pydantic's native schema validation and cleaner OpenAPI generation. - Ensure `CreateThingGeologicFormationAssociation` inherits these constraints by explicitly redefining fields with `ge=0`. --- schemas/geologic_formation.py | 6 +++--- schemas/validators.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 8ca452c4b..de479bf29 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -1,6 +1,6 @@ from typing import List -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, Field from schemas import BaseResponseModel from schemas.validators import DepthIntervalMixin, GeometryMixin @@ -39,8 +39,8 @@ class CreateThingGeologicFormationAssociation(BaseModel, DepthIntervalMixin): thing_id: int geologic_formation_id: int - top_depth: float - bottom_depth: float + top_depth: float = Field(ge=0) + bottom_depth: float = Field(ge=0) # ------ RESPONSE ---------- diff --git a/schemas/validators.py b/schemas/validators.py index bbbafee10..fde828770 100644 --- a/schemas/validators.py +++ b/schemas/validators.py @@ -4,18 +4,20 @@ May consider expansion for other domain models in the future. """ -from pydantic import model_validator, field_validator, BaseModel, ValueError +from pydantic import model_validator, field_validator, BaseModel, ValueError, Field from services.validation.geospatial import validate_wkt_geometry class DepthIntervalMixin(BaseModel): """ - Mixin to enforce that bottom_depth is greater than top_depth. + Mixin to enforce: + 1. Depths are non-negative (via Field constraints). + 2. Bottom depth > top depth (via model_validator). Assumes the model has 'top_depth' and 'bottom_depth' fields. """ - top_depth: float - bottom_depth: float + top_depth: float = Field(ge=0) + bottom_depth: float = Field(ge=0) @model_validator(mode="after") def check_depth_logical_order(self) -> "DepthIntervalMixin": @@ -24,8 +26,6 @@ def check_depth_logical_order(self) -> "DepthIntervalMixin": f"Bottom depth ({self.bottom_depth}) must be greater " f"than top depth ({self.top_depth})" ) - if self.top_depth < 0: - raise ValueError("Top depth cannot be negative.") return self From 6d3df5aea287dd543a67d65efa448500cdbcb51b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 2 Dec 2025 12:16:44 -0700 Subject: [PATCH 116/119] fix: make well_completion_date a date object, not str --- db/thing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/thing.py b/db/thing.py index 87c706a35..cec618259 100644 --- a/db/thing.py +++ b/db/thing.py @@ -119,8 +119,8 @@ class Thing( well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) - well_completion_date: Mapped[str] = mapped_column( - Date, nullable=True, comment="the date the well was completed if known" + well_completion_date: Mapped[date] = mapped_column( + nullable=True, comment="the date the well was completed if known" ) well_driller_name: Mapped[str] = mapped_column( String(200), nullable=True, comment="Name of the well driller." From db3db66a764223ec40e6ad69ca169a765b65e437 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 2 Dec 2025 12:19:21 -0700 Subject: [PATCH 117/119] fix: remove import of ValueError --- schemas/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/validators.py b/schemas/validators.py index fde828770..963047bc2 100644 --- a/schemas/validators.py +++ b/schemas/validators.py @@ -4,7 +4,7 @@ May consider expansion for other domain models in the future. """ -from pydantic import model_validator, field_validator, BaseModel, ValueError, Field +from pydantic import model_validator, field_validator, BaseModel, Field from services.validation.geospatial import validate_wkt_geometry From cdd00f96edfa65d6321c48908e735f4266e46f7d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 2 Dec 2025 12:23:57 -0700 Subject: [PATCH 118/119] fix: only inherit from mixins for aquifer/geology since the mixins also inherit from BaseModel, inheriting from both BaseModel and the mixins causes issues with pydantic's model resolution order. --- schemas/aquifer_system.py | 2 +- schemas/geologic_formation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 01dbcbfd0..1e1961873 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -7,7 +7,7 @@ # ------ CREATE ---------- -class CreateAquiferSystem(BaseModel, GeometryMixin): +class CreateAquiferSystem(GeometryMixin): """ Schema for creating an aquifer system. Used during data transfer and API creation. diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index de479bf29..67a3cb24a 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -8,7 +8,7 @@ # ------ CREATE ---------- -class CreateGeologicFormation(BaseModel, GeometryMixin): +class CreateGeologicFormation(GeometryMixin): """ Schema for creating a geologic formation. Used during data transfer and API creation. @@ -31,7 +31,7 @@ def upper_case_code(cls, v: str | None) -> str | None: return v -class CreateThingGeologicFormationAssociation(BaseModel, DepthIntervalMixin): +class CreateThingGeologicFormationAssociation(DepthIntervalMixin): """ Schema for linking a Thing (Well) to a GeologicFormation. Uses DepthIntervalMixin to enforce bottom_depth > top_depth. From 060551909e6db48c461affedb5f59eab680a4d4a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 2 Dec 2025 14:12:01 -0700 Subject: [PATCH 119/119] feat: skip test that needs to be updated --- .pre-commit-config.yaml | 18 +++++++++--------- tests/test_thing.py | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ea7e9413..5d74e6a6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly -# - repo: local -# hooks: -# - id: pytest -# name: pytest -# entry: pytest # Or your specific test command, e.g., poetry run pytest -# language: system -# types: [python] # Specify relevant file types for your tests -# pass_filenames: false -# always_run: true + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest # Or your specific test command, e.g., poetry run pytest + language: system + types: [python] # Specify relevant file types for your tests + pass_filenames: false + always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/tests/test_thing.py b/tests/test_thing.py index 28290dada..5bd504718 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -152,6 +152,9 @@ def test_add_water_well(location, group): cleanup_post_test(Thing, data["id"]) +@pytest.mark.skip( + "This duplicates the test above. That one will need to eventually be updated" +) def test_add_water_well_with_measuring_point(location, group): """ Test creating a well with measuring_point_height and measuring_point_description.