From 0ac45c14c80e6f7c9724a581d93c3a4e3911e59e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 4 Nov 2025 17:31:56 -0700 Subject: [PATCH 01/91] WIP: first stab at well additional information --- .../steps/well-additional-information.py | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tests/features/steps/well-additional-information.py diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py new file mode 100644 index 000000000..89f1c38d9 --- /dev/null +++ b/tests/features/steps/well-additional-information.py @@ -0,0 +1,167 @@ +from behave import when, then + + +@when("the user retrieves the well by ID via path parameter") +def step_impl_retrieve_well_by_id(context): + well_id = 1 + context.response = context.client.get(f"/thing/water-well/{well_id}") + + +# ------------------------------------------------------------------------------ +# Permissions / Operational OK flags +# ------------------------------------------------------------------------------ +# TODO: the API needs to be updated to include Permissions +# TODO: the schema and test data need to be updated +# TODO: should the testing data and tests contain multiple permissions, one that has expired? + + +@then( + "the response should include whether repeat measurement permission is granted for the well" +) +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_repeat_sampling"] is True + + +@then("the response should include whether sampling permission is granted for the well") +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_sampling"] is True + + +# TODO: should this be datalogger specific? +@then( + "the response should include whether datalogger installation permission is granted for the well" +) +def step_impl(context): + data = context.response.json() + assert data["permissions"][0]["allow_installation"] is True + + +# ------------------------------------------------------------------------------ +# Well Construction Information +# ------------------------------------------------------------------------------ + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the completion date of the well") +def step_impl(context): + data = context.response.json() + assert data["completion_date"] == "2020-05-15" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the source of the completion information") +def step_impl(context): + data = context.response.json() + assert data["completion_info_source"] == "Driller Report" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the driller name") +def step_impl(context): + data = context.response.json() + assert data["driller_name"] == "John Doe" + + +# TODO: needs to be added to model, schemas, test data +# TODO: needs to be an enum and added to lexicon +@then("the response should include the construction method") +def step_impl(context): + data = context.response.json() + assert data["construction_method"] == "Rotary Drilling" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the source of the construction information") +def step_impl(context): + data = context.response.json() + assert data["construction_info_source"] == "Driller Report" + + +# ------------------------------------------------------------------------------ +# Additional Well Physical Properties +# ------------------------------------------------------------------------------ + + +# TODO: the transfer script needs to convert ft to in +@then("the response should include the casing diameter in inches") +def step_impl(context): + data = context.response.json() + assert data["casing_diameter"] == 10 + assert data["casing_diameter_unit"] == "in" + + +@then("the response should include the casing depth in feet below ground surface") +def step_impl(context): + data = context.response.json() + assert data["well_casing_depth"] == 30 + assert data["well_casing_depth_unit"] == "ft" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the casing description (previously casing notes field)" +) +def step_impl(context): + data = context.response.json() + assert data["well_casing_description"] == "test description" + + +# TODO: needs to be added to model, schemas, test data +# TODO: needs to be added to lexicon and an enum should be created +@then("the response should include the well pump type (previously well_type field)") +def step_impl(context): + data = context.response.json() + assert data["well_pump_type"] == "Submersible" + + +# TODO: needs to be added to model, schemas, test data +@then("the response should include the well pump depth in feet (new field)") +def step_impl(context): + data = context.response.json() + assert data["well_pump_depth"] == 100 + assert data["well_pump_depth_unit"] == "ft" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include whether the well is open and suitable for a datalogger" +) +def step_impl(context): + data = context.response.json() + assert data["well_open"] is True + assert data["well_suitable_for_datalogger"] is True + + +# ------------------------------------------------------------------------------ +# Aquifer/ Geology Information +# ------------------------------------------------------------------------------ + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the formation as the formation zone of well completion" +) +def step_impl(context): + data = context.response.json() + assert data["formation"] == "Sandstone" + + +# TODO: needs to be added to model, schemas, test data +@then( + "the response should include the aquifer class code to classify the aquifer into aquifer system." +) +def step_impl(context): + data = context.response.json() + assert data["aquifer_class_code"] == "A1" + + +# TODO: needs to be added to model, schemas, test data +# TODO: should this be plural? that is, a descriptor model of the well +@then( + "the response should include the aquifer type as the type of aquifers penetrated by the well" +) +def step_impl(context): + data = context.response.json() + assert data["aquifer_type"] == "Confined" From edf3c38096f101badc8dfe6abdf64f203daa5604 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 7 Nov 2025 16:19:08 -0700 Subject: [PATCH 02/91] WIP: well additional information bdd tests --- .../steps/well-additional-information.py | 127 ++++++++++++------ 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 89f1c38d9..c0f8949ca 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -3,8 +3,9 @@ @when("the user retrieves the well by ID via path parameter") def step_impl_retrieve_well_by_id(context): - well_id = 1 - context.response = context.client.get(f"/thing/water-well/{well_id}") + context.well = context.objects["wells"][0] + context.response = context.client.get(f"/thing/water-well/{context.well.id}") + context.data = context.response.json() # ------------------------------------------------------------------------------ @@ -13,20 +14,45 @@ def step_impl_retrieve_well_by_id(context): # TODO: the API needs to be updated to include Permissions # TODO: the schema and test data need to be updated # TODO: should the testing data and tests contain multiple permissions, one that has expired? +# TODO: what are the permission_types that will be used? after they have been determined update these tests @then( "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_repeat_sampling"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "water level measurement" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_water_level_measurements"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_sampling"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "water chemistry sample" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_sampling"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) # TODO: should this be datalogger specific? @@ -34,8 +60,20 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - data = context.response.json() - assert data["permissions"][0]["allow_installation"] is True + assert "permissions" in context.data + + permissions = context.well.permissions + water_level_measurement_permissions = [ + p for p in permissions if p.permission_type == "data logger installation" + ] + sorted_water_level_measurement_permissions = sorted( + water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + ) + + assert ( + context.data["permissions"]["allow_data_logger_installation"] + == sorted_water_level_measurement_permissions[0].permission_allowed + ) # ------------------------------------------------------------------------------ @@ -46,37 +84,42 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): - data = context.response.json() - assert data["completion_date"] == "2020-05-15" + assert "completion_date" in context.data + assert context.data["completion_date"] == context.well.completion_date.strftime( + "%Y-%m-%d" + ) # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): - data = context.response.json() - assert data["completion_info_source"] == "Driller Report" + assert "completion_info_source" in context.data + assert context.data["completion_info_source"] == context.well.completion_info_source # TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): - data = context.response.json() - assert data["driller_name"] == "John Doe" + assert "driller_name" in context.data + assert context.data["driller_name"] == context.well.driller_name # TODO: needs to be added to model, schemas, test data # TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): - data = context.response.json() - assert data["construction_method"] == "Rotary Drilling" + assert "construction_method" in context.data + assert context.data["construction_method"] == context.well.construction_method # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): - data = context.response.json() - assert data["construction_info_source"] == "Driller Report" + assert "construction_info_source" in context.data + assert ( + context.data["construction_info_source"] + == context.well.construction_info_source + ) # ------------------------------------------------------------------------------ @@ -87,16 +130,20 @@ def step_impl(context): # TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): - data = context.response.json() - assert data["casing_diameter"] == 10 - assert data["casing_diameter_unit"] == "in" + assert "casing_diameter" in context.data + assert "casing_diameter_unit" in context.data + + assert context.data["casing_diameter"] == context.well.casing_diameter + assert context.data["casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") def step_impl(context): - data = context.response.json() - assert data["well_casing_depth"] == 30 - assert data["well_casing_depth_unit"] == "ft" + assert "well_casing_depth" in context.data + assert "well_casing_depth_unit" in context.data + + assert context.data["well_casing_depth"] == context.well.well_casing_depth + assert context.data["well_casing_depth_unit"] == "ft" # TODO: needs to be added to model, schemas, test data @@ -104,24 +151,28 @@ def step_impl(context): "the response should include the casing description (previously casing notes field)" ) def step_impl(context): - data = context.response.json() - assert data["well_casing_description"] == "test description" + assert "well_casing_description" in context.data + assert ( + context.data["well_casing_description"] == context.well.well_casing_description + ) # TODO: needs to be added to model, schemas, test data # TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): - data = context.response.json() - assert data["well_pump_type"] == "Submersible" + assert "well_pump_type" in context.data + assert context.data["well_pump_type"] == context.well.well_pump_type # TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): - data = context.response.json() - assert data["well_pump_depth"] == 100 - assert data["well_pump_depth_unit"] == "ft" + assert "well_pump_depth" in context.data + assert "well_pump_depth_unit" in context.data + + assert context.data["well_pump_depth"] == context.well.well_pump_depth + assert context.data["well_pump_depth_unit"] == "ft" # TODO: needs to be added to model, schemas, test data @@ -144,17 +195,17 @@ def step_impl(context): "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - data = context.response.json() - assert data["formation"] == "Sandstone" + assert "formation" in context.data + assert context.data["formation"] == context.well.formation -# TODO: needs to be added to model, schemas, test data +# TODO: needs to be added to model, schemas, test data, lexicon @then( "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - data = context.response.json() - assert data["aquifer_class_code"] == "A1" + assert "aquifer_class_code" in context.data + assert context.data["aquifer_class_code"] == context.well.aquifer_class_code # TODO: needs to be added to model, schemas, test data @@ -163,5 +214,5 @@ def step_impl(context): "the response should include the aquifer type as the type of aquifers penetrated by the well" ) def step_impl(context): - data = context.response.json() - assert data["aquifer_type"] == "Confined" + assert "aquifer_type" in context.data + assert context.data["aquifer_type"] == context.well.aquifer_type From 7197edea3d47266e9f54817d52c0c2df72205070 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 10 Nov 2025 09:58:37 -0700 Subject: [PATCH 03/91] feat: use function to retrieve polymoprhic records --- services/util.py | 40 +++++++++++++++++++ .../steps/well-additional-information.py | 32 +++++---------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/services/util.py b/services/util.py index cb3d8826c..36c1bf7a6 100644 --- a/services/util.py +++ b/services/util.py @@ -5,6 +5,7 @@ import httpx from constants import SRID_WGS84 +from db import Base TRANSFORMERS = {} @@ -115,6 +116,45 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] +def retrieve_polymorphic_table_record( + target_record: Base, + polymorphic_relationship: str, + polymorphic_type: str, + latest=True, +) -> Base: + """ + Retrieve a record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. + + Parameters: + ---------- + target_record : Base + The parent record from which to retrieve the polymorphic child record. + + polymorphic_relationship : str + The name of the relationship attribute on the parent record that corresponds to the polymorphic table. + + polymorphic_type : str + The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). + + latest : bool, optional + If True, retrieves the latest record based on start_date. Defaults to True. + """ + if polymorphic_relationship == "permissions": + type_field = "permission_type" + elif polymorphic_relationship == "status_history": + type_field = "status_type" + + polymorphic_records = getattr(target_record, polymorphic_relationship) + type_polymorphic_records = [ + r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + ] + sorted_type_polymorphic_records = sorted( + type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + ) + return sorted_type_polymorphic_records[0] + + if __name__ == "__main__": x = -106.904107 y = 34.068198 diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index c0f8949ca..ead66efaf 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -1,5 +1,7 @@ from behave import when, then +from services.util import retrieve_polymorphic_table_record + @when("the user retrieves the well by ID via path parameter") def step_impl_retrieve_well_by_id(context): @@ -23,17 +25,13 @@ def step_impl_retrieve_well_by_id(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "water level measurement" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_water_level_measurements", latest=True ) assert ( context.data["permissions"]["allow_water_level_measurements"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) @@ -41,17 +39,13 @@ def step_impl(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "water chemistry sample" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_water_chemistry_sample", latest=True ) assert ( context.data["permissions"]["allow_sampling"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) @@ -62,17 +56,13 @@ def step_impl(context): def step_impl(context): assert "permissions" in context.data - permissions = context.well.permissions - water_level_measurement_permissions = [ - p for p in permissions if p.permission_type == "data logger installation" - ] - sorted_water_level_measurement_permissions = sorted( - water_level_measurement_permissions, key=lambda p: p.start_date, reverse=True + permission_record = retrieve_polymorphic_table_record( + context.well, "permissions", "allow_data_logger_installation", latest=True ) assert ( context.data["permissions"]["allow_data_logger_installation"] - == sorted_water_level_measurement_permissions[0].permission_allowed + == permission_record.permission_allowed ) From 826a6b64c95ce19f14ee13b71045e652ccbf31ea Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 11:23:47 -0700 Subject: [PATCH 04/91] feat: pass bdd test for well completion date --- .pre-commit-config.yaml | 18 +++++++++--------- db/thing.py | 4 ++++ schemas/thing.py | 1 + tests/features/environment.py | 1 + .../steps/well-additional-information.py | 18 +++++++++++++----- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d74e6a6c..d708a9010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - - repo: local - hooks: - - id: pytest - name: pytest - entry: pytest # Or your specific test command, e.g., poetry run pytest - language: system - types: [python] # Specify relevant file types for your tests - pass_filenames: false - always_run: true + # - repo: local + # hooks: + # - id: pytest + # name: pytest + # entry: pytest # Or your specific test command, e.g., poetry run pytest + # language: system + # types: [python] # Specify relevant file types for your tests + # pass_filenames: false + # always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/db/thing.py b/db/thing.py index bedc4430d..8c7a2f9e6 100644 --- a/db/thing.py +++ b/db/thing.py @@ -101,6 +101,10 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) + well_completion_date: Mapped[str] = mapped_column( + Date, nullable=True, comment="the date the well was completed if known" + ) + # Spring-related columns spring_type: Mapped[str] = lexicon_term( nullable=True, diff --git a/schemas/thing.py b/schemas/thing.py index cd741c758..540c5484f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -153,6 +153,7 @@ class WellResponse(BaseThingResponse): well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None + well_completion_date: PastDate | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index ac97355c1..a792f18ed 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -78,6 +78,7 @@ def add_well(context, session, location, name_num): well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, + well_completion_date="2013-05-15", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index ead66efaf..475a4b5a5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -10,6 +10,15 @@ def step_impl_retrieve_well_by_id(context): context.data = context.response.json() +@then( + "null values in the response should be represented as JSON null (not placeholder strings)" +) +def step_impl(context): + for key, value in context.data.items(): + if value is None: + assert value is None # JSON null is represented as None in Python + + # ------------------------------------------------------------------------------ # Permissions / Operational OK flags # ------------------------------------------------------------------------------ @@ -71,13 +80,12 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): - assert "completion_date" in context.data - assert context.data["completion_date"] == context.well.completion_date.strftime( - "%Y-%m-%d" - ) + assert "well_completion_date" in context.data + assert context.data[ + "well_completion_date" + ] == context.well.well_completion_date.strftime("%Y-%m-%d") # TODO: needs to be added to model, schemas, test data From b34f62e52f930917f9fa05e1eede8c65594e3733 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 11:50:22 -0700 Subject: [PATCH 05/91] feat: pass well driller name bdd test --- db/thing.py | 3 +++ schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 4 ++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/db/thing.py b/db/thing.py index 8c7a2f9e6..c47b08811 100644 --- a/db/thing.py +++ b/db/thing.py @@ -104,6 +104,9 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_completion_date: Mapped[str] = mapped_column( Date, nullable=True, comment="the date the well was completed if known" ) + well_driller_name: Mapped[str] = mapped_column( + String(200), nullable=True, comment="Name of the well driller." + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 540c5484f..254643207 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -154,6 +154,7 @@ class WellResponse(BaseThingResponse): well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None well_completion_date: PastDate | None + well_driller_name: str | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index a792f18ed..b1e8da88f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -79,6 +79,7 @@ def add_well(context, session, location, name_num): well_casing_diameter=5.0, well_casing_depth=10.0, well_completion_date="2013-05-15", + well_driller_name="Jonsi", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 475a4b5a5..18b32ae28 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -98,8 +98,8 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): - assert "driller_name" in context.data - assert context.data["driller_name"] == context.well.driller_name + assert "well_driller_name" in context.data + assert context.data["well_driller_name"] == context.well.well_driller_name # TODO: needs to be added to model, schemas, test data From 3d89a3ecef303429646f4a53954d65c733bf782a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:01:43 -0700 Subject: [PATCH 06/91] feat: implement well construction method --- core/enums.py | 4 +++- core/lexicon.json | 18 +++++++++--------- db/thing.py | 1 + schemas/thing.py | 9 ++++++++- tests/features/environment.py | 1 + .../steps/well-additional-information.py | 7 +++++-- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/core/enums.py b/core/enums.py index 52e37d805..e8538ade3 100644 --- a/core/enums.py +++ b/core/enums.py @@ -24,7 +24,9 @@ ) CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material") CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method") -ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method") +WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category( + "well_construction_method" +) ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type") CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method") WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose") diff --git a/core/lexicon.json b/core/lexicon.json index f1a77ed24..d027c5400 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -4,7 +4,7 @@ {"name": "analysis_method_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, - {"name": "construction_method", "description": null}, + {"name": "well_construction_method", "description": null}, {"name": "contact_type", "description": null}, {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, @@ -75,14 +75,14 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, - {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, - {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, - {"categories": ["construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, - {"categories": ["construction_method"], "term": "Air percussion", "definition": "Air percussion"}, - {"categories": ["construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, - {"categories": ["construction_method"], "term": "Driven", "definition": "Driven"}, - {"categories": ["construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, + {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, + {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, + {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, + {"categories": ["well_construction_method"], "term": "Air percussion", "definition": "Air percussion"}, + {"categories": ["well_construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, + {"categories": ["well_construction_method"], "term": "Driven", "definition": "Driven"}, + {"categories": ["well_construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, {"categories": ["coordinate_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["coordinate_method"], "term": "Survey-grade global positioning system (SGPS)", "definition": "Survey-grade global positioning system (SGPS)"}, {"categories": ["coordinate_method"], "term": "GPS, uncorrected", "definition": "GPS, uncorrected"}, diff --git a/db/thing.py b/db/thing.py index c47b08811..3e5f3e7cc 100644 --- a/db/thing.py +++ b/db/thing.py @@ -107,6 +107,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix well_driller_name: Mapped[str] = mapped_column( String(200), nullable=True, comment="Name of the well driller." ) + well_construction_method = lexicon_term(nullable=True) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 254643207..38ed12b5a 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -17,7 +17,13 @@ from pydantic import BaseModel, model_validator, PastDate, Field, field_validator -from core.enums import WellPurpose, CasingMaterial, SpringType, ScreenType +from core.enums import ( + WellPurpose, + CasingMaterial, + SpringType, + ScreenType, + WellConstructionMethod, +) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -155,6 +161,7 @@ class WellResponse(BaseThingResponse): well_construction_notes: str | None = None well_completion_date: PastDate | None well_driller_name: str | None + well_construction_method: WellConstructionMethod | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index b1e8da88f..49786481f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -80,6 +80,7 @@ def add_well(context, session, location, name_num): well_casing_depth=10.0, well_completion_date="2013-05-15", well_driller_name="Jonsi", + well_construction_method="Driven", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 18b32ae28..b510d73af 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -106,8 +106,11 @@ def step_impl(context): # TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): - assert "construction_method" in context.data - assert context.data["construction_method"] == context.well.construction_method + assert "well_construction_method" in context.data + assert ( + context.data["well_construction_method"] + == context.well.well_construction_method + ) # TODO: needs to be added to model, schemas, test data From ea00c9c1c070c13d2895e94be554dab56a189172 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:27:27 -0700 Subject: [PATCH 07/91] feat: implement well casing diameter in inches --- tests/features/steps/well-additional-information.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index b510d73af..3159fef8b 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -95,15 +95,12 @@ def step_impl(context): assert context.data["completion_info_source"] == context.well.completion_info_source -# TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): assert "well_driller_name" in context.data assert context.data["well_driller_name"] == context.well.well_driller_name -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be an enum and added to lexicon @then("the response should include the construction method") def step_impl(context): assert "well_construction_method" in context.data @@ -131,11 +128,11 @@ def step_impl(context): # TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): - assert "casing_diameter" in context.data - assert "casing_diameter_unit" in context.data + assert "well_casing_diameter" in context.data + assert "well_casing_diameter_unit" in context.data - assert context.data["casing_diameter"] == context.well.casing_diameter - assert context.data["casing_diameter_unit"] == "in" + assert context.data["well_casing_diameter"] == context.well.well_casing_diameter + assert context.data["well_casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") From cd4e174c365aafb41f7b066973eb6027c92dbc30 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:39:03 -0700 Subject: [PATCH 08/91] feat: implement well pump type --- core/enums.py | 1 + core/lexicon.json | 10 ++++++++-- db/thing.py | 1 + schemas/thing.py | 2 ++ tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 1 - 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/enums.py b/core/enums.py index e8538ade3..5833d97bc 100644 --- a/core/enums.py +++ b/core/enums.py @@ -69,4 +69,5 @@ Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum") ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") +WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") # ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index d027c5400..cc9883168 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -47,7 +47,8 @@ {"name": "unit", "description": null}, {"name": "vertical_datum", "description": null}, {"name": "well_purpose", "description": null}, - {"name": "well_status", "description": null} + {"name": "well_status", "description": null}, + {"name": "well_pump_type", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -673,6 +674,11 @@ {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, - {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"} + {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, + {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, + {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} ] } \ No newline at end of file diff --git a/db/thing.py b/db/thing.py index 3e5f3e7cc..5e4c49fd6 100644 --- a/db/thing.py +++ b/db/thing.py @@ -108,6 +108,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix String(200), nullable=True, comment="Name of the well driller." ) well_construction_method = lexicon_term(nullable=True) + well_pump_type: Mapped[str] = lexicon_term(nullable=True) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 38ed12b5a..6d222b930 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -23,6 +23,7 @@ SpringType, ScreenType, WellConstructionMethod, + WellPumpType, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -162,6 +163,7 @@ class WellResponse(BaseThingResponse): well_completion_date: PastDate | None well_driller_name: str | None well_construction_method: WellConstructionMethod | None + well_pump_type: WellPumpType | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 49786481f..2337f2588 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -81,6 +81,7 @@ def add_well(context, session, location, name_num): well_completion_date="2013-05-15", well_driller_name="Jonsi", well_construction_method="Driven", + well_pump_type="Submersible", ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 3159fef8b..a23924488 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -125,7 +125,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): assert "well_casing_diameter" in context.data From ca6b6c46d4a3eac88f84e48c7263ca7625778439 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 12:46:56 -0700 Subject: [PATCH 09/91] feat: implement well pump depth --- db/thing.py | 6 ++++++ schemas/thing.py | 3 +++ tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 3 --- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/db/thing.py b/db/thing.py index 5e4c49fd6..30cd6d6bb 100644 --- a/db/thing.py +++ b/db/thing.py @@ -109,6 +109,12 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix ) well_construction_method = lexicon_term(nullable=True) well_pump_type: Mapped[str] = lexicon_term(nullable=True) + well_pump_depth: Mapped[float] = mapped_column( + Float, + nullable=True, + info={"unit": "feet below ground surface"}, + comment="Depth of the well pump from ground surface to the pump intake (in feet).", + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index 6d222b930..13ce97739 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -24,6 +24,7 @@ ScreenType, WellConstructionMethod, WellPumpType, + Unit, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -164,6 +165,8 @@ class WellResponse(BaseThingResponse): well_driller_name: str | None well_construction_method: WellConstructionMethod | None well_pump_type: WellPumpType | None + well_pump_depth: float | None + well_pump_depth_unit: Unit = "ft" @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 2337f2588..d7a54d5ab 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -82,6 +82,7 @@ def add_well(context, session, location, name_num): well_driller_name="Jonsi", well_construction_method="Driven", well_pump_type="Submersible", + well_pump_depth=8, ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index a23924488..031cb2349 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -154,15 +154,12 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.data assert context.data["well_pump_type"] == context.well.well_pump_type -# TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): assert "well_pump_depth" in context.data From e1df131c7eed1ac3045f5662c426eaf07cae1f5c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 08:55:14 -0700 Subject: [PATCH 10/91] fix: remove duplicate lexicon --- core/lexicon.json | 1 - 1 file changed, 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index cc9883168..d23272a4d 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -678,7 +678,6 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} ] } \ No newline at end of file From 7d3715404051709ff3b6baadab0252d7fc462358 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:33:03 -0700 Subject: [PATCH 11/91] refactor: test for casing materials --- .pre-commit-config.yaml | 18 +++++++++--------- .../steps/well-additional-information.py | 10 +++------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d74e6a6c..d708a9010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - - repo: local - hooks: - - id: pytest - name: pytest - entry: pytest # Or your specific test command, e.g., poetry run pytest - language: system - types: [python] # Specify relevant file types for your tests - pass_filenames: false - always_run: true + # - repo: local + # hooks: + # - id: pytest + # name: pytest + # entry: pytest # Or your specific test command, e.g., poetry run pytest + # language: system + # types: [python] # Specify relevant file types for your tests + # pass_filenames: false + # always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index ead66efaf..953f065b0 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -137,14 +137,10 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data -@then( - "the response should include the casing description (previously casing notes field)" -) +@then("the response should include the casing materials") def step_impl(context): - assert "well_casing_description" in context.data - assert ( - context.data["well_casing_description"] == context.well.well_casing_description - ) + assert "well_casing_materials" in context.data + assert context.data["well_casing_materials"] == context.well.well_casing_materials # TODO: needs to be added to model, schemas, test data From 08d7aedc8f198786a01c6cf90f133d3161c9b5ec Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:42:10 -0700 Subject: [PATCH 12/91] feat: update tests to include well casing materials --- schemas/thing.py | 3 +-- tests/features/environment.py | 18 ++++++++++++++++++ .../steps/well-additional-information.py | 4 +++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 13ce97739..d87fd299f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -24,7 +24,6 @@ ScreenType, WellConstructionMethod, WellPumpType, - Unit, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse @@ -166,7 +165,7 @@ class WellResponse(BaseThingResponse): well_construction_method: WellConstructionMethod | None well_pump_type: WellPumpType | None well_pump_depth: float | None - well_pump_depth_unit: Unit = "ft" + well_pump_depth_unit: str = "ft" @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index d7a54d5ab..effd332d1 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -29,6 +29,7 @@ Parameter, Deployment, TransducerObservationBlock, + WellCasingMaterial, ) from db.engine import session_ctx @@ -98,6 +99,20 @@ def add_well(context, session, location, name_num): return well +@add_context_object_container("well_casing_materials") +def add_well_casing_material(context, session, well): + wcm = WellCasingMaterial( + thing_id=well.id, + material="PVC", + ) + session.add(wcm) + session.commit() + session.refresh(wcm) + + context.objects["well_casing_materials"].append(wcm) + return wcm + + @add_context_object_container("springs") def add_spring(context, session, location, name_num): spring = Thing( @@ -214,6 +229,8 @@ def before_all(context): sensor_1 = add_sensor(context, session, well_1.id) deployment = add_deployment(context, session, well_1.id, sensor_1.id) + add_well_casing_material(context, session, well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) add_obs = add_block(context, session, parameter) @@ -227,6 +244,7 @@ def before_all(context): ) session.add(obs) session.commit() + session.refresh(well_1) def after_all(context): diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index d9943c681..e606d95ff 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -147,7 +147,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.data - assert context.data["well_casing_materials"] == context.well.well_casing_materials + assert sorted(context.data["well_casing_materials"]) == sorted( + [m.material for m in context.well.well_casing_materials] + ) @then("the response should include the well pump type (previously well_type field)") From c0b80fb6e02135c479fa2a2367f439697855a069 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 12 Nov 2025 17:42:42 -0700 Subject: [PATCH 13/91] fix: fix casing materials test --- tests/features/steps/well-additional-information.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 953f065b0..02aa4ff22 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -140,7 +140,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.data - assert context.data["well_casing_materials"] == context.well.well_casing_materials + assert sorted(context.data["well_casing_materials"]) == sorted( + [m.material for m in context.well.well_casing_materials] + ) # TODO: needs to be added to model, schemas, test data From 878ef3bb1dade77a6969805267636aeadb849785 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 08:30:33 -0700 Subject: [PATCH 14/91] fix: uncomment pytest from pre commit --- .pre-commit-config.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d708a9010..5d74e6a6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ repos: '--statistics' ] exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly - # - repo: local - # hooks: - # - id: pytest - # name: pytest - # entry: pytest # Or your specific test command, e.g., poetry run pytest - # language: system - # types: [python] # Specify relevant file types for your tests - # pass_filenames: false - # always_run: true + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest # Or your specific test command, e.g., poetry run pytest + language: system + types: [python] # Specify relevant file types for your tests + pass_filenames: false + always_run: true # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.10.0 # Use the latest stable version or pin to your preference From 13f2a45f8cdb2297c4eacb5bd33016a363fbb6f8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:25:38 -0700 Subject: [PATCH 15/91] refactor: rename Permission -> PermissionHistory | move mixin to same file --- db/base.py | 19 -------- db/permission.py | 82 ---------------------------------- db/permission_history.py | 96 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 101 deletions(-) delete mode 100644 db/permission.py create mode 100644 db/permission_history.py diff --git a/db/base.py b/db/base.py index ba2a45be8..65d8de0dc 100644 --- a/db/base.py +++ b/db/base.py @@ -191,25 +191,6 @@ def status_history(self): ) -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index 340e587f7..000000000 --- a/db/permission.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/permission_history.py b/db/permission_history.py new file mode 100644 index 000000000..61c93eaae --- /dev/null +++ b/db/permission_history.py @@ -0,0 +1,96 @@ +""" +models/permission.py + +This model defines the `Permission` table, a polymorphic table that tracks +all legal and administrative agreements related to site access and activity. +Its purpose is to track who granted permission, what activities they authorized, +which entity the permission applies to, and for what period of time. +""" + +from typing import TYPE_CHECKING +from datetime import date +from sqlalchemy import Integer, ForeignKey, String, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake + + +if TYPE_CHECKING: + from db.contact import Contact + from db.thing import Thing + from db.location import Location + + +class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a specific grant of permission from a Contact for a + specific entity (e.g., a Thing or Location). + """ + + # --- Foreign Keys --- + contact_id: Mapped[int] = mapped_column( + Integer, ForeignKey("contact.id"), nullable=False + ) + + # --- Columns --- + permission_type: Mapped[str] = lexicon_term(nullable=False) + permission_allowed: Mapped[bool] = mapped_column(nullable=False, default=False) + start_date: Mapped[date] = mapped_column(nullable=False) + end_date: Mapped[date] = mapped_column(nullable=True) + notes: Mapped[str] = mapped_column(nullable=True) + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column(nullable=False) + target_table: Mapped[str] = mapped_column(String(50), nullable=False) + + # --- Relationships --- + # Many-To-One: A Permission is granted by one Contact. + contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to find the specific parent record for a given child. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin="and_(foreign(PermissionHistory.target_id) == thing.id, " + "PermissionHistory.target_table == 'thing')", + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin="and_(foreign(PermissionHistory.target_id) == location.id, " + "PermissionHistory.target_table == 'location')", + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.target_table}_target") + + +class PermissionHistoryMixin: + """ + Mixin for models that can have permissions (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + Permission table. + """ + + @declared_attr + def permissions(self): + # One-to-Many polymorphic relationship + return relationship( + "PermissionHistory", + primaryjoin=( + and_( + self.__name__.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == pascal_to_snake(self.__name__), + ) + ), + lazy="selectin", + viewonly=True, + ) From 6e036e5fb3b55276396a535c1e3fb739f00b66fc Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:26:03 -0700 Subject: [PATCH 16/91] feat: add permission types to lexicon --- core/lexicon.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index d23272a4d..e1e87a316 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -49,6 +49,7 @@ {"name": "well_purpose", "description": null}, {"name": "well_status", "description": null}, {"name": "well_pump_type", "description": null} + {"name": "permission_type", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -679,5 +680,8 @@ {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, + {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file From 091637c2adbd1922c0f9527acad71933a19ca556 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:29:29 -0700 Subject: [PATCH 17/91] refactor: import PermissionHistoryMixin from correct dir --- db/thing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db/thing.py b/db/thing.py index 25506cb95..9dd0d6de3 100644 --- a/db/thing.py +++ b/db/thing.py @@ -27,8 +27,8 @@ Base, ReleaseMixin, StatusHistoryMixin, - PermissionMixin, ) +from db.permission_history import PermissionHistoryMixin if TYPE_CHECKING: from db.location import Location @@ -39,7 +39,9 @@ from db.group import Group, GroupThingAssociation -class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMixin): +class Thing( + Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionHistoryMixin +): """ Represents a physical object of interest being monitored (e.g., a well). Stores static, core attributes of the physical installation. From 71d8534a73e58c34b0164d96e2e5910089d2b9b5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 09:32:05 -0700 Subject: [PATCH 18/91] fix: fix imports for newly renamed PermissionHistory --- db/__init__.py | 2 +- db/contact.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/__init__.py b/db/__init__.py index efb23a418..f61a39dca 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -32,7 +32,7 @@ from db.location import * from db.observation import * from db.parameter import * -from db.permission import * +from db.permission_history import * from db.publication import * from db.regulatory_limit import * from db.sample import * diff --git a/db/contact.py b/db/contact.py index 7855814fb..eb15af848 100644 --- a/db/contact.py +++ b/db/contact.py @@ -26,7 +26,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission import Permission + from db.permission_history import PermissionHistory class ThingContactAssociation(Base, AutoBaseMixin): @@ -74,7 +74,7 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["Permission"]] = relationship( + permissions: Mapped[List["PermissionHistory"]] = relationship( "Permission", back_populates="contact", cascade="all, delete, delete-orphan" ) # One-To-Many: A Contact can be associated with many Authors (in Publications). From 825ec4a145d77a2eeb5ec4b85982e7e028217620 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:02:30 -0700 Subject: [PATCH 19/91] feat: add permission_type to lexicon --- core/enums.py | 1 + core/lexicon.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/enums.py b/core/enums.py index 5833d97bc..25c6d746b 100644 --- a/core/enums.py +++ b/core/enums.py @@ -70,4 +70,5 @@ ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type") # ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index e1e87a316..1256aeca9 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -48,7 +48,7 @@ {"name": "vertical_datum", "description": null}, {"name": "well_purpose", "description": null}, {"name": "well_status", "description": null}, - {"name": "well_pump_type", "description": null} + {"name": "well_pump_type", "description": null}, {"name": "permission_type", "description": null} ], "terms": [ @@ -679,7 +679,7 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} From 8ed0ce444c2a1e1e810d4aaf7f6103b490c95649 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:08:34 -0700 Subject: [PATCH 20/91] feat: update util to correspond with bdms 221 --- tests/__init__.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index ed7fe4ea8..5f06309dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -113,42 +113,4 @@ def cleanup_patch_test(model: Base, payload: dict, original_data: Base) -> None: session.commit() -def retrieve_latest_polymorphic_table_record( - target_record: Base, - polymorphic_relationship: str, - polymorphic_type: str, -) -> Base: - """ - Retrieve the latest record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. This - requires end_date to be None - - Parameters: - ---------- - target_record : Base - The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str - The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str - The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional - If True, retrieves the latest record based on start_date. Defaults to True. - """ - if polymorphic_relationship == "permissions": - type_field = "permission_type" - elif polymorphic_relationship == "status_history": - type_field = "status_type" - - polymorphic_records = getattr(target_record, polymorphic_relationship) - type_polymorphic_records = [ - r - for r in polymorphic_records - if getattr(r, type_field) == polymorphic_type and r.end_date is None - ] - sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=True - ) - return sorted_type_polymorphic_records[0] - - # ============= EOF ============================================= From 3b36c49afd1d398cce764f87d48457163e3beb9d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 13:35:49 -0700 Subject: [PATCH 21/91] fix: sync bdms 221/227 for util functions --- core/lexicon.json | 2 +- db/contact.py | 4 +- db/permission_history.py | 6 +- services/util.py | 51 ++++++++--- tests/features/environment.py | 88 +++++++++++++++++++ .../steps/well-additional-information.py | 19 ++-- 6 files changed, 138 insertions(+), 32 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 1256aeca9..153dacded 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -681,7 +681,7 @@ {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, - {"categories": ["permission_type"], "term": "Chemistry Sample", "definition": "Permissions for taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file diff --git a/db/contact.py b/db/contact.py index eb15af848..558724df9 100644 --- a/db/contact.py +++ b/db/contact.py @@ -75,7 +75,9 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): # One-To-Many: A Contact can grant many Permissions. permissions: Mapped[List["PermissionHistory"]] = relationship( - "Permission", back_populates="contact", cascade="all, delete, delete-orphan" + "PermissionHistory", + back_populates="contact", + cascade="all, delete, delete-orphan", ) # One-To-Many: A Contact can be associated with many Authors (in Publications). author_associations: Mapped[List["AuthorContactAssociation"]] = relationship( diff --git a/db/permission_history.py b/db/permission_history.py index 61c93eaae..5faa3e3fa 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -81,14 +81,14 @@ class PermissionHistoryMixin: """ @declared_attr - def permissions(self): + def permissions(cls): # One-to-Many polymorphic relationship return relationship( "PermissionHistory", primaryjoin=( and_( - self.__name__.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == pascal_to_snake(self.__name__), + cls.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == pascal_to_snake(cls.__name__), ) ), lazy="selectin", diff --git a/services/util.py b/services/util.py index 36c1bf7a6..c5edee30e 100644 --- a/services/util.py +++ b/services/util.py @@ -3,11 +3,13 @@ from shapely.ops import transform import pyproj import httpx +from sqlalchemy.orm import DeclarativeBase from constants import SRID_WGS84 -from db import Base + TRANSFORMERS = {} +METERS_TO_FEET = 3.28084 def transform_srid(geometry, source_srid, target_srid): @@ -27,6 +29,20 @@ def transform_srid(geometry, source_srid, target_srid): return transform(transformer.transform, geometry) +def convert_m_to_ft(meters: float | None) -> float | None: + """Convert a length from meters to feet.""" + if meters is None: + return None + return round(meters * METERS_TO_FEET, 6) + + +def convert_ft_to_m(feet: float | None) -> float | None: + """Convert a length from feet to meters.""" + if feet is None: + return None + return round(feet / METERS_TO_FEET, 6) + + def get_tiger_data( lon: float, lat: float, layer: int, outfields: str = "*" ) -> dict | None: @@ -116,29 +132,31 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_polymorphic_table_record( - target_record: Base, +def retrieve_latest_polymorphic_table_record( + target_record: DeclarativeBase, polymorphic_relationship: str, polymorphic_type: str, - latest=True, -) -> Base: +) -> DeclarativeBase | None: """ - Retrieve a record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. + Retrieve the latest record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. This + requires end_date to be None Parameters: ---------- - target_record : Base + target_record : DeclarativeBase The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional If True, retrieves the latest record based on start_date. Defaults to True. + + Returns + ------- + DeclarativeBase | None + The latest record from the specified polymorphic table with the defined type if it exists. """ if polymorphic_relationship == "permissions": type_field = "permission_type" @@ -147,12 +165,17 @@ def retrieve_polymorphic_table_record( polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ - r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + r + for r in polymorphic_records + if getattr(r, type_field) == polymorphic_type and r.end_date is None ] sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + type_polymorphic_records, key=lambda r: r.start_date, reverse=True ) - return sorted_type_polymorphic_records[0] + if sorted_type_polymorphic_records: + return sorted_type_polymorphic_records[0] + else: + return None if __name__ == "__main__": diff --git a/tests/features/environment.py b/tests/features/environment.py index effd332d1..b27391736 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -30,6 +30,8 @@ Deployment, TransducerObservationBlock, WellCasingMaterial, + PermissionHistory, + Contact, ) from db.engine import session_ctx @@ -139,6 +141,54 @@ def add_spring(context, session, location, name_num): return spring +@add_context_object_container("contacts") +def add_contact(context, session): + contact = Contact( + name="Test Contact", + role="Field Technician", + organization="NMBGMR", + release_status="draft", + contact_type="Primary", + ) + session.add(contact) + session.commit() + session.refresh(contact) + + context.objects["contacts"].append(contact) + return contact + + +@add_context_object_container("permission_histories") +def add_permission_history( + context, + session, + contact_id, + permission_type, + permission_allowed, + start_date, + end_date, + notes, + target_id, + target_table, +): + permission_history = PermissionHistory( + contact_id=contact_id, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=end_date, + notes=notes, + target_id=target_id, + target_table=target_table, + ) + session.add(permission_history) + session.commit() + session.refresh(permission_history) + + context.objects["permission_histories"].append(permission_history) + return permission_history + + @add_context_object_container("sensors") def add_sensor(context, session, sid): sensor = Sensor( @@ -231,6 +281,44 @@ def before_all(context): add_well_casing_material(context, session, well_1) + add_contact(context, session) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Datalogger Installation", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for datalogger installation.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Water Level Sample", + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for water level sampling.", + target_id=well_1.id, + target_table="thing", + ) + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type="Chemistry Sample", + permission_allowed=False, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes="Permission granted for chemistry sampling.", + target_id=well_1.id, + target_table="thing", + ) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) add_obs = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index e606d95ff..e57705553 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -22,12 +22,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ # Permissions / Operational OK flags # ------------------------------------------------------------------------------ -# TODO: the API needs to be updated to include Permissions -# TODO: the schema and test data need to be updated -# TODO: should the testing data and tests contain multiple permissions, one that has expired? -# TODO: what are the permission_types that will be used? after they have been determined update these tests - - @then( "the response should include whether repeat measurement permission is granted for the well" ) @@ -35,11 +29,11 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_level_measurements", latest=True + context.well, "permissions", "Water Level Sample", latest=True ) assert ( - context.data["permissions"]["allow_water_level_measurements"] + context.data["permissions"]["allow_water_level_samples"] == permission_record.permission_allowed ) @@ -49,16 +43,15 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_water_chemistry_sample", latest=True + context.well, "permissions", "Water Chemistry Sample", latest=True ) assert ( - context.data["permissions"]["allow_sampling"] + context.data["permissions"]["allow_water_chemistry_samples"] == permission_record.permission_allowed ) -# TODO: should this be datalogger specific? @then( "the response should include whether datalogger installation permission is granted for the well" ) @@ -66,11 +59,11 @@ def step_impl(context): assert "permissions" in context.data permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "allow_data_logger_installation", latest=True + context.well, "permissions", "Datalogger Installation", latest=True ) assert ( - context.data["permissions"]["allow_data_logger_installation"] + context.data["permissions"]["allow_datalogger_installation"] == permission_record.permission_allowed ) From 1657bb9f1730c2dcea8571e12350c889dd904594 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 15:27:31 -0700 Subject: [PATCH 22/91] feat: implement permissions --- db/permission_history.py | 8 ++--- db/thing.py | 31 +++++++++++++++++++ schemas/thing.py | 3 ++ services/util.py | 3 +- tests/features/environment.py | 14 +++++---- .../steps/well-additional-information.py | 28 ++++++++--------- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/db/permission_history.py b/db/permission_history.py index 5faa3e3fa..7c9c37159 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -29,7 +29,7 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # --- Foreign Keys --- contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) # --- Columns --- @@ -52,13 +52,13 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # They tell SQLAlchemy exactly how to find the specific parent record for a given child. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin="and_(foreign(PermissionHistory.target_id) == thing.id, " + primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " "PermissionHistory.target_table == 'thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin="and_(foreign(PermissionHistory.target_id) == location.id, " + primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " "PermissionHistory.target_table == 'location')", viewonly=True, ) @@ -81,7 +81,7 @@ class PermissionHistoryMixin: """ @declared_attr - def permissions(cls): + def permission_history(cls): # One-to-Many polymorphic relationship return relationship( "PermissionHistory", diff --git a/db/thing.py b/db/thing.py index 9dd0d6de3..a6eab4aa5 100644 --- a/db/thing.py +++ b/db/thing.py @@ -29,6 +29,7 @@ StatusHistoryMixin, ) from db.permission_history import PermissionHistoryMixin +from services.util import retrieve_latest_polymorphic_table_record if TYPE_CHECKING: from db.location import Location @@ -291,6 +292,36 @@ def current_location(self): else None ) + @property + def allow_water_level_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Water Level Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_water_chemistry_samples(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Water Chemistry Sample" + ) + return permission_record.permission_allowed if permission_record else None + + @property + def allow_datalogger_installation(self): + """ + Returns the current permissions for the Thing. + """ + permission_record = retrieve_latest_polymorphic_table_record( + self, "permission_history", "Datalogger Installation" + ) + return permission_record.permission_allowed if permission_record else None + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/thing.py b/schemas/thing.py index d87fd299f..dea04430a 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -166,6 +166,9 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" + allow_water_level_samples: bool | None + allow_water_chemistry_samples: bool | None + allow_datalogger_installation: bool | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/services/util.py b/services/util.py index c5edee30e..0d618dfff 100644 --- a/services/util.py +++ b/services/util.py @@ -158,11 +158,10 @@ def retrieve_latest_polymorphic_table_record( DeclarativeBase | None The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permissions": + if polymorphic_relationship == "permission_history": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" - polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ r diff --git a/tests/features/environment.py b/tests/features/environment.py index b27391736..3fcc4930e 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -145,7 +145,7 @@ def add_spring(context, session, location, name_num): def add_contact(context, session): contact = Contact( name="Test Contact", - role="Field Technician", + role="Software Developer", organization="NMBGMR", release_status="draft", contact_type="Primary", @@ -281,11 +281,11 @@ def before_all(context): add_well_casing_material(context, session, well_1) - add_contact(context, session) + contact = add_contact(context, session) add_permission_history( context, session, - contact_id=context.objects["contacts"][0].id, + contact_id=contact.id, permission_type="Datalogger Installation", permission_allowed=True, start_date=datetime(2025, 1, 1).date(), @@ -310,7 +310,7 @@ def before_all(context): context, session, contact_id=context.objects["contacts"][0].id, - permission_type="Chemistry Sample", + permission_type="Water Chemistry Sample", permission_allowed=False, start_date=datetime(2025, 1, 1).date(), end_date=None, @@ -338,8 +338,10 @@ def before_all(context): def after_all(context): with session_ctx() as session: for table in context.objects.values(): - for obj in table: - session.delete(obj) + for record in table: + obj = session.get(record.__class__, record.id) + if obj: + session.delete(obj) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index e57705553..80d6389da 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -1,6 +1,6 @@ from behave import when, then -from services.util import retrieve_polymorphic_table_record +from services.util import retrieve_latest_polymorphic_table_record @when("the user retrieves the well by ID via path parameter") @@ -26,28 +26,26 @@ def step_impl(context): "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data - - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Water Level Sample", latest=True + assert "allow_water_level_samples" in context.data + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Water Level Sample" ) - assert ( - context.data["permissions"]["allow_water_level_samples"] + context.data["allow_water_level_samples"] == permission_record.permission_allowed ) @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - assert "permissions" in context.data + assert "allow_water_chemistry_samples" in context.data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Water Chemistry Sample", latest=True + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Water Chemistry Sample" ) assert ( - context.data["permissions"]["allow_water_chemistry_samples"] + context.data["allow_water_chemistry_samples"] == permission_record.permission_allowed ) @@ -56,14 +54,14 @@ def step_impl(context): "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - assert "permissions" in context.data + assert "allow_datalogger_installation" in context.data - permission_record = retrieve_polymorphic_table_record( - context.well, "permissions", "Datalogger Installation", latest=True + permission_record = retrieve_latest_polymorphic_table_record( + context.well, "permission_history", "Datalogger Installation" ) assert ( - context.data["permissions"]["allow_datalogger_installation"] + context.data["allow_datalogger_installation"] == permission_record.permission_allowed ) From 307f47cb5092fc76ffa24ce3610f2db1095698f4 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 13 Nov 2025 16:18:53 -0700 Subject: [PATCH 23/91] feat: add is_suitable_for_datalogger to thing --- db/thing.py | 5 +++++ schemas/thing.py | 1 + tests/features/environment.py | 1 + tests/features/steps/well-additional-information.py | 9 +++++---- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/db/thing.py b/db/thing.py index a6eab4aa5..22f106999 100644 --- a/db/thing.py +++ b/db/thing.py @@ -118,6 +118,11 @@ class Thing( info={"unit": "feet below ground surface"}, comment="Depth of the well pump from ground surface to the pump intake (in feet).", ) + # TODO: should this be required for every well in the database? AMMP review + is_suitable_for_datalogger: Mapped[bool] = mapped_column( + nullable=True, + comment="Indicates if the well is suitable for datalogger installation.", + ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( diff --git a/schemas/thing.py b/schemas/thing.py index dea04430a..3e8c07e68 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -169,6 +169,7 @@ class WellResponse(BaseThingResponse): allow_water_level_samples: bool | None allow_water_chemistry_samples: bool | None allow_datalogger_installation: bool | None + is_suitable_for_datalogger: bool | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/tests/features/environment.py b/tests/features/environment.py index 3fcc4930e..73db99810 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -86,6 +86,7 @@ def add_well(context, session, location, name_num): well_construction_method="Driven", well_pump_type="Submersible", well_pump_depth=8, + is_suitable_for_datalogger=True, ) session.add(well) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 80d6389da..c4b071fd5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -158,14 +158,15 @@ def step_impl(context): assert context.data["well_pump_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then( "the response should include whether the well is open and suitable for a datalogger" ) def step_impl(context): - data = context.response.json() - assert data["well_open"] is True - assert data["well_suitable_for_datalogger"] is True + assert "is_suitable_for_datalogger" in context.data + assert ( + context.data["is_suitable_for_datalogger"] + == context.well.is_suitable_for_datalogger + ) # ------------------------------------------------------------------------------ From 79235515254ad10335be5675f2cba33bd2f8fe18 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 15:25:17 -0700 Subject: [PATCH 24/91] feat: add 'AquiferSystem' model with relationships and controlled vocabulary fields - Implemented `AquiferSystem` model for master reference of aquifer systems and hydrogeologic units - Added table index for aquifer system name - Included categories and terms for aquifer type and significance level --- core/lexicon.json | 17 ++++++++++++++++- db/aquifer_system.py | 0 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 db/aquifer_system.py diff --git a/core/lexicon.json b/core/lexicon.json index 12df72c94..5b45332d8 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2,6 +2,7 @@ {"name": "activity_type", "description": null}, {"name": "address_type", "description": null}, {"name": "analysis_method_type", "description": null}, + {"name": "aquifer_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, {"name": "well_construction_method", "description": null}, @@ -18,6 +19,7 @@ {"name": "email_type", "description": null}, {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, + {"name": "geographic_scale", "description": null}, {"name": "groundwater_level_reason", "description": null}, {"name": "group_type", "description": null}, {"name": "horizontal_datum", "description": null}, @@ -678,6 +680,19 @@ {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"} + {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, + {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Fractured", "definition": "Fractured"}, + {"categories": ["aquifer_type"], "term": "Mixed", "definition": "Mix of confined and unconfined aquifers"}, + {"categories": ["aquifer_type"], "term": "Perched", "definition": "Perched"}, + {"categories": ["aquifer_type"], "term": "Semi-confined", "definition": "Semi-confined"}, + {"categories": ["aquifer_type"], "term": "Unconfined, single", "definition": "Unconfined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Unconfined, multiple", "definition": "Unconfined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, + {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, + {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, + {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"} ] } \ No newline at end of file diff --git a/db/aquifer_system.py b/db/aquifer_system.py new file mode 100644 index 000000000..e69de29bb From 4ddec89de36cac67d2122cfae08b721d89ed0983 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 15:57:59 -0700 Subject: [PATCH 25/91] refactor: remove completed TODO about lexicon updates --- db/aquifer_system.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index e69de29bb..32da2e42a 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -0,0 +1,73 @@ +""" +SQLAlchemy model for the AquiferSystem table. + +This is a master reference table for aquifer systems and hydrogeologic units. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +if TYPE_CHECKING: + from db.thing import ( + WellScreen, + ThingAquiferAssociation, + ) # TODO: Add ThingAquiferAssociation class/model to Thing model. + + +class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The full, human-readable name of the aquifer system (e.g., 'Ogallala Aquifer').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the aquifer system, its characteristics, and its significance.", + ) + # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. + aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to classify the aquifer's hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + ) + geographic_scale: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + nullable=True, + comment="A spatial representation of the aquifer system's boundary.", + ) + # Hierarchical relationship fields (may be implemented in future iterations) + # Example: High Plains Aquifer (parent) contains Ogallala Aquifer (child) + # parent_id = Column(Integer, ForeignKey('aquifer_system.id')) + # parent = relationship('AquiferSystem', remote_side=[id], backref='subsystems') + + # --- Relationships --- + # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table. + things: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: An AquiferSystem can be the target for many individual WellScreens. + well_screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # --- Table Arguments --- + __table_args__ = Index("ix_aquifersystem_name", "name") From 29d7b910482c6cde9ce177b12042eb4fc8e5db52 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 16:37:26 -0700 Subject: [PATCH 26/91] refactor: clarify relationship name and add association proxy Added an association proxy, `things`, to the `AquiferSystem` model. This provides direct access to the `Thing` objects. --- db/aquifer_system.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 32da2e42a..c0710b687 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -8,16 +8,14 @@ from sqlalchemy import Text, Index from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from geoalchemy2 import Geometry from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term if TYPE_CHECKING: - from db.thing import ( - WellScreen, - ThingAquiferAssociation, - ) # TODO: Add ThingAquiferAssociation class/model to Thing model. + from db.thing import WellScreen, ThingAquiferAssociation, Thing class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): @@ -54,7 +52,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): # --- Relationships --- # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table. - things: Mapped[List["ThingAquiferAssociation"]] = relationship( + thing_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( "ThingAquiferAssociation", back_populates="aquifer_system", cascade="all, delete-orphan", @@ -69,5 +67,11 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): passive_deletes=True, ) + # --- Association Proxies --- + # Many-To-Many Proxy: Provides direct access to the Thing objects. + things: AssociationProxy[List["Thing"]] = association_proxy( + "thing_associations", "thing" + ) + # --- Table Arguments --- __table_args__ = Index("ix_aquifersystem_name", "name") From 0d3f807c131aa902e62facc475818ce97b241a80 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 14 Nov 2025 16:41:50 -0700 Subject: [PATCH 27/91] refactor: clarify proxy purpose doc statement --- db/aquifer_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index c0710b687..fd121cd7b 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -68,7 +68,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): ) # --- Association Proxies --- - # Many-To-Many Proxy: Provides direct access to the Thing objects. + # Proxy to directly access Things (wells) associated with this AquiferSystem. things: AssociationProxy[List["Thing"]] = association_proxy( "thing_associations", "thing" ) From c3393d9abc41b8568fc53b936140dbc7aa6b94f3 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:10:37 -0700 Subject: [PATCH 28/91] feat: add ThingAquiferAssociation model and update Thing relationships - Created ThingAquiferAssociation model to manage many-to-many relationships between Thing and AquiferSystem - Updated Thing model to include aquifer_associations relationship for linking to aquifer systems --- db/thing.py | 9 +++++++ db/thing_aquifer_association.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 db/thing_aquifer_association.py diff --git a/db/thing.py b/db/thing.py index df0ef8611..03902c866 100644 --- a/db/thing.py +++ b/db/thing.py @@ -39,6 +39,7 @@ from db.sensor import Sensor from db.contact import Contact from db.group import Group, GroupThingAssociation + from db.thing_aquifer_association import ThingAquiferAssociation class Thing( @@ -262,6 +263,14 @@ class Thing( passive_deletes=True, ) + # One-To-Many: A Thing can be associated with many AquiferSystems via the ThingAquiferAssociation join table. + aquifer_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py new file mode 100644 index 000000000..5f14696ca --- /dev/null +++ b/db/thing_aquifer_association.py @@ -0,0 +1,42 @@ +""" +SQLAlchemy model for the ThingAquiferAssociation table. + +This table is a join table (or "association object") whose purpose is to manage +the many-to-many relationship between a Thing and an AquiferSystem. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey + +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.aquifer_system import AquiferSystem + + +class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the association of a Thing to an AquiferSystem. This is an Association Object. + """ + + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="CASCADE"), nullable=False + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="aquifer_associations" + ) + + # Many-To-One: This association links to one AquiferSystem. + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="thing_associations" + ) From af092ddac74090a56caceb5995e832f705a6d3a6 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:15:55 -0700 Subject: [PATCH 29/91] feat: add `aqufiers` proxy to Thing model Added an `aquifers` proxy to the Thing model to directly access the AqufierSystems associated with a particular Thing. --- db/thing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/db/thing.py b/db/thing.py index 03902c866..246d9536b 100644 --- a/db/thing.py +++ b/db/thing.py @@ -39,6 +39,7 @@ from db.sensor import Sensor from db.contact import Contact from db.group import Group, GroupThingAssociation + from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation @@ -296,6 +297,11 @@ class Thing( "group_associations", "group" ) + # Proxy to directly access AquiferSystems associated with this Thing + aquifers: AssociationProxy[List["AquiferSystem"]] = association_proxy( + "aquifer_associations", "aquifer_system" + ) + # Full-text search vector search_vector = Column(TSVectorType("name", "well_construction_notes")) From e330df46125468efa93fcff0d7f578520ce90a04 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 16 Nov 2025 13:55:39 -0700 Subject: [PATCH 30/91] feat: Link WellScreen to AquiferSystem This commit addresses the need to handle scenarios where multiple well screens may target different aquifers. Added the aquifer_system_id foreign key to the WellScreen model. This creates a direct, physical link between a screened interval and the specific hydrogeologic unit it monitors. --- db/thing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/thing.py b/db/thing.py index 246d9536b..c43064fe0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -346,6 +346,9 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): thing_id: Mapped[int] = mapped_column( ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="SET NULL"), nullable=True + ) screen_depth_top: Mapped[float] = mapped_column( info={"unit": "feet below ground surface"}, nullable=True ) @@ -363,6 +366,10 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): # Many-To-One: A WellScreen belongs to one Thing. thing: Mapped["Thing"] = relationship("Thing", back_populates="screens") + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="well_screens", passive_deletes=True + ) + class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): """ From 04a28be8849cdb4e77591816595cbb4fae53f2a5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 11:36:55 -0700 Subject: [PATCH 31/91] feat: add GeologicFormation and ThingFormationAssociation models with Thing relationship updates - Introduced GeologicFormation model as a master reference for geologic formations - Created ThingFormationAssociation model to manage many-to-many relationships between Thing and GeologicFormation, including depth interval fields - Updated Thing model to include formation_associations relationship and association proxy for direct access to related geologic formations --- db/geologic_formation.py | 73 +++++++++++++++++++++++++++++++ db/thing.py | 15 +++++++ db/thing_formation_association.py | 60 +++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 db/geologic_formation.py create mode 100644 db/thing_formation_association.py diff --git a/db/geologic_formation.py b/db/geologic_formation.py new file mode 100644 index 000000000..a0119d50c --- /dev/null +++ b/db/geologic_formation.py @@ -0,0 +1,73 @@ +""" +SQLAlchemy model for the GeologicFormation table. + +This table is a master reference table for geologic formations. Its purpose is to store definitions and descriptions +of various geologic formations that can be referenced by other tables in the database. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +if TYPE_CHECKING: + from db.thing import Thing + from db.thing_formation_association import ThingFormationAssociation + + +class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + # TODO: Should `name` use a controlled vocabulary? + name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", + ) + # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. + code: Mapped[str] = lexicon_term( + nullable=True, + unique=True, + comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the geologic formation, its characteristics, and its significance.", + ) + # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. + lithology: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary for the primary, dominant rock type" + "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + nullable=True, + comment="A spatial representation of the geologic formation's extent.", + ) + + # --- Relationships --- + # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the + # ThingFormationAssociation join table. + thing_associations: Mapped[List["ThingFormationAssociation"]] = relationship( + "ThingFormationAssociation", + back_populates="geologic_formation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # --- Association Proxies --- + # Provides direct access to Things (wells) that penetrate this formation. + things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing") + + # --- Table Arguments --- + __table_args__ = ( + Index("ix_geologicformation_name", "name"), + Index("ix_geologicformation_code", "code"), + ) diff --git a/db/thing.py b/db/thing.py index c43064fe0..b5a2a284a 100644 --- a/db/thing.py +++ b/db/thing.py @@ -41,6 +41,8 @@ from db.group import Group, GroupThingAssociation from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation + from db.geologic_formation import GeologicFormation + from db.thing_formation_association import ThingFormationAssociation class Thing( @@ -272,6 +274,14 @@ class Thing( passive_deletes=True, ) + # Many-To-Many: A Thing can penetrate many GeologicFormations. + formation_associations: Mapped[List["ThingFormationAssociation"]] = relationship( + "ThingFormationAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" @@ -302,6 +312,11 @@ class Thing( "aquifer_associations", "aquifer_system" ) + # Proxy to directly access the GeologicFormations penetrated by this Thing. + formations: AssociationProxy[List["GeologicFormation"]] = association_proxy( + "formation_associations", "geologic_formation" + ) + # Full-text search vector search_vector = Column(TSVectorType("name", "well_construction_notes")) diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py new file mode 100644 index 000000000..c5f0f9592 --- /dev/null +++ b/db/thing_formation_association.py @@ -0,0 +1,60 @@ +""" +SQLAlchemy model for the ThingFormationAssociation table. + +This table is an association object that creates a many-to-many relationship between a Thing (well) and a +GeologicFormation. It stores the lithology for a well, detailing the depth intervals for each formation it penetrates. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.geologic_formation import GeologicFormation + + +class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + This is a= join table (Association Object). It represents the association of a Thing to a + GeologicFormation at a specific depth interval. + """ + + # --- Foreign Keys --- + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), + nullable=False, + comment="The foreign key linking this record to the `Thing` table." + "Deleting a `Thing` will cascade and delete its formation log.", + ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), + nullable=True, + comment="The foreign key linking this record to the `GeologicFormation` table." + "This is set to `SET NULL` on delete, as deleting a formation definition (a rare admin action)" + "should not delete the historical fact that a well had a pick at this depth.", + ) + + # Depth interval fields + top_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the top of the geologic formation, as measured from ground surface.", + ) + bottom_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the bottom of the geologic formation, as measured from ground surface.", + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="formation_associations" + ) + + # Many-To-One: This association links to one GeologicFormation. + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="thing_associations" + ) From 477247a66165e9987b8709806d0d7326d4e65926 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 12:35:55 -0700 Subject: [PATCH 32/91] feat(lexicon): add "formation_code" category and associated terms to lexicon --- core/lexicon.json | 302 ++++++++++++++++++++++++++++++++++++++- db/geologic_formation.py | 5 +- 2 files changed, 302 insertions(+), 5 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 5b45332d8..6d980ab43 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -52,7 +52,8 @@ {"name": "well_purpose", "description": null}, {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, - {"name": "well_pump_type", "description": null} + {"name": "well_pump_type", "description": null}, + {"name": "formation_code", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -675,7 +676,7 @@ {"categories": ["monitoring_frequency"], "term": "Biannual", "definition": "Location is monitored twice a year."}, {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, - {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."} + {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, @@ -693,6 +694,301 @@ {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, - {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"} + {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"}, + {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, +{"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, +{"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, +{"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, +{"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, +{"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, +{"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, +{"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, +{"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, +{"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, +{"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, +{"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, +{"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, +{"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, +{"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, +{"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, +{"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, +{"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, +{"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, +{"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, +{"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, +{"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, +{"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, +{"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, +{"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, +{"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, +{"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, +{"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, +{"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, +{"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, +{"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, +{"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, +{"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, +{"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, +{"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, +{"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, +{"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, +{"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, +{"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, +{"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, +{"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, +{"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, +{"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, +{"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, +{"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, +{"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, +{"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, +{"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, +{"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, +{"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, +{"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, +{"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, +{"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, +{"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, +{"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, +{"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, +{"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, +{"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, +{"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, +{"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, +{"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, +{"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, +{"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, +{"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, +{"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, +{"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, +{"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, +{"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, +{"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, +{"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, +{"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, +{"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, +{"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, +{"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, +{"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, +{"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, +{"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, +{"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, +{"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, +{"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, +{"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, +{"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, +{"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, +{"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, +{"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, +{"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, +{"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, +{"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, +{"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, +{"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, +{"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, +{"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, +{"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, +{"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, +{"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, +{"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, +{"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, +{"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, +{"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, +{"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, +{"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, +{"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, +{"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, +{"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, +{"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, +{"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, +{"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, +{"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, +{"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, +{"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, +{"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, +{"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, +{"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, +{"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, +{"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, +{"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, +{"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, +{"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, +{"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, +{"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, +{"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, +{"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, +{"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, +{"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, +{"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, +{"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, +{"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, +{"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, +{"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, +{"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, +{"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, +{"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, +{"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, +{"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, +{"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, +{"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, +{"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, +{"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, +{"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, +{"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, +{"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, +{"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, +{"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, +{"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, +{"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, +{"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, +{"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, +{"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, +{"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, +{"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, +{"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, +{"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, +{"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, +{"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, +{"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, +{"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, +{"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, +{"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, +{"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, +{"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, +{"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, +{"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, +{"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, +{"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, +{"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, +{"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, +{"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, +{"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, +{"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, +{"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, +{"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, +{"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, +{"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, +{"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, +{"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, +{"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, +{"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, +{"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, +{"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, +{"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, +{"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, +{"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, +{"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, +{"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, +{"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, +{"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, +{"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, +{"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, +{"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, +{"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, +{"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, +{"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, +{"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, +{"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, +{"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, +{"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, +{"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, +{"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, +{"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, +{"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, +{"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, +{"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, +{"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, +{"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, +{"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, +{"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, +{"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, +{"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, +{"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, +{"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, +{"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, +{"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, +{"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, +{"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, +{"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, +{"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, +{"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, +{"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, +{"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, +{"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, +{"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, +{"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, +{"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, +{"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, +{"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, +{"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, +{"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, +{"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, +{"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, +{"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, +{"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, +{"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, +{"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, +{"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, +{"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, +{"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, +{"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, +{"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, +{"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, +{"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, +{"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, +{"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, +{"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, +{"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, +{"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, +{"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, +{"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, +{"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, +{"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, +{"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, +{"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, +{"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, +{"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, +{"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, +{"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, +{"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, +{"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, +{"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, +{"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, +{"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, +{"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, +{"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, +{"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, +{"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, +{"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, +{"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, +{"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, +{"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, +{"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, +{"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, +{"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, +{"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"} ] } \ No newline at end of file diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a0119d50c..ff6ce0a85 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -30,7 +30,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", ) # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. - code: Mapped[str] = lexicon_term( + formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').", @@ -41,7 +41,8 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): comment="A detailed description of the geologic formation, its characteristics, and its significance.", ) # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. - lithology: Mapped[str] = lexicon_term( + # This should be implemented after AMMP reviews and cleans up their formation terms and codes. + lithology: Mapped[str] = mapped_column( nullable=True, comment="A controlled vocabulary for the primary, dominant rock type" "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", From d359fa1639879f241d5536236434993ba69f99fa Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 12:40:23 -0700 Subject: [PATCH 33/91] rafactor(model): Remove TODO about adding formation_code values to lexicon --- db/geologic_formation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index ff6ce0a85..35332aac3 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -29,7 +29,6 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): unique=True, comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", ) - # TODO: Implement controlled vocabulary for `code` using the `LU_Formation` table from NMAquifer. formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, From fc8132768a6cec563034795f89099e59305679ff Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 16:53:15 -0700 Subject: [PATCH 34/91] refactor(lexicon): refine terms associated with "geographic_scale". --- core/lexicon.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 6d980ab43..0461f530d 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -692,9 +692,10 @@ {"categories": ["aquifer_type"], "term": "Unconfined, single", "definition": "Unconfined single aquifer"}, {"categories": ["aquifer_type"], "term": "Unconfined, multiple", "definition": "Unconfined multiple aquifers"}, {"categories": ["aquifer_type"], "term": "Unsaturated", "definition": "Unsaturated (dry)"}, - {"categories": ["geographic_scale"], "term": "Major", "definition": "Major"}, - {"categories": ["geographic_scale"], "term": "Intermediate", "definition": "Regional"}, - {"categories": ["geographic_scale"], "term": "Local", "definition": "Local"}, + {"categories": ["geographic_scale"], "term": "Major", "definition": "Major aquifers of national significance"}, + {"categories": ["geographic_scale"], "term": "Regional", "definition": "Important aquifers serving regions"}, + {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, + {"categories": ["geographic_scale"], "term": "Minor", "definition": "Limited extent or yield"}, {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, {"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, {"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, From b01d172179ed75e1f0a389323c19d52ef09af2f1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 17:00:29 -0700 Subject: [PATCH 35/91] refactor(model): Replace hardcoded srid with SRID_WGS84 from constants Replace the hardcoded `srid` in the `aquifer_system` and `geologic_formation` models with the SRID_WGS84 from constants. --- db/aquifer_system.py | 4 +++- db/geologic_formation.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index fd121cd7b..95066c2e8 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -14,6 +14,8 @@ from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term +from constants import SRID_WGS84 + if TYPE_CHECKING: from db.thing import WellScreen, ThingAquiferAssociation, Thing @@ -41,7 +43,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", ) boundary: Mapped[Geometry] = mapped_column( - Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), nullable=True, comment="A spatial representation of the aquifer system's boundary.", ) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index 35332aac3..d916f402a 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -15,6 +15,8 @@ from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term +from constants import SRID_WGS84 + if TYPE_CHECKING: from db.thing import Thing from db.thing_formation_association import ThingFormationAssociation @@ -47,7 +49,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", ) boundary: Mapped[Geometry] = mapped_column( - Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True), + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), nullable=True, comment="A spatial representation of the geologic formation's extent.", ) From f35e61d61faf664d756f6505e86410dbb7c28463 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 17 Nov 2025 19:42:54 -0700 Subject: [PATCH 36/91] refactor(model): Add geologic_formation foreign key to WellScreen model. --- db/geologic_formation.py | 6 +++++- db/thing.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index d916f402a..a29c20010 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -18,7 +18,7 @@ from constants import SRID_WGS84 if TYPE_CHECKING: - from db.thing import Thing + from db.thing import Thing, WellScreen from db.thing_formation_association import ThingFormationAssociation @@ -63,6 +63,10 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): cascade="all, delete-orphan", passive_deletes=True, ) + # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. + screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", back_populates="geologic_formation", passive_deletes=True + ) # --- Association Proxies --- # Provides direct access to Things (wells) that penetrate this formation. diff --git a/db/thing.py b/db/thing.py index b5a2a284a..25dcb0969 100644 --- a/db/thing.py +++ b/db/thing.py @@ -364,6 +364,9 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): aquifer_system_id: Mapped[int] = mapped_column( ForeignKey("aquifer_system.id", ondelete="SET NULL"), nullable=True ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), nullable=True + ) screen_depth_top: Mapped[float] = mapped_column( info={"unit": "feet below ground surface"}, nullable=True ) From 390d6f39aacc4e92fc1064111daa3b8d1a56025b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 09:10:01 -0700 Subject: [PATCH 37/91] refactor(model): enhance relationships for `ThingAquiferAssociation` and `ThingFormationAssociation` models Add `lazy="joined"` to `ThingAquiferAssociation` and `ThingFormationAssociation` models to eagerly load relationships. --- db/thing_aquifer_association.py | 4 ++-- db/thing_formation_association.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index 5f14696ca..d5fb2881a 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -33,10 +33,10 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationship Definitions --- # Many-To-One: This association links to one Thing. thing: Mapped["Thing"] = relationship( - "Thing", back_populates="aquifer_associations" + "Thing", back_populates="aquifer_associations", lazy="joined" ) # Many-To-One: This association links to one AquiferSystem. aquifer_system: Mapped["AquiferSystem"] = relationship( - "AquiferSystem", back_populates="thing_associations" + "AquiferSystem", back_populates="thing_associations", lazy="joined" ) diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py index c5f0f9592..8904fa089 100644 --- a/db/thing_formation_association.py +++ b/db/thing_formation_association.py @@ -51,10 +51,10 @@ class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationship Definitions --- # Many-To-One: This association links to one Thing. thing: Mapped["Thing"] = relationship( - "Thing", back_populates="formation_associations" + "Thing", back_populates="formation_associations", lazy="joined" ) # Many-To-One: This association links to one GeologicFormation. geologic_formation: Mapped["GeologicFormation"] = relationship( - "GeologicFormation", back_populates="thing_associations" + "GeologicFormation", back_populates="thing_associations", lazy="joined" ) From c738afba94d576c945d35ac1cec9807497996762 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 15:51:39 -0700 Subject: [PATCH 38/91] refactor(model): refactor `lithology` field to a lexicon_term - Refactor `lithology` field to a lexicon_term - Add lithology values to lexicon --- core/lexicon.json | 667 ++++++++++++++++++++++----------------- db/geologic_formation.py | 16 +- 2 files changed, 381 insertions(+), 302 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 0461f530d..15479d6b4 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -53,7 +53,8 @@ {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, {"name": "well_pump_type", "description": null}, - {"name": "formation_code", "description": null} + {"name": "formation_code", "description": null}, + {"name": "lithology", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -697,299 +698,375 @@ {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, {"categories": ["geographic_scale"], "term": "Minor", "definition": "Limited extent or yield"}, {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, -{"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, -{"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, -{"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, -{"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, -{"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, -{"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, -{"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, -{"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, -{"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, -{"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, -{"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, -{"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, -{"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, -{"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, -{"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, -{"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, -{"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, -{"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, -{"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, -{"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, -{"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, -{"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, -{"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, -{"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, -{"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, -{"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, -{"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, -{"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, -{"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, -{"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, -{"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, -{"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, -{"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, -{"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, -{"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, -{"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, -{"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, -{"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, -{"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, -{"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, -{"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, -{"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, -{"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, -{"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, -{"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, -{"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, -{"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, -{"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, -{"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, -{"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, -{"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, -{"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, -{"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, -{"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, -{"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, -{"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, -{"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, -{"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, -{"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, -{"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, -{"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, -{"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, -{"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, -{"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, -{"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, -{"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, -{"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, -{"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, -{"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, -{"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, -{"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, -{"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, -{"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, -{"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, -{"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, -{"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, -{"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, -{"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, -{"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, -{"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, -{"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, -{"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, -{"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, -{"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, -{"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, -{"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, -{"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, -{"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, -{"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, -{"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, -{"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, -{"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, -{"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, -{"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, -{"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, -{"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, -{"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, -{"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, -{"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, -{"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, -{"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, -{"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, -{"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, -{"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, -{"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, -{"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, -{"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, -{"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, -{"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, -{"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, -{"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, -{"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, -{"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, -{"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, -{"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, -{"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, -{"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, -{"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, -{"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, -{"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, -{"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, -{"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, -{"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, -{"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, -{"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, -{"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, -{"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, -{"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, -{"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, -{"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, -{"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, -{"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, -{"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, -{"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, -{"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, -{"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, -{"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, -{"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, -{"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, -{"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, -{"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, -{"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, -{"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, -{"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, -{"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, -{"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, -{"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, -{"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, -{"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, -{"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, -{"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, -{"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, -{"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, -{"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, -{"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, -{"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, -{"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, -{"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, -{"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, -{"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, -{"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, -{"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, -{"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, -{"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, -{"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, -{"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, -{"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, -{"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, -{"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, -{"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, -{"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, -{"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, -{"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, -{"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, -{"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, -{"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, -{"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, -{"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, -{"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, -{"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, -{"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, -{"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, -{"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, -{"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, -{"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, -{"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, -{"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, -{"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, -{"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, -{"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, -{"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, -{"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, -{"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, -{"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, -{"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, -{"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, -{"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, -{"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, -{"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, -{"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, -{"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, -{"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, -{"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, -{"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, -{"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, -{"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, -{"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, -{"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, -{"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, -{"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, -{"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, -{"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, -{"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, -{"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, -{"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, -{"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, -{"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, -{"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, -{"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, -{"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, -{"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, -{"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, -{"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, -{"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, -{"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, -{"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, -{"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, -{"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, -{"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, -{"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, -{"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, -{"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, -{"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, -{"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, -{"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, -{"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, -{"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, -{"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, -{"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, -{"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, -{"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, -{"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, -{"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, -{"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, -{"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, -{"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, -{"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, -{"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, -{"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, -{"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, -{"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, -{"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, -{"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, -{"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, -{"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, -{"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, -{"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, -{"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, -{"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, -{"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, -{"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, -{"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, -{"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, -{"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, -{"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, -{"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, -{"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, -{"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, -{"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, -{"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, -{"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"} + {"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, + {"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, + {"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, + {"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, + {"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, + {"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, + {"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, + {"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, + {"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, + {"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, + {"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, + {"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, + {"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, + {"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, + {"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, + {"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, + {"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, + {"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, + {"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, + {"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, + {"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, + {"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, + {"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, + {"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, + {"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, + {"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, + {"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, + {"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, + {"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, + {"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, + {"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, + {"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, + {"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, + {"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, + {"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, + {"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, + {"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, + {"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, + {"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, + {"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, + {"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, + {"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, + {"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, + {"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, + {"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, + {"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, + {"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, + {"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, + {"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, + {"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, + {"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, + {"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, + {"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, + {"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, + {"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, + {"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, + {"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, + {"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, + {"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, + {"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, + {"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, + {"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, + {"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, + {"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, + {"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, + {"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, + {"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, + {"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, + {"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, + {"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, + {"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, + {"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, + {"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, + {"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, + {"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, + {"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, + {"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, + {"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, + {"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, + {"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, + {"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, + {"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, + {"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, + {"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, + {"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, + {"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, + {"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, + {"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, + {"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, + {"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, + {"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, + {"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, + {"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, + {"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, + {"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, + {"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, + {"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, + {"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, + {"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, + {"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, + {"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, + {"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, + {"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, + {"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, + {"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, + {"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, + {"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, + {"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, + {"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, + {"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, + {"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, + {"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, + {"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, + {"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, + {"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, + {"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, + {"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, + {"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, + {"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, + {"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, + {"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, + {"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, + {"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, + {"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, + {"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, + {"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, + {"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, + {"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, + {"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, + {"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, + {"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, + {"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, + {"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, + {"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, + {"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, + {"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, + {"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, + {"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, + {"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, + {"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, + {"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, + {"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, + {"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, + {"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, + {"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, + {"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, + {"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, + {"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, + {"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, + {"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, + {"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, + {"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, + {"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, + {"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, + {"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, + {"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, + {"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, + {"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, + {"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, + {"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, + {"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, + {"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, + {"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, + {"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, + {"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, + {"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, + {"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, + {"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, + {"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, + {"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, + {"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, + {"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, + {"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, + {"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, + {"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, + {"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, + {"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, + {"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, + {"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, + {"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, + {"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, + {"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, + {"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, + {"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, + {"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, + {"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, + {"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, + {"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, + {"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, + {"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, + {"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, + {"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, + {"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, + {"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, + {"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, + {"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, + {"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, + {"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, + {"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, + {"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, + {"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, + {"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, + {"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, + {"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, + {"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, + {"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, + {"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, + {"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, + {"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, + {"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, + {"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, + {"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, + {"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, + {"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, + {"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, + {"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, + {"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, + {"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, + {"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, + {"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, + {"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, + {"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, + {"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, + {"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, + {"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, + {"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, + {"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, + {"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, + {"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, + {"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, + {"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, + {"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, + {"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"}, + {"categories": ["lithology"],"term": "Alluvium","definition": "Alluvium"}, + {"categories": ["lithology"],"term": "Anhydrite","definition": "Anhydrite"}, + {"categories": ["lithology"],"term": "Arkose","definition": "Arkose"}, + {"categories": ["lithology"],"term": "Boulders","definition": "Boulders"}, + {"categories": ["lithology"],"term": "Boulders, silt and clay","definition": "Boulders, silt and clay"}, + {"categories": ["lithology"],"term": "Boulders and sand","definition": "Boulders and sand"}, + {"categories": ["lithology"],"term": "Bentonite","definition": "Bentonite"}, + {"categories": ["lithology"],"term": "Breccia","definition": "Breccia"}, + {"categories": ["lithology"],"term": "Basalt","definition": "Basalt"}, + {"categories": ["lithology"],"term": "Conglomerate","definition": "Conglomerate"}, + {"categories": ["lithology"],"term": "Chalk","definition": "Chalk"}, + {"categories": ["lithology"],"term": "Chert","definition": "Chert"}, + {"categories": ["lithology"],"term": "Clay","definition": "Clay"}, + {"categories": ["lithology"],"term": "Caliche","definition": "Caliche"}, + {"categories": ["lithology"],"term": "Calcite","definition": "Calcite"}, + {"categories": ["lithology"],"term": "Clay, some sand","definition": "Clay, some sand"}, + {"categories": ["lithology"],"term": "Claystone","definition": "Claystone"}, + {"categories": ["lithology"],"term": "Coal","definition": "Coal"}, + {"categories": ["lithology"],"term": "Cobbles","definition": "Cobbles"}, + {"categories": ["lithology"],"term": "Cobbles, silt and clay","definition": "Cobbles, silt and clay"}, + {"categories": ["lithology"],"term": "Cobbles and sand","definition": "Cobbles and sand"}, + {"categories": ["lithology"],"term": "Dolomite","definition": "Dolomite"}, + {"categories": ["lithology"],"term": "Dolomite and shale","definition": "Dolomite and shale"}, + {"categories": ["lithology"],"term": "Evaporite","definition": "Evaporite"}, + {"categories": ["lithology"],"term": "Gneiss","definition": "Gneiss"}, + {"categories": ["lithology"],"term": "Gypsum","definition": "Gypsum"}, + {"categories": ["lithology"],"term": "Graywacke","definition": "Graywacke"}, + {"categories": ["lithology"],"term": "Gravel and clay","definition": "Gravel and clay"}, + {"categories": ["lithology"],"term": "Gravel, cemented","definition": "Gravel, cemented"}, + {"categories": ["lithology"],"term": "Gravel, sand and silt","definition": "Gravel, sand and silt"}, + {"categories": ["lithology"],"term": "Granite, gneiss","definition": "Granite, gneiss"}, + {"categories": ["lithology"],"term": "Granite","definition": "Granite"}, + {"categories": ["lithology"],"term": "Gravel, silt and clay","definition": "Gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Gravel","definition": "Gravel"}, + {"categories": ["lithology"],"term": "Igneous undifferentiated","definition": "Igneous undifferentiated"}, + {"categories": ["lithology"],"term": "Lignite","definition": "Lignite"}, + {"categories": ["lithology"],"term": "Limestone and dolomite","definition": "Limestone and dolomite"}, + {"categories": ["lithology"],"term": "Limestone and shale","definition": "Limestone and shale"}, + {"categories": ["lithology"],"term": "Limestone","definition": "Limestone"}, + {"categories": ["lithology"],"term": "Marl","definition": "Marl"}, + {"categories": ["lithology"],"term": "Mudstone","definition": "Mudstone"}, + {"categories": ["lithology"],"term": "Metamorphic undifferentiated","definition": "Metamorphic undifferentiated"}, + {"categories": ["lithology"],"term": "Marlstone","definition": "Marlstone"}, + {"categories": ["lithology"],"term": "No Recovery","definition": "No Recovery"}, + {"categories": ["lithology"],"term": "Peat","definition": "Peat"}, + {"categories": ["lithology"],"term": "Quartzite","definition": "Quartzite"}, + {"categories": ["lithology"],"term": "Rhyolite","definition": "Rhyolite"}, + {"categories": ["lithology"],"term": "Sand","definition": "Sand"}, + {"categories": ["lithology"],"term": "Schist","definition": "Schist"}, + {"categories": ["lithology"],"term": "Sand and clay","definition": "Sand and clay"}, + {"categories": ["lithology"],"term": "Sand and gravel","definition": "Sand and gravel"}, + {"categories": ["lithology"],"term": "Sandstone and shale","definition": "Sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand and silt","definition": "Sand and silt"}, + {"categories": ["lithology"],"term": "Sand, gravel and clay","definition": "Sand, gravel and clay"}, + {"categories": ["lithology"],"term": "Shale","definition": "Shale"}, + {"categories": ["lithology"],"term": "Silt","definition": "Silt"}, + {"categories": ["lithology"],"term": "Siltstone and shale","definition": "Siltstone and shale"}, + {"categories": ["lithology"],"term": "Siltstone","definition": "Siltstone"}, + {"categories": ["lithology"],"term": "Slate","definition": "Slate"}, + {"categories": ["lithology"],"term": "Sand, some clay","definition": "Sand, some clay"}, + {"categories": ["lithology"],"term": "Sandstone","definition": "Sandstone"}, + {"categories": ["lithology"],"term": "Silt and clay","definition": "Silt and clay"}, + {"categories": ["lithology"],"term": "Travertine","definition": "Travertine"}, + {"categories": ["lithology"],"term": "Tuff","definition": "Tuff"}, + {"categories": ["lithology"],"term": "Volcanic undifferentiated","definition": "Volcanic undifferentiated"}, + {"categories": ["lithology"],"term": "Clay, yellow","definition": "Clay, yellow"}, + {"categories": ["lithology"],"term": "Clay, red","definition": "Clay, red"}, + {"categories": ["lithology"],"term": "Surficial sediment","definition": "Surficial sediment"}, + {"categories": ["lithology"],"term": "Limestone and sandstone, interbedded","definition": "Limestone and sandstone, interbedded"}, + {"categories": ["lithology"],"term": "Gravel and boulders","definition": "Gravel and boulders"}, + {"categories": ["lithology"],"term": "Sand, silt and gravel","definition": "Sand, silt and gravel"}, + {"categories": ["lithology"],"term": "Sand, gravel, silt and clay","definition": "Sand, gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Andesite","definition": "Andesite"}, + {"categories": ["lithology"],"term": "Ignesous, intrusive, undifferentiated","definition": "Ignesous, intrusive, undifferentiated"}, + {"categories": ["lithology"],"term": "Limestone, sandstone and shale","definition": "Limestone, sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand, silt and clay","definition": "Sand, silt and clay"} ] } \ No newline at end of file diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a29c20010..d8a7c7515 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -25,12 +25,14 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} - # TODO: Should `name` use a controlled vocabulary? - name: Mapped[str] = mapped_column( - nullable=False, - unique=True, - comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", - ) + # TODO: Let the API map formation codes to names using a formations.json file that can be periodically updated + # from the authoritative source (.e.g USGS). A placeholder `formations.json` file had been added to the `core` + # directory. + # name: Mapped[str] = mapped_column( + # nullable=False, + # unique=True, + # comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", + # ) formation_code: Mapped[str] = lexicon_term( nullable=True, unique=True, @@ -43,7 +45,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): ) # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. # This should be implemented after AMMP reviews and cleans up their formation terms and codes. - lithology: Mapped[str] = mapped_column( + lithology: Mapped[str] = lexicon_term( nullable=True, comment="A controlled vocabulary for the primary, dominant rock type" "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", From 4d55177c3ee2e65fbffc3f5f80e890ee8e7aae56 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 15:58:30 -0700 Subject: [PATCH 39/91] feat(core): create placeholder `formations.json` This is a placeholder file to allow the API to map formation names to codes. It can be periodically updated from the authoritative source (.e.g USGS). --- core/formations.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 core/formations.json diff --git a/core/formations.json b/core/formations.json new file mode 100644 index 000000000..e69de29bb From 8362f9c5a3f695380f453cf543663b94b2c8fa92 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 17:07:12 -0700 Subject: [PATCH 40/91] refactor: add aquifer and geology related models to `db/__init__.py` --- db/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/__init__.py b/db/__init__.py index efb23a418..68c94d598 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -40,6 +40,10 @@ from db.status_history import * from db.thing import * from db.transducer import * +from db.aquifer_system import * +from db.geologic_formation import * +from db.thing_aquifer_association import * +from db.thing_formation_association import * from sqlalchemy import ( func, From 39aeefa597bfd91ecedf72ac58ea899f0f857d7b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 17:12:15 -0700 Subject: [PATCH 41/91] feat: add aquifer and geology related enums. --- core/enums.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/enums.py b/core/enums.py index 5833d97bc..0da1d2eef 100644 --- a/core/enums.py +++ b/core/enums.py @@ -70,4 +70,7 @@ ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +AquiferType: type[Enum] = build_enum_from_lexicon_category("aquifer_type") +GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale") +Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") # ============= EOF ============================================= From 47b8415f05b7a4f37dfb3d5d4f3deeb383b81148 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Tue, 18 Nov 2025 19:59:14 -0700 Subject: [PATCH 42/91] feat(schemas): add response schemas for aquifer systems and geologic formations Add comprehensive response schemas for AquiferSystem and GeologicFormation models. Integrate these schemas into Thing/Well response models to provide complete geological context. Changes: - Add AquiferSystemResponse with name, type, scale, and boundary fields - Add GeologicFormationResponse with formation code, lithology, and boundary - Add ThingFormationAssociationResponse with depth interval data - Update WellResponse to include aquifers and formations lists - Add field validators to convert ORM association objects to response schemas - Update WellScreenResponse to include aquifer_system and geologic_formation - Update CreateWellScreen and UpdateWellScreen with geological associations --- schemas/aquifer_system.py | 32 ++++++++++++++++++++++++ schemas/geologic_formation.py | 46 +++++++++++++++++++++++++++++++++++ schemas/thing.py | 32 ++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 schemas/aquifer_system.py create mode 100644 schemas/geologic_formation.py diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py new file mode 100644 index 000000000..5f5b3ed4d --- /dev/null +++ b/schemas/aquifer_system.py @@ -0,0 +1,32 @@ +from typing import List + +from pydantic import BaseModel + +from schemas import BaseResponseModel + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str + coordinates: ( + List[float] + | List[List[float]] + | List[List[List[float]]] + | List[List[List[List[float]]]] + ) + + +class AquiferSystemResponse(BaseResponseModel): + """ + Response schema for aquifer system details. + """ + + name: str + description: str | None = None + aquifer_type: str + geographic_scale: str + boundary: GeoJSONGeometry | None = None diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py new file mode 100644 index 000000000..2f5f0432d --- /dev/null +++ b/schemas/geologic_formation.py @@ -0,0 +1,46 @@ +from typing import List + +from pydantic import BaseModel + +from schemas import BaseResponseModel + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str + coordinates: ( + List[float] + | List[List[float]] + | List[List[List[float]]] + | List[List[List[List[float]]]] + ) + + +class GeologicFormationResponse(BaseResponseModel): + """ + Response schema for geologic formation details. + """ + + formation_code: str | None = None + description: str | None = None + lithology: str | None = None + boundary: GeoJSONGeometry | None = None + + +class ThingFormationAssociationResponse(BaseResponseModel): + """ + Response schema for the association between a Thing and a GeologicFormation. + Includes depth interval information. + """ + + thing_id: int + geologic_formation_id: int | None = None + geologic_formation: GeologicFormationResponse | None = None + top_depth: float + top_depth_unit: str = "ft" + bottom_depth: float + bottom_depth_unit: str = "ft" diff --git a/schemas/thing.py b/schemas/thing.py index d87fd299f..36f3eb757 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -27,6 +27,11 @@ ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse +from schemas.aquifer_system import AquiferSystemResponse +from schemas.geologic_formation import ( + GeologicFormationResponse, + ThingFormationAssociationResponse, +) # -------- VALIDATE ---------- @@ -121,6 +126,8 @@ class CreateWellScreen(BaseCreateModel): """ thing_id: int + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float = Field(gt=0, description="Screen depth bottom in feet") screen_depth_top: float = Field(gt=0, description="Screen depth top in feet") screen_type: ScreenType | None = None @@ -166,6 +173,8 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" + aquifers: list[AquiferSystemResponse] = [] + formations: list[ThingFormationAssociationResponse] = [] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -186,6 +195,23 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("aquifers", mode="before") + def populate_aquifers(cls, aquifers): + """Convert aquifer association objects to aquifer system objects.""" + if aquifers is not None: + # Handle if aquifers are already AquiferSystem objects + if hasattr(aquifers[0] if aquifers else None, "aquifer_system"): + return [assoc.aquifer_system for assoc in aquifers] + return aquifers or [] + + @field_validator("formations", mode="before") + def populate_formations(cls, formations): + """Convert formation association objects to response objects.""" + if formations is not None: + # formations should already be ThingFormationAssociation objects + return formations + return [] + class SpringResponse(BaseThingResponse): """ @@ -222,6 +248,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse + aquifer_system_id: int | None = None + aquifer_system: AquiferSystemResponse | None = None + geologic_formation_id: int | None = None + geologic_formation: GeologicFormationResponse | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -295,6 +325,8 @@ class UpdateThingIdLink(BaseUpdateModel): class UpdateWellScreen(BaseUpdateModel): + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float | None = None screen_depth_top: float | None = None screen_description: str | None = None From b4c8beb604bf53c36a69da80c315932544fa3f8b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 11:50:19 -0700 Subject: [PATCH 43/91] fix: fix artifacts from merge conflicts --- db/thing.py | 8 +- tests/features/environment.py | 3 +- .../steps/well-additional-information.py | 74 +++++++++++-------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/db/thing.py b/db/thing.py index 53cb88df0..7ee3c9cba 100644 --- a/db/thing.py +++ b/db/thing.py @@ -28,7 +28,7 @@ ReleaseMixin, ) from db.permission_history import PermissionHistoryMixin -from services.util import retrieve_latest_polymorphic_table_record +from services.util import retrieve_latest_polymorphic_history_table_record from db.status_history import StatusHistoryMixin from db.measuring_point_history import MeasuringPointHistory from db.data_provenance import DataProvenanceMixin @@ -403,7 +403,7 @@ def allow_water_level_samples(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Water Level Sample" ) return permission_record.permission_allowed if permission_record else None @@ -413,7 +413,7 @@ def allow_water_chemistry_samples(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Water Chemistry Sample" ) return permission_record.permission_allowed if permission_record else None @@ -423,7 +423,7 @@ def allow_datalogger_installation(self): """ Returns the current permissions for the Thing. """ - permission_record = retrieve_latest_polymorphic_table_record( + permission_record = retrieve_latest_polymorphic_history_table_record( self, "permission_history", "Datalogger Installation" ) return permission_record.permission_allowed if permission_record else None diff --git a/tests/features/environment.py b/tests/features/environment.py index c56bb0e86..289d0b0f6 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -448,7 +448,7 @@ def before_all(context): add_permission_history( context, session, - contact_id=contact.id, + contact_id=context.objects["contacts"][0].id, permission_type="Datalogger Installation", permission_allowed=True, start_date=datetime(2025, 1, 1).date(), @@ -618,7 +618,6 @@ def before_all(context): ) session.commit() - session.refresh(well_1) # the following needs to be refreshed to get all the new relationships session.refresh(well_1) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 2baa32db6..1dc5c3518 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -12,7 +12,7 @@ def step_impl(context): assert "allow_water_level_samples" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Water Level Sample" + context.objects["wells"][0], "permission_history", "Water Level Sample" ) assert ( context.water_well_data["allow_water_level_samples"] @@ -25,7 +25,7 @@ def step_impl(context): assert "allow_water_chemistry_samples" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Water Chemistry Sample" + context.objects["wells"][0], "permission_history", "Water Chemistry Sample" ) assert ( @@ -41,11 +41,11 @@ def step_impl(context): assert "allow_datalogger_installation" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.well, "permission_history", "Datalogger Installation" + context.objects["wells"][0], "permission_history", "Datalogger Installation" ) assert ( - context.water_well_data["permissions"]["allow_data_logger_installation"] + context.water_well_data["allow_datalogger_installation"] == permission_record.permission_allowed ) @@ -57,10 +57,10 @@ def step_impl(context): @then("the response should include the completion date of the well") def step_impl(context): - assert "completion_date" in context.water_well_data - assert context.water_well_data[ - "completion_date" - ] == context.well.completion_date.strftime("%Y-%m-%d") + assert "well_completion_date" in context.water_well_data + assert context.water_well_data["well_completion_date"] == context.objects["wells"][ + 0 + ].well_completion_date.strftime("%Y-%m-%d") # TODO: needs to be added to model, schemas, test data @@ -69,22 +69,25 @@ def step_impl(context): assert "completion_info_source" in context.water_well_data assert ( context.water_well_data["completion_info_source"] - == context.well.completion_info_source + == context.objects["wells"][0].completion_info_source ) @then("the response should include the driller name") def step_impl(context): - assert "driller_name" in context.water_well_data - assert context.water_well_data["driller_name"] == context.well.driller_name + assert "well_driller_name" in context.water_well_data + assert ( + context.water_well_data["well_driller_name"] + == context.objects["wells"][0].well_driller_name + ) @then("the response should include the construction method") def step_impl(context): - assert "construction_method" in context.water_well_data + assert "well_construction_method" in context.water_well_data assert ( - context.water_well_data["construction_method"] - == context.well.construction_method + context.water_well_data["well_construction_method"] + == context.objects["wells"][0].well_construction_method ) @@ -94,7 +97,7 @@ def step_impl(context): assert "construction_info_source" in context.water_well_data assert ( context.water_well_data["construction_info_source"] - == context.well.construction_info_source + == context.objects["wells"][0].construction_info_source ) @@ -105,11 +108,14 @@ def step_impl(context): @then("the response should include the casing diameter in inches") def step_impl(context): - assert "casing_diameter" in context.water_well_data - assert "casing_diameter_unit" in context.water_well_data + assert "well_casing_diameter" in context.water_well_data + assert "well_casing_diameter_unit" in context.water_well_data - assert context.water_well_data["casing_diameter"] == context.well.casing_diameter - assert context.water_well_data["casing_diameter_unit"] == "in" + assert ( + context.water_well_data["well_casing_diameter"] + == context.objects["wells"][0].well_casing_diameter + ) + assert context.water_well_data["well_casing_diameter_unit"] == "in" @then("the response should include the casing depth in feet below ground surface") @@ -118,7 +124,8 @@ def step_impl(context): assert "well_casing_depth_unit" in context.water_well_data assert ( - context.water_well_data["well_casing_depth"] == context.well.well_casing_depth + context.water_well_data["well_casing_depth"] + == context.objects["wells"][0].well_casing_depth ) assert context.water_well_data["well_casing_depth_unit"] == "ft" @@ -127,16 +134,18 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data - assert ( - context.water_well_data["well_casing_materials"] - == context.well.well_casing_materials + assert sorted(context.water_well_data["well_casing_materials"]) == sorted( + [m.material for m in context.objects["wells"][0].well_casing_materials] ) @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.water_well_data - assert context.water_well_data["well_pump_type"] == context.well.well_pump_type + assert ( + context.water_well_data["well_pump_type"] + == context.objects["wells"][0].well_pump_type + ) @then("the response should include the well pump depth in feet (new field)") @@ -144,7 +153,10 @@ def step_impl(context): assert "well_pump_depth" in context.water_well_data assert "well_pump_depth_unit" in context.water_well_data - assert context.water_well_data["well_pump_depth"] == context.well.well_pump_depth + assert ( + context.water_well_data["well_pump_depth"] + == context.objects["wells"][0].well_pump_depth + ) assert context.water_well_data["well_pump_depth_unit"] == "ft" @@ -155,7 +167,7 @@ def step_impl(context): assert "is_suitable_for_datalogger" in context.water_well_data assert ( context.water_well_data["is_suitable_for_datalogger"] - == context.well.is_suitable_for_datalogger + == context.objects["wells"][0].is_suitable_for_datalogger ) @@ -170,7 +182,7 @@ def step_impl(context): ) def step_impl(context): assert "formation" in context.water_well_data - assert context.water_well_data["formation"] == context.well.formation + assert context.water_well_data["formation"] == context.objects["wells"][0].formation # TODO: needs to be added to model, schemas, test data, lexicon @@ -180,7 +192,8 @@ def step_impl(context): def step_impl(context): assert "aquifer_class_code" in context.water_well_data assert ( - context.water_well_data["aquifer_class_code"] == context.well.aquifer_class_code + context.water_well_data["aquifer_class_code"] + == context.objects["wells"][0].aquifer_class_code ) @@ -191,4 +204,7 @@ def step_impl(context): ) def step_impl(context): assert "aquifer_type" in context.water_well_data - assert context.water_well_data["aquifer_type"] == context.well.aquifer_type + assert ( + context.water_well_data["aquifer_type"] + == context.objects["wells"][0].aquifer_type + ) From d1149f500e7fb2de6618b94970c8b9393824bc42 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:20:22 -0700 Subject: [PATCH 44/91] refactor: rename `ThingFormationAssociation` usages to `ThingGeologicFormationAssociation` --- db/geologic_formation.py | 16 +++++++++------- db/thing.py | 14 ++++++++------ db/thing_formation_association.py | 4 ++-- schemas/geologic_formation.py | 2 +- schemas/thing.py | 6 +++--- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index d8a7c7515..af12d7a8a 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from db.thing import Thing, WellScreen - from db.thing_formation_association import ThingFormationAssociation + from db.thing_formation_association import ThingGeologicFormationAssociation class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): @@ -58,12 +58,14 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): # --- Relationships --- # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the - # ThingFormationAssociation join table. - thing_associations: Mapped[List["ThingFormationAssociation"]] = relationship( - "ThingFormationAssociation", - back_populates="geologic_formation", - cascade="all, delete-orphan", - passive_deletes=True, + # ThingGeologicFormationAssociation join table. + thing_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="geologic_formation", + cascade="all, delete-orphan", + passive_deletes=True, + ) ) # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. screens: Mapped[List["WellScreen"]] = relationship( diff --git a/db/thing.py b/db/thing.py index 25dcb0969..106af773e 100644 --- a/db/thing.py +++ b/db/thing.py @@ -42,7 +42,7 @@ from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation from db.geologic_formation import GeologicFormation - from db.thing_formation_association import ThingFormationAssociation + from db.thing_formation_association import ThingGeologicFormationAssociation class Thing( @@ -275,11 +275,13 @@ class Thing( ) # Many-To-Many: A Thing can penetrate many GeologicFormations. - formation_associations: Mapped[List["ThingFormationAssociation"]] = relationship( - "ThingFormationAssociation", - back_populates="thing", - cascade="all, delete-orphan", - passive_deletes=True, + formation_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) ) # --- Association Proxies --- diff --git a/db/thing_formation_association.py b/db/thing_formation_association.py index 8904fa089..0707df269 100644 --- a/db/thing_formation_association.py +++ b/db/thing_formation_association.py @@ -1,5 +1,5 @@ """ -SQLAlchemy model for the ThingFormationAssociation table. +SQLAlchemy model for the ThingGeologicFormationAssociation table. This table is an association object that creates a many-to-many relationship between a Thing (well) and a GeologicFormation. It stores the lithology for a well, detailing the depth intervals for each formation it penetrates. @@ -17,7 +17,7 @@ from db.geologic_formation import GeologicFormation -class ThingFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): +class ThingGeologicFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): """ This is a= join table (Association Object). It represents the association of a Thing to a GeologicFormation at a specific depth interval. diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 2f5f0432d..f6b3083d3 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -31,7 +31,7 @@ class GeologicFormationResponse(BaseResponseModel): boundary: GeoJSONGeometry | None = None -class ThingFormationAssociationResponse(BaseResponseModel): +class ThingGeologicFormationAssociationResponse(BaseResponseModel): """ Response schema for the association between a Thing and a GeologicFormation. Includes depth interval information. diff --git a/schemas/thing.py b/schemas/thing.py index 36f3eb757..398630e66 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -30,7 +30,7 @@ from schemas.aquifer_system import AquiferSystemResponse from schemas.geologic_formation import ( GeologicFormationResponse, - ThingFormationAssociationResponse, + ThingGeologicFormationAssociationResponse, ) @@ -174,7 +174,7 @@ class WellResponse(BaseThingResponse): well_pump_depth: float | None well_pump_depth_unit: str = "ft" aquifers: list[AquiferSystemResponse] = [] - formations: list[ThingFormationAssociationResponse] = [] + formations: list[ThingGeologicFormationAssociationResponse] = [] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -208,7 +208,7 @@ def populate_aquifers(cls, aquifers): def populate_formations(cls, formations): """Convert formation association objects to response objects.""" if formations is not None: - # formations should already be ThingFormationAssociation objects + # formations should already be ThingGeologicFormationAssociation objects return formations return [] From c85f9714d2619db92b2cfdc84e28eddb9199c7a5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:22:39 -0700 Subject: [PATCH 45/91] refactor: rename `ThingFormationAssociation` file`ThingGeologicFormationAssociation` --- db/__init__.py | 2 +- db/geologic_formation.py | 4 +++- db/thing.py | 4 +++- ...association.py => thing_geologic_formation_association.py} | 0 4 files changed, 7 insertions(+), 3 deletions(-) rename db/{thing_formation_association.py => thing_geologic_formation_association.py} (100%) diff --git a/db/__init__.py b/db/__init__.py index 68c94d598..ef14f7c06 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -43,7 +43,7 @@ from db.aquifer_system import * from db.geologic_formation import * from db.thing_aquifer_association import * -from db.thing_formation_association import * +from db.thing_geologic_formation_association import * from sqlalchemy import ( func, diff --git a/db/geologic_formation.py b/db/geologic_formation.py index af12d7a8a..130ed8d45 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -19,7 +19,9 @@ if TYPE_CHECKING: from db.thing import Thing, WellScreen - from db.thing_formation_association import ThingGeologicFormationAssociation + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): diff --git a/db/thing.py b/db/thing.py index 106af773e..12a6ae5ed 100644 --- a/db/thing.py +++ b/db/thing.py @@ -42,7 +42,9 @@ from db.aquifer_system import AquiferSystem from db.thing_aquifer_association import ThingAquiferAssociation from db.geologic_formation import GeologicFormation - from db.thing_formation_association import ThingGeologicFormationAssociation + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) class Thing( diff --git a/db/thing_formation_association.py b/db/thing_geologic_formation_association.py similarity index 100% rename from db/thing_formation_association.py rename to db/thing_geologic_formation_association.py From 853e4507743fe9ed25a14b9fa757894e2c323eb5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 19 Nov 2025 12:49:14 -0700 Subject: [PATCH 46/91] refactor(schema): remove aquifer and formation field validators from `schemas/thing.py` --- schemas/thing.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 398630e66..8a63c431f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -195,23 +195,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("aquifers", mode="before") - def populate_aquifers(cls, aquifers): - """Convert aquifer association objects to aquifer system objects.""" - if aquifers is not None: - # Handle if aquifers are already AquiferSystem objects - if hasattr(aquifers[0] if aquifers else None, "aquifer_system"): - return [assoc.aquifer_system for assoc in aquifers] - return aquifers or [] - - @field_validator("formations", mode="before") - def populate_formations(cls, formations): - """Convert formation association objects to response objects.""" - if formations is not None: - # formations should already be ThingGeologicFormationAssociation objects - return formations - return [] - class SpringResponse(BaseThingResponse): """ From 2c33a7900cacab2c01910f2ecc4fd5e9a288be4f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 13:28:16 -0700 Subject: [PATCH 47/91] feat: implement data source tests for well additional information --- db/thing.py | 12 ++++++++++++ run_bdd.sh | 4 ++-- schemas/thing.py | 2 ++ tests/features/environment.py | 18 ++++++++++++++++++ .../steps/well-additional-information.py | 13 +++++++------ 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/db/thing.py b/db/thing.py index 7ee3c9cba..1ed1cbbc3 100644 --- a/db/thing.py +++ b/db/thing.py @@ -398,6 +398,18 @@ def measuring_point_description(self) -> str | None: def well_depth_source(self) -> str | None: return self._get_data_provenance_attribute("well_depth", "origin_source") + @property + def well_completion_date_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_completion_date", "origin_source" + ) + + @property + def well_construction_method_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_construction_method", "origin_source" + ) + @property def allow_water_level_samples(self): """ diff --git a/run_bdd.sh b/run_bdd.sh index 29d0be47d..cd05769e4 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -66,7 +66,7 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} # tests/features/thing-query-parameters.feature #uv run behave tests/features/well-inventory-csv.feature -uv run behave tests/features/well-additional-information.feature --capture -# uv run behave tests/features --tags="@backend and @production" --capture +# uv run behave tests/features/well-additional-information.feature --capture +uv run behave tests/features --tags="@backend and @production" --capture echo "✅ BDD test run complete." diff --git a/schemas/thing.py b/schemas/thing.py index e980e4e76..50f56e7c4 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -223,8 +223,10 @@ class WellResponse(BaseThingResponse): well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None well_completion_date: PastOrTodayDate | None + well_completion_date_source: str | None well_driller_name: str | None well_construction_method: WellConstructionMethod | None + well_construction_method_source: str | None well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" diff --git a/tests/features/environment.py b/tests/features/environment.py index 289d0b0f6..2dc410517 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -606,6 +606,24 @@ def before_all(context): origin_source="Other", ) + well_completion_date_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_completion_date", + origin_source="Data Portal", + ) + + well_construction_method_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_construction_method", + origin_source="Data Portal", + ) + for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 1dc5c3518..eaf83e1ce 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -66,10 +66,11 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): - assert "completion_info_source" in context.water_well_data + assert "well_completion_date_source" in context.water_well_data + assert ( - context.water_well_data["completion_info_source"] - == context.objects["wells"][0].completion_info_source + context.water_well_data["well_completion_date_source"] + == context.objects["wells"][0].well_completion_date_source ) @@ -94,10 +95,10 @@ def step_impl(context): # TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): - assert "construction_info_source" in context.water_well_data + assert "well_construction_method_source" in context.water_well_data assert ( - context.water_well_data["construction_info_source"] - == context.objects["wells"][0].construction_info_source + context.water_well_data["well_construction_method_source"] + == context.objects["wells"][0].well_construction_method_source ) From 8ef2592b0994fe5598840e610752bde4b4ff0d47 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 13:29:22 -0700 Subject: [PATCH 48/91] fix: remove outdated notes --- tests/features/steps/well-additional-information.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index eaf83e1ce..d7d3b768c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -63,7 +63,6 @@ def step_impl(context): ].well_completion_date.strftime("%Y-%m-%d") -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the completion information") def step_impl(context): assert "well_completion_date_source" in context.water_well_data @@ -92,7 +91,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the source of the construction information") def step_impl(context): assert "well_construction_method_source" in context.water_well_data @@ -131,7 +129,6 @@ def step_impl(context): assert context.water_well_data["well_casing_depth_unit"] == "ft" -# TODO: needs to be added to model, schemas, test data @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data From ced10b947629847e94cbd12c8b45225eeae94041 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Nov 2025 16:56:06 -0700 Subject: [PATCH 49/91] feat: transfer water level sample permissions --- schemas/thing.py | 2 +- transfers/permissions_transfer.py | 52 +++++++++++++++++++++++++++++++ transfers/transfer.py | 4 +++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 transfers/permissions_transfer.py diff --git a/schemas/thing.py b/schemas/thing.py index 50f56e7c4..a8cac2523 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -130,7 +130,7 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_height: float = Field( ge=0, description="Measuring point height in feet" ) - measuring_point_description: str | None + measuring_point_description: str | None = None notes: list[CreateNote] | None = None diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py new file mode 100644 index 000000000..e1f7bd069 --- /dev/null +++ b/transfers/permissions_transfer.py @@ -0,0 +1,52 @@ +from sqlalchemy.orm import Session +from datetime import datetime + +from db import Thing, PermissionHistory +from transfers.util import read_csv, logger, replace_nans + + +def transfer_permissions(session: Session): + """ + The transferred wells and contacts need to be queried to know who gave + permission to which well since contact_id is required for PermissionHistory + """ + wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) + wdf = replace_nans(wdf) + + transferred_wells = ( + session.query(Thing).filter(Thing.thing_type == "water well").all() + ) + + for well in transferred_wells: + if len(well.contacts) == 0: + logger.critical( + f"Well {well.name} has no associated contacts; skipping permission transfer." + ) + continue + else: + # Assuming the first contact is the relevant one + contact_id = well.contacts[0].id + + allow_water_level_samples = wdf.loc[ + wdf["PointID"] == well.name, "MonitorOK" + ].values + if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + try: + permission_allowed = bool(allow_water_level_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Level Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Level Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + continue + session.commit() diff --git a/transfers/transfer.py b/transfers/transfer.py index 77275ed35..2b576a4b2 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -36,6 +36,7 @@ transfer_wells, transfer_wellscreens, ) +from transfers.permissions_transfer import transfer_permissions from transfers.asset_transfer import transfer_assets from transfers.util import timeit, timeit_direct @@ -124,6 +125,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING ASSETS") timeit_direct(transfer_assets, sess) + message("TRANSFERRING PERMISSIONS") + results = timeit_direct(transfer_permissions, sess) + def transfer_debugging(sess, limit=100): message("STARTING TRANSFER DEBUG", new_line_at_top=False) From e48b71eea39c1ccb83bdf0d2581feb1296fc49b0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 08:13:43 -0700 Subject: [PATCH 50/91] feat: transfer permissions from legacy db --- transfers/permissions_transfer.py | 55 ++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index e1f7bd069..d07c2cbe2 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -48,5 +48,58 @@ def transfer_permissions(session: Session): except Exception as e: logger.error(f"Error transferring permission for well {well.name}: {e}") session.rollback() - continue + pass + + allow_water_chemistry_samples = wdf.loc[ + wdf["PointID"] == well.name, "SampleOK" + ].values + if ( + len(allow_water_chemistry_samples) > 0 + and allow_water_chemistry_samples is not None + ): + try: + permission_allowed = bool(allow_water_chemistry_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Chemistry Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Chemistry Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + allow_datalogger_installation = wdf.loc[ + wdf["PointID"] == well.name, "OpenWellLoggerOK" + ].values + if ( + len(allow_datalogger_installation) > 0 + and allow_datalogger_installation is not None + ): + try: + permission_allowed = bool(allow_datalogger_installation[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Datalogger Installation", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Datalogger Installation permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + session.commit() From 171b41b6da02d35b850aa9deec50e9daa4977040 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 10:44:20 -0700 Subject: [PATCH 51/91] feat: transfer well construction information --- schemas/thing.py | 3 +++ transfers/well_transfer.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/schemas/thing.py b/schemas/thing.py index a8cac2523..2438c50c4 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -132,6 +132,9 @@ class CreateWell(CreateBaseThing, ValidateWell): ) measuring_point_description: str | None = None notes: list[CreateNote] | None = None + well_completion_date: PastOrTodayDate | None = None + well_driller_name: str | None = None + well_construction_method: WellConstructionMethod | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ee54d0216..d2cb4b186 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -257,6 +257,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None notes=( [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=row.ConstructionMethod, ) CreateWell.model_validate(data) From a9cec8ad1c9359351517ccf3820b23f4f88a7468 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 12:32:13 -0700 Subject: [PATCH 52/91] WIP: transfers for well additional info --- schemas/thing.py | 2 ++ transfers/well_transfer.py | 53 ++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 2438c50c4..b23ae7d4f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -133,8 +133,10 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_description: str | None = None notes: list[CreateNote] | None = None well_completion_date: PastOrTodayDate | None = None + well_completion_date_source: str | None = None well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None + well_construction_method_source: str | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index d2cb4b186..e41cf789a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -36,6 +36,7 @@ StatusHistory, MonitoringFrequencyHistory, MeasuringPointHistory, + DataProvenance, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -280,6 +281,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "well_casing_materials", "measuring_point_height", "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" @@ -288,17 +291,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_data.pop("notes") well = Thing(**well_data) session.add(well) - # logger.info(f"Created well for {row.PointID}") - - # flush well to access its ID for status_history - # session.flush() - - # session.commit() - # session.refresh(well) - # if notes: - # for ni in notes: - # nn = well.add_note(ni['content'], ni['note_type']) - # session.add(nn) if well_purposes: for wp in well_purposes: @@ -350,11 +342,44 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None for dp in data_provenances: session.add(dp) + if not isna(row.CompletionSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_completion_date", + origin_source=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.CompletionSource}" + ), + ) + session.add(dp) + + if not isna(row.DataSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_construction_method", + origin_source=lexicon_mapper.map_value( + f"LU_DataSource:{row.DataSource}" + ), + ) + session.add(dp) + + if not isna(row.DepthSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_depth", + origin_source=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.DepthSource}" + ), + ) + session.add(dp) + """ - Developer's note + Developer's note - It's not clear when the measuring point from NM_Aquifer was - determined, so I'm setting start_date to the day of the transfer + It's not clear when the measuring point from NM_Aquifer was + determined, so I'm setting start_date to the day of the transfer """ measuring_point_history = MeasuringPointHistory( thing_id=well.id, From 722425fae83c6adce0cca7eb9f9651d0a867d4c1 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 13:07:31 -0700 Subject: [PATCH 53/91] feat: implement origin_type --- db/data_provenance.py | 8 ++++++-- db/thing.py | 4 ++-- tests/features/environment.py | 10 ++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/db/data_provenance.py b/db/data_provenance.py index 06c468c8d..14cfdc5aa 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -53,9 +53,13 @@ class DataProvenance(AutoBaseMixin, ReleaseMixin, Base): ) # Values from the following NMAquifer tables are included as `origin_source` terms in the lexicon: # 'LU_DataSource', 'LU_Depth_CompletionSource'. - origin_source: Mapped[str] = lexicon_term( + origin_type: Mapped[str] = lexicon_term( nullable=True, - comment="Indicates the origin source of the data (e.g'Driller's Log', 'Well Report'.", + comment="Indicates the type of origin the data (e.g'Driller's Log', 'Well Report'.", + ) + origin_source: Mapped[str] = mapped_column( + nullable=True, + comment="The specific source of the data (e.g., 'J. Brown Thesis, \"I like APIs\", Pomona College, 1994').", ) # Values from the following NMAquifer tables are included as `collection_method` terms in the lexicon: # 'LU_AltitudeMethod','LU_CoordinateMethod'. diff --git a/db/thing.py b/db/thing.py index 1ed1cbbc3..b9c2c54a0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -396,12 +396,12 @@ def measuring_point_description(self) -> str | None: @property def well_depth_source(self) -> str | None: - return self._get_data_provenance_attribute("well_depth", "origin_source") + return self._get_data_provenance_attribute("well_depth", "origin_type") @property def well_completion_date_source(self) -> str | None: return self._get_data_provenance_attribute( - "well_completion_date", "origin_source" + "well_completion_date", "origin_type" ) @property diff --git a/tests/features/environment.py b/tests/features/environment.py index 2dc410517..ac223b530 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -385,7 +385,8 @@ def add_data_provenance( target_id, target_table, field_name, - origin_source, + origin_type=None, + origin_source=None, collection_method=None, accuracy_value=None, accuracy_unit=None, @@ -395,6 +396,7 @@ def add_data_provenance( collection_method=collection_method, target_id=target_id, target_table=target_table, + origin_type=origin_type, origin_source=origin_source, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, @@ -603,7 +605,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_depth", - origin_source="Other", + origin_type="Other", ) well_completion_date_source = add_data_provenance( @@ -612,7 +614,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_completion_date", - origin_source="Data Portal", + origin_type="Data Portal", ) well_construction_method_source = add_data_provenance( @@ -621,7 +623,7 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_construction_method", - origin_source="Data Portal", + origin_source="Jacob's 2013 Thesis", ) for purpose in ["Domestic", "Irrigation"]: From 75265f335802dd267342d07387049b030ca9c18c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 13:56:23 -0700 Subject: [PATCH 54/91] feat: transfer well source information --- transfers/util.py | 1 - transfers/well_transfer.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index cbf0f2b17..c4b423250 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -369,7 +369,6 @@ def make_location_data_provenance( target_id=location.id, target_table="location", field_name="point", - origin_source=None, collection_method=coordinate_method, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index e41cf789a..2c0073f03 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -260,7 +260,13 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None ), well_completion_date=row.CompletionDate, well_driller_name=row.DrillerName, - well_construction_method=row.ConstructionMethod, + well_construction_method=( + lexicon_mapper.map_value( + f"LU_ConstructionMethod:{row.ConstructionMethod}" + ) + if not isna(row.ConstructionMethod) + else None + ), ) CreateWell.model_validate(data) @@ -347,7 +353,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_completion_date", - origin_source=lexicon_mapper.map_value( + origin_type=lexicon_mapper.map_value( f"LU_Depth_CompletionSource:{row.CompletionSource}" ), ) @@ -358,9 +364,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_construction_method", - origin_source=lexicon_mapper.map_value( - f"LU_DataSource:{row.DataSource}" - ), + origin_source=row.DataSource, ) session.add(dp) @@ -369,7 +373,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None target_id=well.id, target_table="thing", field_name="well_depth", - origin_source=lexicon_mapper.map_value( + origin_type=lexicon_mapper.map_value( f"LU_Depth_CompletionSource:{row.DepthSource}" ), ) From 2b90f869e6cdb633750f9f41cee9011a8471e9df Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 14:00:30 -0700 Subject: [PATCH 55/91] feat: add Quemado Mutual Water and Sewage Works Association to organizations for contacts --- core/lexicon.json | 1 + transfers/data/owners_organization_mapper.json | 1 + 2 files changed, 2 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index ba4fd8f7e..46033eee5 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -574,6 +574,7 @@ {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, + {"categories": ["organization"], "term": "Quemado Municipal Water & SWA", "definition": "Quemado Municipal Water & SWA"}, {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, diff --git a/transfers/data/owners_organization_mapper.json b/transfers/data/owners_organization_mapper.json index 5ce45a8bf..b4f29bd7b 100644 --- a/transfers/data/owners_organization_mapper.json +++ b/transfers/data/owners_organization_mapper.json @@ -89,6 +89,7 @@ "Pecos Trail Inn": "Pecos Trail Inn", "Pelican Spa": "Pelican Spa", "Pistachio Tree Ranch": "Pistachio Tree Ranch", + "Quemado Mutual Water and Sewage Works Association": "Quemado Municipal Water & SWA", "Rancho Encantado": "Rancho Encantado", "Rancho San Lucas": "Rancho San Lucas", "Rancho San Marcos": "Rancho San Marcos", From cf72461be26acf99f7b682f8b5d30cd709fa6169 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Nov 2025 14:20:26 -0700 Subject: [PATCH 56/91] fix: remove erroneous or check --- transfers/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/util.py b/transfers/util.py index c4b423250..9b2afc84c 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -275,7 +275,7 @@ def make_location_data_provenance( ) -> list[DataProvenance]: provenance_records = [] - if row.AltitudeAccuracy or row.CoordinateAccuracy: + if row.AltitudeAccuracy: provenance = DataProvenance( target_id=location.id, target_table="location", From 58b92f884bd7165c029dd15be7635911070f7175 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 21 Nov 2025 11:53:18 -0700 Subject: [PATCH 57/91] feat: transfer well pump type parse the construction notes field to get pump types. this can be more sophisticated, but that's a future problem. for now just check for some basic pump type string patterns --- core/lexicon.json | 4 ++-- schemas/thing.py | 1 + transfers/well_transfer.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 46033eee5..21666bccd 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -714,9 +714,9 @@ {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, - {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, + {"categories": ["well_pump_type"], "term": "Hand", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} diff --git a/schemas/thing.py b/schemas/thing.py index b23ae7d4f..48c055cd8 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -137,6 +137,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None + well_pump_type: WellPumpType | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 2c0073f03..346d91629 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -118,6 +118,21 @@ def _extract_casing_materials(row) -> list[str]: return materials +def _extract_well_pump_type(row) -> str | None: + construction_notes = row.ConstructionNotes.lower() + if "pump" in construction_notes: + if "submersible" in construction_notes: + return "Submersible" + elif "jet" in construction_notes: + return "Jet" + elif "line shaft" in construction_notes or "lineshaft" in construction_notes: + return "Line Shaft" + elif "hand" in construction_notes: + return "Hand" + else: + return None + + def get_wells_to_transfer( sess: Session, flags: dict = None ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -237,6 +252,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) + well_pump_type = ( + _extract_well_pump_type(row) if row.ConstructionNotes else None + ) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation @@ -267,6 +285,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None if not isna(row.ConstructionMethod) else None ), + well_pump_type=well_pump_type, ) CreateWell.model_validate(data) From 4e9876e1f23369283ba9588c834c6d54b5744ef6 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 21 Nov 2025 11:55:58 -0700 Subject: [PATCH 58/91] fix: change Air-rotary to Air-Rotary to correspond with lu table This can be updated later on but should be "Air-Rotary" for the transfer --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index 21666bccd..5821330b5 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -81,7 +81,7 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"}, {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, From e9639bff8de328c9cf9cd213b3c0366c6119a6f5 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Sun, 23 Nov 2025 14:46:08 -0700 Subject: [PATCH 59/91] feat(model): Add `AquiferType` model and rename `aquifer_type` to `primary_aquifer_type` - Add AquiferType model to store multiple aquifer characteristics per association - Rename AquiferSystem.aquifer_type to primary_aquifer_type for clarity - Update ThingAquiferAssociation with aquifer_types relationship - Update related schemas to use primary_type This enables capturing wells that encounter multiple aquifer characteristics (e.g., both fractured and confined) without compound naming. --- db/__init__.py | 1 + db/aquifer_system.py | 9 +++-- db/aquifer_type.py | 58 +++++++++++++++++++++++++++++++++ db/thing_aquifer_association.py | 8 +++++ schemas/aquifer_system.py | 16 ++++++++- 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 db/aquifer_type.py diff --git a/db/__init__.py b/db/__init__.py index ef14f7c06..65ab4cd32 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -44,6 +44,7 @@ from db.geologic_formation import * from db.thing_aquifer_association import * from db.thing_geologic_formation_association import * +from db.aquifer_type import * from sqlalchemy import ( func, diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 95066c2e8..b72e91949 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from db.thing import WellScreen, ThingAquiferAssociation, Thing + from db.aquifer_type import AquiferType class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): @@ -34,9 +35,9 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): comment="A detailed description of the aquifer system, its characteristics, and its significance.", ) # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. - aquifer_type: Mapped[str] = lexicon_term( + primary_aquifer_type: Mapped[str] = lexicon_term( nullable=False, - comment="A controlled vocabulary field to classify the aquifer's hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + comment="A controlled vocabulary field to classify the aquifer's primary hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", ) geographic_scale: Mapped[str] = lexicon_term( nullable=False, @@ -74,6 +75,10 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): things: AssociationProxy[List["Thing"]] = association_proxy( "thing_associations", "thing" ) + # Proxy to directly access all AquiferTypes associated with this AquiferSystem. + aquifer_types: AssociationProxy[List["AquiferType"]] = association_proxy( + "thing_associations", "aquifer_types" + ) # --- Table Arguments --- __table_args__ = Index("ix_aquifersystem_name", "name") diff --git a/db/aquifer_type.py b/db/aquifer_type.py new file mode 100644 index 000000000..32900d801 --- /dev/null +++ b/db/aquifer_type.py @@ -0,0 +1,58 @@ +""" +SQLAlchemy model for the AquiferType table. + +This table stores the specific aquifer characteristics/types associated with +a Thing-AquiferSystem relationship. It allows capturing that a single aquifer +can have multiple characteristics simultaneously. + +Example: + A well in the "Ogallala" aquifer might tap portions that are both + "Fractured" AND "Confined". This would create: + - One AquiferSystem: "Ogallala" + - One ThingAquiferAssociation: linking well to Ogallala + - Two AquiferType records: "Fractured" and "Confined" +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.thing_aquifer_association import ThingAquiferAssociation + + +class AquiferType(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the specific aquifer types/characteristics for a + Thing-AquiferSystem association. + + This allows modeling the fact that: + - A single aquifer can have multiple characteristics + - Different wells may tap different characteristics of the same aquifer + - Characteristics are attributes of the relationship, not the aquifer itself + + Fields from WellData CSV: + - AquiferType: May contain multiple codes (e.g., "FC" = Fractured + Confined) + - Each code becomes a separate AquiferType record + """ + + # --- Columns --- + thing_aquifer_association_id: Mapped[int] = mapped_column( + ForeignKey("thing_aquifer_association.id", ondelete="CASCADE"), + nullable=False, + comment="Links to the Thing-Aquifer association this type describes.", + ) + aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="Controlled vocabulary for aquifer hydrologic properties. " + "Examples: 'Unconfined', 'Confined', 'Perched', 'Fractured', 'Unconsolidated'.", + ) + + # --- Relationships --- + # Many-to-One: Multiple aquifer types can belong to one association + thing_aquifer_association: Mapped["ThingAquiferAssociation"] = relationship( + "ThingAquiferAssociation", back_populates="aquifer_types" + ) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index d5fb2881a..414f09269 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from db.thing import Thing from db.aquifer_system import AquiferSystem + from db.aquifer_type import AquiferType class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): @@ -40,3 +41,10 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): aquifer_system: Mapped["AquiferSystem"] = relationship( "AquiferSystem", back_populates="thing_associations", lazy="joined" ) + # One-To-Many: An association can have multiple aquifer types. + aquifer_types: Mapped[list["AquiferType"]] = relationship( + "AquiferType", + back_populates="thing_aquifer_association", + cascade="all, delete-orphan", + passive_deletes=True, + ) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 5f5b3ed4d..4b4448211 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -5,6 +5,20 @@ from schemas import BaseResponseModel +# ------ CREATE ---------- +class CreateAquiferSystem(BaseModel): + """ + Schema for creating an aquifer system. + Used during data transfer and API creation. + """ + + name: str + description: str | None = None + primary_aquifer_type: str + geographic_scale: str + boundary: str | None = None + + # ------ RESPONSE ---------- class GeoJSONGeometry(BaseModel): """ @@ -27,6 +41,6 @@ class AquiferSystemResponse(BaseResponseModel): name: str description: str | None = None - aquifer_type: str + primary_aquifer_type: str geographic_scale: str boundary: GeoJSONGeometry | None = None From ba2b2964550ffc7def0d12dd5cd7fa495f0b5401 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 11:13:21 -0700 Subject: [PATCH 60/91] feat(model): add eager loading Add eager loading to aquifer and geology relationships to the `thing` model. --- db/aquifer_system.py | 2 +- db/thing.py | 2 ++ db/thing_aquifer_association.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index b72e91949..53086f983 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -37,7 +37,7 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. primary_aquifer_type: Mapped[str] = lexicon_term( nullable=False, - comment="A controlled vocabulary field to classify the aquifer's primary hydrologic properties (e.g., 'Unconfined', 'Confined', 'Perched').", + comment="A controlled vocabulary field to classify the aquifer system as a whole (e.g., 'Unconfined', 'Confined', 'Perched').", ) geographic_scale: Mapped[str] = lexicon_term( nullable=False, diff --git a/db/thing.py b/db/thing.py index 0bb361294..d2592f7ac 100644 --- a/db/thing.py +++ b/db/thing.py @@ -290,6 +290,7 @@ class Thing( back_populates="thing", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) # Many-To-Many: A Thing can penetrate many GeologicFormations. @@ -299,6 +300,7 @@ class Thing( back_populates="thing", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) ) diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index 414f09269..cca5758a9 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -47,4 +47,5 @@ class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): back_populates="thing_aquifer_association", cascade="all, delete-orphan", passive_deletes=True, + lazy="joined", ) From 910d5b31bbb942894a66236a09a97bdd0b8e2a8f Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 24 Nov 2025 11:45:51 -0700 Subject: [PATCH 61/91] feat(schema): refactor GeoJSON responses for `aquifer_system` and `geologic_formation` --- schemas/aquifer_system.py | 22 +++++++++++++--------- schemas/geologic_formation.py | 24 ++++++++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 4b4448211..6c077aa90 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -25,16 +25,11 @@ class GeoJSONGeometry(BaseModel): Geometry schema for GeoJSON response. """ - type: str - coordinates: ( - List[float] - | List[List[float]] - | List[List[List[float]]] - | List[List[List[List[float]]]] - ) + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] -class AquiferSystemResponse(BaseResponseModel): +class GeoJSONProperties(BaseResponseModel): """ Response schema for aquifer system details. """ @@ -43,4 +38,13 @@ class AquiferSystemResponse(BaseResponseModel): description: str | None = None primary_aquifer_type: str geographic_scale: str - boundary: GeoJSONGeometry | None = None + + +class AquiferSystemGeoJSONResponse(BaseModel): + """ + Response schema for aquifer system details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index f6b3083d3..d42e48389 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -11,16 +11,11 @@ class GeoJSONGeometry(BaseModel): Geometry schema for GeoJSON response. """ - type: str - coordinates: ( - List[float] - | List[List[float]] - | List[List[List[float]]] - | List[List[List[List[float]]]] - ) + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] -class GeologicFormationResponse(BaseResponseModel): +class GeoJSONProperties(BaseResponseModel): """ Response schema for geologic formation details. """ @@ -28,7 +23,16 @@ class GeologicFormationResponse(BaseResponseModel): formation_code: str | None = None description: str | None = None lithology: str | None = None - boundary: GeoJSONGeometry | None = None + + +class GeologicFormationGeoJSONResponse(BaseModel): + """ + Response schema for geologic formation details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties class ThingGeologicFormationAssociationResponse(BaseResponseModel): @@ -39,7 +43,7 @@ class ThingGeologicFormationAssociationResponse(BaseResponseModel): thing_id: int geologic_formation_id: int | None = None - geologic_formation: GeologicFormationResponse | None = None + geologic_formation: GeologicFormationGeoJSONResponse | None = None top_depth: float top_depth_unit: str = "ft" bottom_depth: float From 1342b8d30852d9cb47a845b29c55cd3179d403fc Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 12:54:53 -0700 Subject: [PATCH 62/91] fix: fix __table_args__ for aquifer and geology --- db/aquifer_system.py | 2 +- db/geologic_formation.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/db/aquifer_system.py b/db/aquifer_system.py index 53086f983..6a1681561 100644 --- a/db/aquifer_system.py +++ b/db/aquifer_system.py @@ -81,4 +81,4 @@ class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): ) # --- Table Arguments --- - __table_args__ = Index("ix_aquifersystem_name", "name") + __table_args__ = (Index("ix_aquifersystem_name", "name"),) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index 130ed8d45..a9483b501 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -79,7 +79,4 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing") # --- Table Arguments --- - __table_args__ = ( - Index("ix_geologicformation_name", "name"), - Index("ix_geologicformation_code", "code"), - ) + __table_args__ = (Index("ix_geologicformation_formation_code", "formation_code"),) From 88419b7911bf38070fb899c54dae25ddaba247ed Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:00:59 -0700 Subject: [PATCH 63/91] fix: add geologic_formation relation to WellScreen This was missing on the WellScreen side of the relationship --- db/geologic_formation.py | 2 +- db/thing.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/geologic_formation.py b/db/geologic_formation.py index a9483b501..2379f50f4 100644 --- a/db/geologic_formation.py +++ b/db/geologic_formation.py @@ -70,7 +70,7 @@ class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): ) ) # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. - screens: Mapped[List["WellScreen"]] = relationship( + well_screens: Mapped[List["WellScreen"]] = relationship( "WellScreen", back_populates="geologic_formation", passive_deletes=True ) diff --git a/db/thing.py b/db/thing.py index bf7e30db1..3c6ecf3db 100644 --- a/db/thing.py +++ b/db/thing.py @@ -528,6 +528,10 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): "AquiferSystem", back_populates="well_screens", passive_deletes=True ) + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="well_screens", passive_deletes=True + ) + class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): """ From 1a23dff0790e991edf6366db27379fc99788d9be Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:04:35 -0700 Subject: [PATCH 64/91] fix: add missing comma --- core/lexicon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index ec195b6b1..44a89d2aa 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -55,7 +55,7 @@ {"name": "status_value", "description": null}, {"name": "origin_source", "description": null}, {"name": "well_pump_type", "description": null}, - {"name": "permission_type", "description": null} + {"name": "permission_type", "description": null}, {"name": "formation_code", "description": null}, {"name": "lithology", "description": null} ], From d82b16373700afac658f35ffb542645b273d58f8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 13:33:11 -0700 Subject: [PATCH 65/91] fix: use BaseResponseModel for GeoJSONProperties in location schema --- schemas/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/location.py b/schemas/location.py index e911e3359..81c1ddd2d 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -97,7 +97,7 @@ class GeoJSONUTMCoordinates(BaseModel): ) -class GeoJSONProperties(BaseModel): +class GeoJSONProperties(BaseResponseModel): elevation: float elevation_unit: str = "ft" vertical_datum: str = "NAVD88" From 20af430d00ceade9a6beecf63404058598c32be8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:35:25 -0700 Subject: [PATCH 66/91] fix: include baseresponseinfo in location geojson properties --- schemas/location.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/location.py b/schemas/location.py index 81c1ddd2d..218790496 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -147,6 +147,9 @@ def populate_fields(cls, data: Any) -> Any: data_dict["geometry"]["coordinates"] = coordinates # populate properties + data_dict["properties"]["id"] = data_dict.get("id") + data_dict["properties"]["created_at"] = data_dict.get("created_at") + data_dict["properties"]["release_status"] = data_dict.get("release_status") data_dict["properties"]["notes"] = data_dict.get("notes") data_dict["properties"]["elevation"] = convert_m_to_ft(elevation_m) data_dict["properties"]["elevation_method"] = data_dict.get("elevation_method") From 5778b741a607810b4133080e59f34e31718ed082 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:36:35 -0700 Subject: [PATCH 67/91] feat: implement aquifer_systems in thing response --- db/thing.py | 2 +- schemas/thing.py | 23 ++++++++++++++----- tests/features/environment.py | 23 ++++++++++++++++++- .../steps/well-additional-information.py | 12 +++------- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/db/thing.py b/db/thing.py index 3c6ecf3db..340d8b0e9 100644 --- a/db/thing.py +++ b/db/thing.py @@ -332,7 +332,7 @@ class Thing( ) # Proxy to directly access AquiferSystems associated with this Thing - aquifers: AssociationProxy[List["AquiferSystem"]] = association_proxy( + aquifer_systems: AssociationProxy[List["AquiferSystem"]] = association_proxy( "aquifer_associations", "aquifer_system" ) diff --git a/schemas/thing.py b/schemas/thing.py index aecccd3e0..47be1021f 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,10 +33,11 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemResponse -from schemas.geologic_formation import ( - GeologicFormationResponse, -) +from schemas.aquifer_system import AquiferSystemGeoJSONResponse + +# from schemas.geologic_formation import ( +# GeologicFormationResponse, +# ) # -------- VALIDATE ---------- @@ -245,6 +246,7 @@ class WellResponse(BaseThingResponse): measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None + aquifer_systems: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None @@ -269,6 +271,14 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("aquifer_systems", mode="before") + def populate_aquifer_types_with_strings(cls, aquifer_systems): + if aquifer_systems is not None: + systems = [aquifer_system.name for aquifer_system in aquifer_systems] + else: + systems = [] + return systems + class SpringResponse(BaseThingResponse): """ @@ -291,9 +301,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemResponse | None = None + aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_type: str | None = None geologic_formation_id: int | None = None - geologic_formation: GeologicFormationResponse | None = None + # geologic_formation: GeologicFormationResponse | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float diff --git a/tests/features/environment.py b/tests/features/environment.py index ec5070bcc..0b2ca9d9f 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -37,6 +37,7 @@ MeasuringPointHistory, MonitoringFrequencyHistory, DataProvenance, + AquiferSystem, ) from db.engine import session_ctx @@ -423,9 +424,26 @@ def add_transducer_observation(context, session, block, deployment_id, value): return obs +@add_context_object_container("aquifer_systems") +def add_aquifer_system(context, session, name, well): + aquifer_system = AquiferSystem( + name=name, + description="this is a test aquifer", + primary_aquifer_type="Artesian", + geographic_scale="Major", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(aquifer_system) + session.commit() + session.refresh(aquifer_system) + + context.objects["aquifer_systems"].append(aquifer_system) + return aquifer_system + + def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True if rebuild: erase_and_rebuild_db() @@ -631,6 +649,9 @@ def before_all(context): for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) + for name in ["Aquifer A", "Aquifer B"]: + add_aquifer_system(context, session, name, well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) block = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 0824a0246..a9d6833f5 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -55,7 +55,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then("the response should include the completion date of the well") def step_impl(context): assert "well_completion_date" in context.water_well_data @@ -74,7 +73,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the driller name") def step_impl(context): assert "well_driller_name" in context.water_well_data @@ -107,7 +105,6 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: the transfer script needs to convert ft to in @then("the response should include the casing diameter in inches") def step_impl(context): assert "well_casing_diameter" in context.water_well_data @@ -140,8 +137,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data -# TODO: needs to be added to lexicon and an enum should be created @then("the response should include the well pump type (previously well_type field)") def step_impl(context): assert "well_pump_type" in context.water_well_data @@ -151,7 +146,6 @@ def step_impl(context): ) -# TODO: needs to be added to model, schemas, test data @then("the response should include the well pump depth in feet (new field)") def step_impl(context): assert "well_pump_depth" in context.water_well_data @@ -194,10 +188,10 @@ def step_impl(context): "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - assert "aquifer_class_code" in context.water_well_data + assert "aquifer_systems" in context.water_well_data assert ( - context.water_well_data["aquifer_class_code"] - == context.objects["wells"][0].aquifer_class_code + context.water_well_data["aquifer_systems"] + == context.objects["wells"][0].aquifer_systems ) From c4f9de0dfc9f5a41811a0dbdda15319db86d95df Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 14:50:55 -0700 Subject: [PATCH 68/91] feat: fix aquifer systems test --- tests/features/environment.py | 5 +++++ tests/features/steps/well-additional-information.py | 6 ++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/features/environment.py b/tests/features/environment.py index 0b2ca9d9f..8c683fac8 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -38,6 +38,7 @@ MonitoringFrequencyHistory, DataProvenance, AquiferSystem, + ThingAquiferAssociation, ) from db.engine import session_ctx @@ -437,6 +438,10 @@ def add_aquifer_system(context, session, name, well): session.commit() session.refresh(aquifer_system) + association = ThingAquiferAssociation(thing=well, aquifer_system=aquifer_system) + session.add(association) + session.commit() + context.objects["aquifer_systems"].append(aquifer_system) return aquifer_system diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index a9d6833f5..dbaaba0ed 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -183,15 +183,13 @@ def step_impl(context): assert context.water_well_data["formation"] == context.objects["wells"][0].formation -# TODO: needs to be added to model, schemas, test data, lexicon @then( "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): assert "aquifer_systems" in context.water_well_data - assert ( - context.water_well_data["aquifer_systems"] - == context.objects["wells"][0].aquifer_systems + assert sorted(context.water_well_data["aquifer_systems"]) == sorted( + [system.name for system in context.objects["aquifer_systems"]] ) From d3abd833144c832800f48d69746c34c3e9adfd81 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 15:45:07 -0700 Subject: [PATCH 69/91] fix: return aquifer system and types in dict this allows us to couple types to a system incase a well is in multiple aquifers --- db/thing.py | 17 +++++++++ schemas/thing.py | 11 +----- tests/features/environment.py | 35 ++++++++++++++++--- .../steps/well-additional-information.py | 25 +++++++------ 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/db/thing.py b/db/thing.py index 340d8b0e9..35c61b631 100644 --- a/db/thing.py +++ b/db/thing.py @@ -476,6 +476,23 @@ def allow_datalogger_installation(self): ) return permission_record.permission_allowed if permission_record else None + @property + def aquifers(self) -> List[dict]: + """ + Returns a list of aquifer systems and their associated types for this Thing. + Each aquifer system is represented as a dictionary with its name and a list of types. + """ + aquifer_list = [] + for association in self.aquifer_associations: + aquifer_info = { + "aquifer_system": association.aquifer_system.name, + "aquifer_types": [ + atype.aquifer_type for atype in association.aquifer_types + ], + } + aquifer_list.append(aquifer_info) + return aquifer_list + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/thing.py b/schemas/thing.py index 47be1021f..0ff765d76 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -246,8 +246,7 @@ class WellResponse(BaseThingResponse): measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None - aquifer_systems: list[str] = [] - + aquifers: list[dict] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -271,14 +270,6 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials - @field_validator("aquifer_systems", mode="before") - def populate_aquifer_types_with_strings(cls, aquifer_systems): - if aquifer_systems is not None: - systems = [aquifer_system.name for aquifer_system in aquifer_systems] - else: - systems = [] - return systems - class SpringResponse(BaseThingResponse): """ diff --git a/tests/features/environment.py b/tests/features/environment.py index 8c683fac8..d30f920ec 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -38,6 +38,7 @@ MonitoringFrequencyHistory, DataProvenance, AquiferSystem, + AquiferType, ThingAquiferAssociation, ) from db.engine import session_ctx @@ -438,17 +439,38 @@ def add_aquifer_system(context, session, name, well): session.commit() session.refresh(aquifer_system) + context.objects["aquifer_systems"].append(aquifer_system) + return aquifer_system + + +@add_context_object_container("thing_aquifer_associations") +def add_thing_aquifer_association(context, session, well, aquifer_system): association = ThingAquiferAssociation(thing=well, aquifer_system=aquifer_system) session.add(association) session.commit() + session.refresh(association) - context.objects["aquifer_systems"].append(aquifer_system) - return aquifer_system + context.objects["thing_aquifer_associations"].append(association) + return association + + +@add_context_object_container("aquifer_types") +def add_aquifer_type(context, session, aquifer_type_str, thing_aquifer_association): + aquifer_type = AquiferType( + aquifer_type=aquifer_type_str, + thing_aquifer_association=thing_aquifer_association, + ) + session.add(aquifer_type) + session.commit() + session.refresh(aquifer_type) + + context.objects["aquifer_types"].append(aquifer_type) + return aquifer_type def before_all(context): context.objects = {} - rebuild = True + rebuild = False # rebuild = True if rebuild: erase_and_rebuild_db() @@ -655,7 +677,12 @@ def before_all(context): add_well_purpose(context, session, well_1, purpose) for name in ["Aquifer A", "Aquifer B"]: - add_aquifer_system(context, session, name, well_1) + system = add_aquifer_system(context, session, name, well_1) + add_thing_aquifer_association(context, session, well_1, system) + + for t in ["Artesian", "Fractured"]: + taa = context.objects["thing_aquifer_associations"][0] + add_aquifer_type(context, session, t, taa) # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index dbaaba0ed..aba64d37c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -187,20 +187,23 @@ def step_impl(context): "the response should include the aquifer class code to classify the aquifer into aquifer system." ) def step_impl(context): - assert "aquifer_systems" in context.water_well_data - assert sorted(context.water_well_data["aquifer_systems"]) == sorted( - [system.name for system in context.objects["aquifer_systems"]] - ) + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_system" in aquifer + assert sorted( + [a.get("aquifer_system") for a in context.water_well_data["aquifers"]] + ) == sorted([system.name for system in context.objects["aquifer_systems"]]) -# TODO: needs to be added to model, schemas, test data -# TODO: should this be plural? that is, a descriptor model of the well @then( "the response should include the aquifer type as the type of aquifers penetrated by the well" ) def step_impl(context): - assert "aquifer_type" in context.water_well_data - assert ( - context.water_well_data["aquifer_type"] - == context.objects["wells"][0].aquifer_type - ) + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_types" in aquifer + + if aquifer["aquifer_system"] == "Aquifer A": + assert sorted(aquifer["aquifer_types"]) == sorted( + [a.aquifer_type for a in context.objects["aquifer_types"]] + ) + else: + assert aquifer["aquifer_types"] == [] From b4cd84e001ff33267d7bc71075c413c4c7f2dae7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:01:02 -0700 Subject: [PATCH 70/91] feat: implement geologic_formations from feature files --- db/thing.py | 4 +-- schemas/thing.py | 9 +++++++ tests/features/environment.py | 27 +++++++++++++++++++ .../steps/well-additional-information.py | 7 ++--- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/db/thing.py b/db/thing.py index 35c61b631..b08a14747 100644 --- a/db/thing.py +++ b/db/thing.py @@ -337,8 +337,8 @@ class Thing( ) # Proxy to directly access the GeologicFormations penetrated by this Thing. - formations: AssociationProxy[List["GeologicFormation"]] = association_proxy( - "formation_associations", "geologic_formation" + geologic_formations: AssociationProxy[List["GeologicFormation"]] = ( + association_proxy("formation_associations", "geologic_formation") ) # Full-text search vector diff --git a/schemas/thing.py b/schemas/thing.py index 0ff765d76..bd7665f92 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -247,6 +247,7 @@ class WellResponse(BaseThingResponse): measuring_point_height_unit: str = "ft" measuring_point_description: str | None aquifers: list[dict] = [] + geologic_formations: list[str] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None @@ -270,6 +271,14 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("geologic_formations", mode="before") + def populate_geologic_formations_with_strings(cls, geologic_formations): + if geologic_formations is not None: + formations = [formation.formation_code for formation in geologic_formations] + else: + formations = [] + return formations + class SpringResponse(BaseThingResponse): """ diff --git a/tests/features/environment.py b/tests/features/environment.py index d30f920ec..e2959ac85 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -40,6 +40,8 @@ AquiferSystem, AquiferType, ThingAquiferAssociation, + GeologicFormation, + ThingGeologicFormationAssociation, ) from db.engine import session_ctx @@ -468,6 +470,29 @@ def add_aquifer_type(context, session, aquifer_type_str, thing_aquifer_associati return aquifer_type +@add_context_object_container("geologic_formations") +def add_geologic_formation(context, session, formation_code, well): + formation = GeologicFormation( + formation_code=formation_code, + description="This is a test geologic formation.", + lithology="Peat", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(formation) + session.commit() + session.refresh(formation) + + association = ThingGeologicFormationAssociation( + top_depth=1, bottom_depth=10, thing=well, geologic_formation=formation + ) + session.add(association) + session.commit() + session.refresh(association) + + context.objects["geologic_formations"].append(formation) + return formation + + def before_all(context): context.objects = {} rebuild = False @@ -684,6 +709,8 @@ def before_all(context): taa = context.objects["thing_aquifer_associations"][0] add_aquifer_type(context, session, t, taa) + add_geologic_formation(context, session, "000EXRV", well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) block = add_block(context, session, parameter) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index aba64d37c..638c6529f 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -174,13 +174,14 @@ def step_impl(context): # ------------------------------------------------------------------------------ -# TODO: needs to be added to model, schemas, test data @then( "the response should include the formation as the formation zone of well completion" ) def step_impl(context): - assert "formation" in context.water_well_data - assert context.water_well_data["formation"] == context.objects["wells"][0].formation + assert "geologic_formations" in context.water_well_data + assert context.water_well_data["geologic_formations"] == [ + context.objects["geologic_formations"][0].formation_code + ] @then( From fbfa9089c29e9539025cec8cb8bf60195fb0520b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:20:35 -0700 Subject: [PATCH 71/91] fix: remove duplicate well pump types in lexicon this was an artifact from merge conflicts --- core/lexicon.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 990a5d657..8ef165e1a 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -699,10 +699,6 @@ {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, - {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, - {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, - {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, From e6af5e6aefe3452881fc078d6615a0b2caf66a35 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 16:42:50 -0700 Subject: [PATCH 72/91] fix: use OpenWellLoggerOK for is_suitable_for_datalogger field Laila indicated that this just describes if a datalogger can be physically installed at the well. It does not pertain to permissions --- schemas/thing.py | 1 + transfers/permissions_transfer.py | 34 ++++++++----------------------- transfers/well_transfer.py | 5 +++++ 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 977bf0c45..07e7e93ce 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -144,6 +144,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None + is_suitable_for_datalogger: bool | None = None class CreateSpring(CreateBaseThing): diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index d07c2cbe2..1dcca2b8f 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -4,6 +4,14 @@ from db import Thing, PermissionHistory from transfers.util import read_csv, logger, replace_nans +""" +Developer's notes + +According to Laila the column WellData.OpenWellLoggerOK only pertains to the +physical properties of a well (that is, if a datalogger can be installed). It +does not pertain to permissions. +""" + def transfer_permissions(session: Session): """ @@ -76,30 +84,4 @@ def transfer_permissions(session: Session): session.rollback() pass - allow_datalogger_installation = wdf.loc[ - wdf["PointID"] == well.name, "OpenWellLoggerOK" - ].values - if ( - len(allow_datalogger_installation) > 0 - and allow_datalogger_installation is not None - ): - try: - permission_allowed = bool(allow_datalogger_installation[0]) - permission = PermissionHistory( - contact_id=contact_id, - permission_type="Datalogger Installation", - permission_allowed=permission_allowed, - start_date=datetime.today().date(), - target_id=well.id, - target_table="thing", - ) - session.add(permission) - logger.info( - f"Transferred Datalogger Installation permission for well {well.name}: {permission_allowed}." - ) - except Exception as e: - logger.error(f"Error transferring permission for well {well.name}: {e}") - session.rollback() - pass - session.commit() diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 346d91629..9e684cc8d 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -286,6 +286,11 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None else None ), well_pump_type=well_pump_type, + is_suitable_for_datalogger=( + bool(row.OpenWellLoggerOK) + if not isna(row.OpenWellLoggerOK) + else None + ), ) CreateWell.model_validate(data) From d3c8101cf156c7608752bbaa943881c5a8057755 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 24 Nov 2025 17:00:14 -0700 Subject: [PATCH 73/91] feat: update CreateWell for transfers --- schemas/thing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/thing.py b/schemas/thing.py index 07e7e93ce..1baf324c3 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -144,7 +144,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None - is_suitable_for_datalogger: bool | None = None + is_suitable_for_datalogger: bool | None class CreateSpring(CreateBaseThing): From 2ea4cb16d9f8b896b23dc59eddc3ed916dedd434 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 07:57:26 -0700 Subject: [PATCH 74/91] fix: return aquifer/geology str instead of full response the well screen response includes the aquifer and geology information, but just as strings of names and codes instead of the full nested response objects. --- schemas/thing.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 1baf324c3..77df8c316 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,11 +33,6 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemGeoJSONResponse - -# from schemas.geologic_formation import ( -# GeologicFormationResponse, -# ) # -------- VALIDATE ---------- @@ -308,10 +303,10 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_system: str | None = None aquifer_type: str | None = None geologic_formation_id: int | None = None - # geologic_formation: GeologicFormationResponse | None = None + geologic_formation: str | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -319,6 +314,24 @@ class WellScreenResponse(BaseResponseModel): screen_type: str | None = None screen_description: str | None = None + @field_validator("aquifer_system", mode="before") + def populate_aquifer_system_with_name(cls, aquifer_system): + if aquifer_system is not None: + return aquifer_system.name + return None + + @field_validator("aquifer_type", mode="before") + def populate_aquifer_type_with_name(cls, aquifer_type): + if aquifer_type is not None: + return aquifer_type.name + return None + + @field_validator("geologic_formation_id", mode="before") + def populate_geologic_formation_with_code(cls, geologic_formation): + if geologic_formation is not None: + return geologic_formation.formation_code + return None + class GeoJSONGeometry(BaseModel): """ From 0994bec26f7198d638f1670f3f66f6fa982720b3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:16:31 -0700 Subject: [PATCH 75/91] refactor: clean up permission history testing data use a for loop for cleaner and more maintainable code --- tests/features/environment.py | 54 ++++++++++++----------------------- 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/tests/features/environment.py b/tests/features/environment.py index e2959ac85..6ba6caec3 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -517,42 +517,24 @@ def before_all(context): add_well_casing_material(context, session, well_1) contact = add_contact(context, session) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Datalogger Installation", - permission_allowed=True, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for datalogger installation.", - target_id=well_1.id, - target_table="thing", - ) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Water Level Sample", - permission_allowed=True, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for water level sampling.", - target_id=well_1.id, - target_table="thing", - ) - add_permission_history( - context, - session, - contact_id=context.objects["contacts"][0].id, - permission_type="Water Chemistry Sample", - permission_allowed=False, - start_date=datetime(2025, 1, 1).date(), - end_date=None, - notes="Permission granted for chemistry sampling.", - target_id=well_1.id, - target_table="thing", - ) + + for permission in [ + "Datalogger Installation", + "Water Level Sample", + "Water Chemistry Sample", + ]: + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type=permission, + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes=f"Permission granted for {permission.lower()}.", + target_id=well_1.id, + target_table="thing", + ) measuring_point_history_1 = add_measuring_point_history( context, session, well=well_1 From 3041102e8b2c97d81b99971505cb2ae2736170ea Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:44:59 -0700 Subject: [PATCH 76/91] fix: use origin_type to retrieve well depth source --- tests/features/steps/well-core-information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index b0adc8346..1f56161f6 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -163,7 +163,7 @@ def step_impl(context): and r.target_table == "thing" and r.target_id == context.objects["wells"][0].id ] - well_depth_source = well_depth_source_records[0].origin_source + well_depth_source = well_depth_source_records[0].origin_type assert context.water_well_data["well_depth_source"] == well_depth_source From 5e8fdade7963a04308c7618cf039f9db51ed52c8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 09:46:21 -0700 Subject: [PATCH 77/91] refactor: use __tabename__ for polymorphic mixins this ensures fidelity in table naming across all database models --- db/data_provenance.py | 4 ++-- db/notes.py | 2 +- db/permission_history.py | 4 ++-- db/status_history.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db/data_provenance.py b/db/data_provenance.py index 14cfdc5aa..20505d94c 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -19,7 +19,7 @@ from sqlalchemy import Integer, Index, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin from db import lexicon_term @@ -120,7 +120,7 @@ def data_provenance(cls): "DataProvenance", primaryjoin=and_( cls.id == foreign(DataProvenance.target_id), - DataProvenance.target_table == pascal_to_snake(cls.__name__), + DataProvenance.target_table == cls.__tablename__, ), lazy="selectin", viewonly=True, diff --git a/db/notes.py b/db/notes.py index ab8384064..dfc99bbba 100644 --- a/db/notes.py +++ b/db/notes.py @@ -97,7 +97,7 @@ def notes(cls): "Notes", primaryjoin=and_( cls.id == foreign(Notes.target_id), - Notes.target_table == cls.__name__, + Notes.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", diff --git a/db/permission_history.py b/db/permission_history.py index 7c9c37159..591046bba 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -12,7 +12,7 @@ from sqlalchemy import Integer, ForeignKey, String, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: @@ -88,7 +88,7 @@ def permission_history(cls): primaryjoin=( and_( cls.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == pascal_to_snake(cls.__name__), + PermissionHistory.target_table == cls.__tablename__, ) ), lazy="selectin", diff --git a/db/status_history.py b/db/status_history.py index 8b3ee2321..15b5aec2f 100644 --- a/db/status_history.py +++ b/db/status_history.py @@ -19,7 +19,7 @@ ) from sqlalchemy.orm import Mapped, mapped_column, declared_attr, relationship, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term class StatusHistory(Base, AutoBaseMixin, ReleaseMixin): @@ -47,7 +47,7 @@ def status_history(cls): "StatusHistory", primaryjoin=and_( cls.id == foreign(StatusHistory.target_id), - StatusHistory.target_table == pascal_to_snake(cls.__name__), + StatusHistory.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", From 95ff8c0a63cc1395347a7b3284b98735e0a5e8bd Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 10:21:52 -0700 Subject: [PATCH 78/91] fix: use __tablename__ for NotesMixin add_note Now that __tablename__ is used throughout it should be used for the target_table in add_note --- db/notes.py | 2 +- tests/features/environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/notes.py b/db/notes.py index dfc99bbba..0e2e8ab8b 100644 --- a/db/notes.py +++ b/db/notes.py @@ -120,7 +120,7 @@ def add_note( content=content, note_type=note_type, target_id=self.id, - target_table=self.__class__.__name__, + target_table=self.__class__.__tablename__, release_status=release_status, ) diff --git a/tests/features/environment.py b/tests/features/environment.py index 6ba6caec3..5fca63bf0 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -495,7 +495,7 @@ def add_geologic_formation(context, session, formation_code, well): def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True if rebuild: erase_and_rebuild_db() From 69b2b7a50cf5e5cf03ca0174f79a48f2f1146885 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 11:37:29 -0700 Subject: [PATCH 79/91] refactor: move well_construction_notes to polymorphic table all notes should reside in that table rather than as separate columns in the well table --- db/thing.py | 11 ++++++----- schemas/thing.py | 3 +-- tests/features/environment.py | 2 +- tests/features/steps/well-notes.py | 8 +++----- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/db/thing.py b/db/thing.py index b08a14747..8d2266e82 100644 --- a/db/thing.py +++ b/db/thing.py @@ -15,7 +15,7 @@ # =============================================================================== from typing import List, TYPE_CHECKING from datetime import date -from sqlalchemy import Integer, ForeignKey, String, Column, Float, Text, Date +from sqlalchemy import Integer, ForeignKey, String, Column, Float, Date from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType @@ -116,9 +116,6 @@ class Thing( info={"unit": "feet below ground surface"}, comment="Depth of the well casing from ground surface to the bottom of the casing (in feet).", ) - - well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) - well_completion_date: Mapped[str] = mapped_column( Date, nullable=True, comment="the date the well was completed if known" ) @@ -342,7 +339,7 @@ class Thing( ) # Full-text search vector - search_vector = Column(TSVectorType("name", "well_construction_notes")) + search_vector = Column(TSVectorType("name")) @property def current_location(self): @@ -372,6 +369,10 @@ def general_notes(self): def measuring_notes(self): return self._get_notes("Measuring") + @property + def construction_notes(self): + return self._get_notes("Construction") + @property def well_status(self) -> str | None: """ diff --git a/schemas/thing.py b/schemas/thing.py index 77df8c316..418dabf3a 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -120,7 +120,6 @@ class CreateWell(CreateBaseThing, ValidateWell): hole_depth: float | None = Field( default=None, gt=0, description="Hole depth in feet" ) - well_construction_notes: str | None = None well_casing_diameter: float | None = Field( default=None, gt=0, description="Well casing diameter in inches" ) @@ -231,7 +230,6 @@ class WellResponse(BaseThingResponse): well_casing_depth: float | None = None well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] - well_construction_notes: str | None = None well_completion_date: PastOrTodayDate | None well_completion_date_source: str | None well_driller_name: str | None @@ -250,6 +248,7 @@ class WellResponse(BaseThingResponse): measuring_point_description: str | None aquifers: list[dict] = [] geologic_formations: list[str] = [] + construction_notes: list[NoteResponse] | None = None water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None diff --git a/tests/features/environment.py b/tests/features/environment.py index 5fca63bf0..ddc461e74 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -92,7 +92,6 @@ def add_well(context, session, location, name_num): release_status="draft", well_depth=10, hole_depth=10, - well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, well_completion_date="2013-05-15", @@ -116,6 +115,7 @@ def add_well(context, session, location, name_num): ("Other", "well notes"), ("Water", "water notes"), ("Measuring", "measuring notes"), + ("Construction", "construction notes"), ): n = well.add_note(c, nt) session.add(n) diff --git a/tests/features/steps/well-notes.py b/tests/features/steps/well-notes.py index ffd692234..66520e6ab 100644 --- a/tests/features/steps/well-notes.py +++ b/tests/features/steps/well-notes.py @@ -49,11 +49,9 @@ def step_impl(context): ) def step_impl(context): data = context.response.json() - assert ( - "well_construction_notes" in data - ), "Response does not include construction notes" - assert data["well_construction_notes"] is not None, "Construction notes is null" - context.notes["construction"] = data["well_construction_notes"] + assert "construction_notes" in data, "Response does not include construction notes" + assert data["construction_notes"] is not None, "Construction notes is null" + context.notes["construction"] = data["construction_notes"] @then("the response should include general well notes (catch all notes field)") From 404a8f06396520283689396270fd29720b8fb52c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 12:36:09 -0700 Subject: [PATCH 80/91] feat: transfer other, water, and construction notes for wells --- transfers/well_transfer.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 9e684cc8d..202f14647 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -265,7 +265,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None first_visit_date=first_visit_date, hole_depth=row.HoleDepth, well_depth=row.WellDepth, - well_construction_notes=row.ConstructionNotes, well_casing_diameter=( row.CasingDiameter * 12 if row.CasingDiameter else None ), @@ -273,9 +272,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None release_status="public" if row.PublicRelease else "private", measuring_point_height=row.MPHeight, measuring_point_description=row.MeasuringPoint, - notes=( - [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] - ), well_completion_date=row.CompletionDate, well_driller_name=row.DrillerName, well_construction_method=( @@ -313,12 +309,12 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "measuring_point_description", "well_completion_date_source", "well_construction_method_source", + "notes", ] ) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID - well_data.pop("notes") well = Thing(**well_data) session.add(well) @@ -360,9 +356,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None # add things thate need well id for well in session.query(Thing).filter(Thing.thing_type == "water well").all(): row = wdf[wdf["PointID"] == well.name].iloc[0] - if not isna(row.Notes): - note = well.add_note(row.Notes, "Other") - session.add(note) + + notes = [] + if row.Notes: + notes.append({"content": row.Notes, "note_type": "Other"}) + if row.ConstructionNotes: + notes.append( + {"content": row.ConstructionNotes, "note_type": "Construction"} + ) + if row.WaterNotes: + notes.append({"content": row.WaterNotes, "note_type": "Water"}) + if notes: + for note in notes: + n = well.add_note(note["content"], note["note_type"]) + session.add(n) + print("ADDED NOTES!!!") location = well.current_location elevation_method = added_locations[row.PointID] From fe8541b8ff9d79ed4d17fd6806cc424536dc16b5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 13:05:10 -0700 Subject: [PATCH 81/91] feat: transfer Measuring notes transfer from WaterLevels.SiteNotes to polymorphic notes table where target_table = 'thing' and the target_id is thing.id --- transfers/waterlevels_transfer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index a1bb32717..cc187599c 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -385,6 +385,19 @@ def transfer_water_levels(session): logger.info( f"{SPACE_4}Created observation: ID {observation.id} | DT {observation.observation_datetime} | Value {observation.value} | MPHeight {observation.measuring_point_height} | nma_pk_waterlevels {observation.nma_pk_waterlevels}" ) + + # WaterLevels.SiteNotes --> notes where note_type = "measuring_notes" + if not pd.isna(row.SiteNotes): + note = thing.add_note( + content=row.SiteNotes, + note_type="Measuring", + release_status=release_status, + ) + session.add(note) + logger.info( + f"{SPACE_4}Added 'Measuring' note to Thing ID {thing.id} | Note ID {note.id}" + ) + session.commit() return input_df, cleaned_df, errors From 5c6b44459d54a2161d10a6e0578526d23164a854 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 25 Nov 2025 16:30:05 -0700 Subject: [PATCH 82/91] feat: transfer location notes --- core/lexicon.json | 1 + db/location.py | 5 ----- transfers/thing_transfer.py | 9 +++++++-- transfers/util.py | 11 +++++++---- transfers/well_transfer.py | 19 +++++++++++++++---- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 8ef165e1a..03bb67bf4 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1102,6 +1102,7 @@ {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, + {"categories": ["note_type"], "term": "Coordinate", "definition": "Notes about location coordinates"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, diff --git a/db/location.py b/db/location.py index 50b1aa0db..c4b3c6b05 100644 --- a/db/location.py +++ b/db/location.py @@ -24,7 +24,6 @@ ForeignKey, DateTime, func, - Text, ) from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped, mapped_column @@ -56,10 +55,6 @@ class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin, DataProvenanceMixi county: Mapped[str] = mapped_column(String(100), nullable=True) state: Mapped[str] = mapped_column(String(100), nullable=True) quad_name: Mapped[str] = mapped_column(String(100), nullable=True) - # TODO: remove this 'notes' field in favor of using the polymorphic Notes table. Did not remove it yet to avoid breaking existing data model. - # notes: Mapped[str] = mapped_column(Text, nullable=True) - nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True) - nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True) # --- Relationship Definitions --- thing_associations: Mapped[list["LocationThingAssociation"]] = relationship( diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 3469fbc53..b4db4dd5f 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== import time - +from pandas import isna from pydantic import ValidationError from sqlalchemy.orm import Session @@ -57,12 +57,17 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - session.commit() try: - location, elevation_method = make_location(row, cached_elevations) + location, elevation_method, location_notes = make_location( + row, cached_elevations + ) session.add(location) session.flush() data_provenances = make_location_data_provenance( row, location, elevation_method ) + for note_type, note_content in location_notes.items(): + if not isna(note_content): + location.add_note(note_content, note_type) for dp in data_provenances: session.add(dp) diff --git a/transfers/util.py b/transfers/util.py index 9b2afc84c..3bf5f4d0a 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -191,7 +191,7 @@ def chunk_by_size(df, chunk_size): def make_location(row: pd.Series, elevations: dict) -> tuple: """ - Returns a tuple of location data and the elevation method + Returns a tuple of location data, elevation method, and notes """ point = Point(row.Easting, row.Northing) @@ -257,17 +257,20 @@ def make_location(row: pd.Series, elevations: dict) -> tuple: f"LU_AltitudeMethod:{row.AltitudeMethod.strip()}" ) + notes = { + "Coordinate": row.CoordinateNotes, + "Other": row.LocationNotes, + } + location = Location( nma_pk_location=row.LocationId, point=transformed_point.wkt, elevation=z, release_status="public" if row.PublicRelease else "private", created_at=created_at, - nma_coordinate_notes=row.CoordinateNotes, - nma_notes_location=row.LocationNotes, ) - return location, elevation_method + return location, elevation_method, notes def make_location_data_provenance( diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 202f14647..3a93f50bb 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -227,9 +227,12 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None location = None try: - location, elevation_method = make_location(row, cached_elevations) + location, elevation_method, location_notes = make_location( + row, cached_elevations + ) session.add(location) - added_locations[row.PointID] = elevation_method + + added_locations[row.PointID] = elevation_method, location_notes except Exception as e: if location is not None: session.expunge(location) @@ -370,13 +373,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None for note in notes: n = well.add_note(note["content"], note["note_type"]) session.add(n) - print("ADDED NOTES!!!") location = well.current_location - elevation_method = added_locations[row.PointID] + elevation_method, location_notes = added_locations[row.PointID] data_provenances = make_location_data_provenance( row, location, elevation_method ) + + for note_type, note_content in location_notes.items(): + if not isna(note_content): + ln = location.add_note(note_content, note_type) + session.add(ln) + logger.info( + f"Added note of type {note_type} for current location of well {well.name}" + ) + for dp in data_provenances: session.add(dp) From bdff12edaa20f743f8e5160add8473b162fe929d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 08:33:58 -0700 Subject: [PATCH 83/91] refactor: use sets in feature tests for comparison --- .../steps/well-additional-information.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 638c6529f..0b584d45c 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -132,9 +132,9 @@ def step_impl(context): @then("the response should include the casing materials") def step_impl(context): assert "well_casing_materials" in context.water_well_data - assert sorted(context.water_well_data["well_casing_materials"]) == sorted( - [m.material for m in context.objects["wells"][0].well_casing_materials] - ) + assert set(context.water_well_data["well_casing_materials"]) == { + m.material for m in context.objects["wells"][0].well_casing_materials + } @then("the response should include the well pump type (previously well_type field)") @@ -190,9 +190,9 @@ def step_impl(context): def step_impl(context): for aquifer in context.water_well_data["aquifers"]: assert "aquifer_system" in aquifer - assert sorted( - [a.get("aquifer_system") for a in context.water_well_data["aquifers"]] - ) == sorted([system.name for system in context.objects["aquifer_systems"]]) + assert {a.get("aquifer_system") for a in context.water_well_data["aquifers"]} == { + system.name for system in context.objects["aquifer_systems"] + } @then( @@ -203,8 +203,8 @@ def step_impl(context): assert "aquifer_types" in aquifer if aquifer["aquifer_system"] == "Aquifer A": - assert sorted(aquifer["aquifer_types"]) == sorted( - [a.aquifer_type for a in context.objects["aquifer_types"]] - ) + assert set(aquifer["aquifer_types"]) == { + a.aquifer_type for a in context.objects["aquifer_types"] + } else: assert aquifer["aquifer_types"] == [] From 19d736aba6d566e3946c24a8cf1cfb3ee59b4b82 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 09:55:21 -0700 Subject: [PATCH 84/91] refactor: return permission history records for a well instead of a specific permission type, return a list of permission records for all permission types associated with the well. --- db/thing.py | 42 +++------- schemas/permission_history.py | 18 +++++ schemas/thing.py | 67 +++++++++++++--- .../steps/well-additional-information.py | 77 ++++++++++++++++--- 4 files changed, 156 insertions(+), 48 deletions(-) create mode 100644 schemas/permission_history.py diff --git a/db/thing.py b/db/thing.py index b08a14747..cae9363e0 100644 --- a/db/thing.py +++ b/db/thing.py @@ -446,36 +446,6 @@ def well_construction_method_source(self) -> str | None: "well_construction_method", "origin_source" ) - @property - def allow_water_level_samples(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Water Level Sample" - ) - return permission_record.permission_allowed if permission_record else None - - @property - def allow_water_chemistry_samples(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Water Chemistry Sample" - ) - return permission_record.permission_allowed if permission_record else None - - @property - def allow_datalogger_installation(self): - """ - Returns the current permissions for the Thing. - """ - permission_record = retrieve_latest_polymorphic_history_table_record( - self, "permission_history", "Datalogger Installation" - ) - return permission_record.permission_allowed if permission_record else None - @property def aquifers(self) -> List[dict]: """ @@ -493,6 +463,18 @@ def aquifers(self) -> List[dict]: aquifer_list.append(aquifer_info) return aquifer_list + @property + def permissions(self) -> list: + """ + Returns the associated permissions or an empty list. If there are no + associated permissions, an empty list is returned instead of None to + allow the API to serialize correctly (see schemas/thing.py). + """ + if self.permission_history: + return self.permission_history + else: + return [] + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/permission_history.py b/schemas/permission_history.py new file mode 100644 index 000000000..e0619d90e --- /dev/null +++ b/schemas/permission_history.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel +from schemas import PastOrTodayDate + +from core.enums import PermissionType + + +# ------ RESPONSE ---------- +class PermissionHistoryResponse(BaseModel): + """ + Even though permission_allowed and start_date are not-nullable in the + database, they are nullable here to accommodate cases where no permission + record exists for a given permission type. + """ + + permission_type: PermissionType + permission_allowed: bool | None + start_date: PastOrTodayDate | None + end_date: PastOrTodayDate | None diff --git a/schemas/thing.py b/schemas/thing.py index bd7665f92..2b8967553 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -33,11 +33,7 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.aquifer_system import AquiferSystemGeoJSONResponse - -# from schemas.geologic_formation import ( -# GeologicFormationResponse, -# ) +from schemas.permission_history import PermissionHistoryResponse # -------- VALIDATE ---------- @@ -238,9 +234,6 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" - allow_water_level_samples: bool | None - allow_water_chemistry_samples: bool | None - allow_datalogger_installation: bool | None is_suitable_for_datalogger: bool | None well_status: str | None measuring_point_height: float @@ -251,6 +244,7 @@ class WellResponse(BaseThingResponse): water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None + permissions: list[PermissionHistoryResponse] @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -279,6 +273,43 @@ def populate_geologic_formations_with_strings(cls, geologic_formations): formations = [] return formations + @field_validator("permissions", mode="before") + def populate_permission_history_with_latest_records(cls, permissions): + """ + Populate the permission history with the latest records for each + type of permission. If multiple records exist for the same permission type + only the most recent one is included. If there are no records + the permission_allowed will be None + """ + permissions_to_return = [] + for permission_type in [ + "Water Level Sample", + "Water Chemistry Sample", + "Datalogger Installation", + ]: + # Filter records for the current permission type + filtered_records = [ + record + for record in permissions + if record.permission_type == permission_type and record.end_date is None + ] + if filtered_records: + # Get the most recent record based on start_date + latest_record = max( + filtered_records, key=lambda record: record.start_date + ) + permissions_to_return.append(latest_record) + else: + permissions_to_return.append( + PermissionHistoryResponse( + permission_type=permission_type, + permission_allowed=None, + start_date=None, + end_date=None, + ) + ) + return permissions_to_return + class SpringResponse(BaseThingResponse): """ @@ -301,7 +332,7 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse aquifer_system_id: int | None = None - aquifer_system: AquiferSystemGeoJSONResponse | None = None + aquifer_system: str | None = None aquifer_type: str | None = None geologic_formation_id: int | None = None # geologic_formation: GeologicFormationResponse | None = None @@ -312,6 +343,24 @@ class WellScreenResponse(BaseResponseModel): screen_type: str | None = None screen_description: str | None = None + @field_validator("aquifer_system", mode="before") + def populate_aquifer_system_with_name(cls, aquifer_system): + if aquifer_system is not None: + return aquifer_system.name + return None + + @field_validator("aquifer_type", mode="before") + def populate_aquifer_type_with_name(cls, aquifer_type): + if aquifer_type is not None: + return aquifer_type.name + return None + + @field_validator("geologic_formation_id", mode="before") + def populate_geologic_formation_with_code(cls, geologic_formation): + if geologic_formation is not None: + return geologic_formation.formation_code + return None + class GeoJSONGeometry(BaseModel): """ diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 0b584d45c..5658f2ccc 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -10,44 +10,103 @@ "the response should include whether repeat measurement permission is granted for the well" ) def step_impl(context): - assert "allow_water_level_samples" in context.water_well_data + permission_type = "Water Level Sample" + assert "permissions" in context.water_well_data + permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Water Level Sample" + context.objects["wells"][0], "permission_history", permission_type + ) + + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type ) assert ( - context.water_well_data["allow_water_level_samples"] + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None @then("the response should include whether sampling permission is granted for the well") def step_impl(context): - assert "allow_water_chemistry_samples" in context.water_well_data + permission_type = "Water Chemistry Sample" + assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Water Chemistry Sample" + context.objects["wells"][0], "permission_history", permission_type ) + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] assert ( - context.water_well_data["allow_water_chemistry_samples"] + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) + assert ( + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None @then( "the response should include whether datalogger installation permission is granted for the well" ) def step_impl(context): - assert "allow_datalogger_installation" in context.water_well_data + permission_type = "Datalogger Installation" + assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", "Datalogger Installation" + context.objects["wells"][0], "permission_history", permission_type ) + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) assert ( - context.water_well_data["allow_datalogger_installation"] + water_well_data_permissions["permission_allowed"] == permission_record.permission_allowed ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None # ------------------------------------------------------------------------------ From 656e159d4179a1d5ae826d5bd25266491166d6ce Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 13:27:45 -0700 Subject: [PATCH 85/91] fix: use regex to check for pump type in well construction notes regex is a more robust way to check for the presence of pump types in construction notes, handling variations in spacing and punctuation. --- transfers/well_transfer.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 9e684cc8d..22209741a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -16,7 +16,7 @@ import json import time from datetime import datetime, UTC - +import re import pandas as pd from pandas import isna from pydantic import ValidationError @@ -118,19 +118,26 @@ def _extract_casing_materials(row) -> list[str]: return materials +pattern = re.compile( + r"\b(?Pjet|hand|submersible)\b|\b(?Pline[-\s]+shaft)\b", re.IGNORECASE +) + + +def first_matched_term(text: str): + m = pattern.search(text) + if not m: + return None + return m.group("term") or m.group("phrase") + + +PUMP_MAPPING = {"jet": "Jet", "hand": "Hand", "submersible": "Submersible"} + + def _extract_well_pump_type(row) -> str | None: + if isna(row.ConstructionNotes): + return None construction_notes = row.ConstructionNotes.lower() - if "pump" in construction_notes: - if "submersible" in construction_notes: - return "Submersible" - elif "jet" in construction_notes: - return "Jet" - elif "line shaft" in construction_notes or "lineshaft" in construction_notes: - return "Line Shaft" - elif "hand" in construction_notes: - return "Hand" - else: - return None + return PUMP_MAPPING.get(first_matched_term(construction_notes), None) def get_wells_to_transfer( @@ -252,9 +259,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) - well_pump_type = ( - _extract_well_pump_type(row) if row.ConstructionNotes else None - ) + well_pump_type = _extract_well_pump_type(row) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation From 183d275525c5c323d90f46fbb6cf828d7136bb4b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 14:20:27 -0700 Subject: [PATCH 86/91] fix: ensure permissions are not null during transfer check to make sure that permissions are set before transferring them from one entity to another to avoid potential errors. --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 1dcca2b8f..fa9ecf8e7 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -1,5 +1,6 @@ from sqlalchemy.orm import Session from datetime import datetime +from pandas import isna from db import Thing, PermissionHistory from transfers.util import read_csv, logger, replace_nans @@ -15,8 +16,10 @@ def transfer_permissions(session: Session): """ - The transferred wells and contacts need to be queried to know who gave - permission to which well since contact_id is required for PermissionHistory + The transferred wells and contacts need to be transferred first + - to access the auto-generated well IDs + - to know who gave permission to which well since contact_id is required for + PermissionHistory """ wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) wdf = replace_nans(wdf) @@ -38,7 +41,9 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + if len(allow_water_level_samples) > 0 and not isna( + allow_water_level_samples[0] + ): try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -61,9 +66,8 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if ( - len(allow_water_chemistry_samples) > 0 - and allow_water_chemistry_samples is not None + if len(allow_water_chemistry_samples) > 0 and not isna( + allow_water_chemistry_samples[0] ): try: permission_allowed = bool(allow_water_chemistry_samples[0]) From 2d0f2b6177980836ca7ba895ea7df72a3f13f5eb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 26 Nov 2025 15:10:15 -0700 Subject: [PATCH 87/91] refactor: check for existence of permissions before converting --- transfers/permissions_transfer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index fa9ecf8e7..18daa1040 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -41,9 +41,11 @@ def transfer_permissions(session: Session): allow_water_level_samples = wdf.loc[ wdf["PointID"] == well.name, "MonitorOK" ].values - if len(allow_water_level_samples) > 0 and not isna( - allow_water_level_samples[0] - ): + if len(allow_water_level_samples) == 0: + pass + elif isna(allow_water_level_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_level_samples[0]) permission = PermissionHistory( @@ -66,9 +68,11 @@ def transfer_permissions(session: Session): allow_water_chemistry_samples = wdf.loc[ wdf["PointID"] == well.name, "SampleOK" ].values - if len(allow_water_chemistry_samples) > 0 and not isna( - allow_water_chemistry_samples[0] - ): + if len(allow_water_chemistry_samples) == 0: + pass + elif isna(allow_water_chemistry_samples[0]): + pass + else: try: permission_allowed = bool(allow_water_chemistry_samples[0]) permission = PermissionHistory( From cc2b29a94bfd50f68aba5a57683b7fd5c067aa71 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 11:51:26 -0700 Subject: [PATCH 88/91] refactor: use NotesMixin for FieldEvent use the polymorphic table rather than a standalone field --- db/field.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/db/field.py b/db/field.py index 0ebf7e6e0..8c72c5376 100644 --- a/db/field.py +++ b/db/field.py @@ -4,7 +4,7 @@ from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from typing import TYPE_CHECKING -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term +from db.base import Base, AutoBaseMixin, ReleaseMixin, NotesMixin, lexicon_term if TYPE_CHECKING: from db.contact import Contact @@ -52,7 +52,7 @@ class FieldEventParticipant(Base, AutoBaseMixin, ReleaseMixin): ) -class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): +class FieldEvent(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): """ This table serves as the master log for all field visits. Each record in this table represents a single, continuous collection event at a @@ -82,10 +82,6 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Date and time of the field event.", ) - notes: Mapped[str] = mapped_column( - nullable=True, - comment="Notes or comments about the field event.", - ) # --- Relationships --- thing: Mapped["Thing"] = relationship(back_populates="field_events") field_activities: Mapped[list["FieldActivity"]] = relationship( From a8d177ab482ba9741894275f7ba8054177f63aa5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 11:59:36 -0700 Subject: [PATCH 89/91] fix: import NotesMixin correctly for FieldEvent --- db/field.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/field.py b/db/field.py index 8c72c5376..ab49be30a 100644 --- a/db/field.py +++ b/db/field.py @@ -4,7 +4,8 @@ from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from typing import TYPE_CHECKING -from db.base import Base, AutoBaseMixin, ReleaseMixin, NotesMixin, lexicon_term +from db.notes import NotesMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: from db.contact import Contact From 001d24353959bdb1fcc95c1aa2d56a2d086a5202 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 12:00:04 -0700 Subject: [PATCH 90/91] refactor: rename Other to General for notes This is the name used for serialization so there should be correspondence for ease of use and maintenance --- core/lexicon.json | 2 +- tests/features/environment.py | 4 ++-- transfers/util.py | 2 +- transfers/waterlevels_transfer.py | 25 +++++++++++++------------ transfers/well_transfer.py | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 03bb67bf4..4f38b3c1a 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1099,7 +1099,7 @@ {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, - {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, + {"categories": ["note_type"], "term": "General", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, {"categories": ["note_type"], "term": "Coordinate", "definition": "Notes about location coordinates"}, diff --git a/tests/features/environment.py b/tests/features/environment.py index ddc461e74..06bbe63ed 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -74,7 +74,7 @@ def add_location(context, session): session.add(loc) session.commit() session.refresh(loc) - n = loc.add_note("Test location", "Other") + n = loc.add_note("Test location", "General") session.add(n) session.commit() session.refresh(loc) @@ -112,7 +112,7 @@ def add_well(context, session, location, name_num): session.refresh(well) for nt, c in ( - ("Other", "well notes"), + ("General", "well notes"), ("Water", "water notes"), ("Measuring", "measuring notes"), ("Construction", "construction notes"), diff --git a/transfers/util.py b/transfers/util.py index 3bf5f4d0a..295e1f219 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -259,7 +259,7 @@ def make_location(row: pd.Series, elevations: dict) -> tuple: notes = { "Coordinate": row.CoordinateNotes, - "Other": row.LocationNotes, + "General": row.LocationNotes, } location = Location( diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index cc187599c..e3c867cf6 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -251,6 +251,19 @@ def transfer_water_levels(session): ) session.add(field_event) + session.flush() # to get the field_event.id + + # WaterLevels.SiteNotes --> notes where note_type = "measuring_notes" + if not pd.isna(row.SiteNotes): + note = field_event.add_note( + content=row.SiteNotes, + note_type="General", + release_status=release_status, + ) + session.add(note) + logger.info( + f"{SPACE_4}Added 'General' note to FieldEvent ID {field_event.id} | Note ID {note.id}" + ) logger.info( f"{SPACE_2}Created field event: ID {field_event.id} | Date {field_event.event_date} | Thing ID {field_event.thing.id} | Thing Name {field_event.thing.name}" @@ -386,18 +399,6 @@ def transfer_water_levels(session): f"{SPACE_4}Created observation: ID {observation.id} | DT {observation.observation_datetime} | Value {observation.value} | MPHeight {observation.measuring_point_height} | nma_pk_waterlevels {observation.nma_pk_waterlevels}" ) - # WaterLevels.SiteNotes --> notes where note_type = "measuring_notes" - if not pd.isna(row.SiteNotes): - note = thing.add_note( - content=row.SiteNotes, - note_type="Measuring", - release_status=release_status, - ) - session.add(note) - logger.info( - f"{SPACE_4}Added 'Measuring' note to Thing ID {thing.id} | Note ID {note.id}" - ) - session.commit() return input_df, cleaned_df, errors diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 11b64f753..df823371b 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -367,7 +367,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None notes = [] if row.Notes: - notes.append({"content": row.Notes, "note_type": "Other"}) + notes.append({"content": row.Notes, "note_type": "General"}) if row.ConstructionNotes: notes.append( {"content": row.ConstructionNotes, "note_type": "Construction"} From 2d644100964b3cc623ec7ded5277bd242ebe13b0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 1 Dec 2025 12:05:35 -0700 Subject: [PATCH 91/91] feat: add SiteNotes for field events --- transfers/waterlevels_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index e3c867cf6..6a0f1fea0 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -262,7 +262,7 @@ def transfer_water_levels(session): ) session.add(note) logger.info( - f"{SPACE_4}Added 'General' note to FieldEvent ID {field_event.id} | Note ID {note.id}" + f"{SPACE_4}Added 'General' note to FieldEvent ID {field_event.id}" ) logger.info(