From 6de7807c1857a99eb1c30d8f77e1fd50de232a9d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 09:44:13 -0700 Subject: [PATCH 1/8] refactor: rename 'Other' note type to 'General' this is the nomenclature used by AMP and in the feature files, so this change improves consistency across the codebase and teams --- core/lexicon.json | 2 +- tests/features/environment.py | 4 +- tests/features/geojson-response.feature | 23 +++++++++ tests/features/location-notes.feature | 30 +++++++++++ tests/features/search-response.feature | 44 ++++++++++++++++ tests/features/sensor-notes.feature | 39 +++++++++++++++ tests/features/thing-query-parameters.feature | 24 +++++++++ .../thing-type-path-parameters.feature | 24 +++++++++ .../features/transducer-data-response.feature | 30 +++++++++++ .../well-additional-information.feature | 40 +++++++++++++++ tests/features/well-core-information.feature | 50 +++++++++++++++++++ tests/features/well-notes.feature | 29 +++++++++++ 12 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 tests/features/geojson-response.feature create mode 100644 tests/features/location-notes.feature create mode 100644 tests/features/search-response.feature create mode 100644 tests/features/sensor-notes.feature create mode 100644 tests/features/thing-query-parameters.feature create mode 100644 tests/features/thing-type-path-parameters.feature create mode 100644 tests/features/transducer-data-response.feature create mode 100644 tests/features/well-additional-information.feature create mode 100644 tests/features/well-core-information.feature create mode 100644 tests/features/well-notes.feature diff --git a/core/lexicon.json b/core/lexicon.json index 423be5332..43542275f 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1176,7 +1176,7 @@ {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, - {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, + {"categories": ["note_type"], "term": "General", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, diff --git a/tests/features/environment.py b/tests/features/environment.py index 13bcdead3..da4d3a00e 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -74,7 +74,7 @@ def add_location(context, session): session.add(loc) session.commit() session.refresh(loc) - n = loc.add_note("Test location", "Other") + n = loc.add_note("Test location", "General") session.add(n) session.commit() session.refresh(loc) @@ -114,7 +114,7 @@ def add_well(context, session, location, name_num): session.refresh(well) for nt, c in ( - ("Other", "well notes"), + ("General", "well notes"), ("Water", "water notes"), ("Measuring", "measuring notes"), ): diff --git a/tests/features/geojson-response.feature b/tests/features/geojson-response.feature new file mode 100644 index 000000000..7e5757dcb --- /dev/null +++ b/tests/features/geojson-response.feature @@ -0,0 +1,23 @@ +# Created by jakeross at 10/22/25 +@backend @production +Feature: Geojson Response + # Enter feature description here + Background: + Given a functioning api + And the system has valid well and location data in the database + + @positive @happy_path + Scenario: Request all wells as geojson + When the user requests all the wells as geojson + Then the system should return a 200 status code + And the system should return a response in GEOJSON format + And the response should be a feature collection + And the feature collection should have 3 features + + @positive @happy_path + Scenario: Request all wells in a group as geojson + When the user requests all the wells for group Collabnet + Then the system should return a 200 status code + And the system should return a response in GEOJSON format + And the response should be a feature collection + And the feature collection should have 2 features diff --git a/tests/features/location-notes.feature b/tests/features/location-notes.feature new file mode 100644 index 000000000..3d08bd4d7 --- /dev/null +++ b/tests/features/location-notes.feature @@ -0,0 +1,30 @@ +# Created by jakeross at 10/21/25 +@backend @BDMS-199 @production +Feature: Retrieve location notes by well name + As a user + I want to retrieve location notes for a given well name + So that I can view important information about the well's location + Background: + Given a functioning api + And the system has valid well and location data in the database + + @positive @happy_path + Scenario: Retrieve location notes for an existing well + When the user retrieves the well by ID via path parameter + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include a current location + And the current location should include notes + And the notes should be a list of dictionaries + And each note dictionary should have "content" and "note_type" keys + And each note in the notes list should be a non-empty string + + @positive @happy_path + Scenario: Retrieve location notes by location ID + When the user retrieves the location by ID via path parameter + Then the system should return a 200 status code + And the system should return a response in JSON format + And the location response should include notes + And the notes should be a list of dictionaries + And each note dictionary should have "content" and "note_type" keys + And each note in the notes list should be a non-empty string diff --git a/tests/features/search-response.feature b/tests/features/search-response.feature new file mode 100644 index 000000000..8b3761379 --- /dev/null +++ b/tests/features/search-response.feature @@ -0,0 +1,44 @@ +@backend @BDMS-169 +Feature: Unified search API returns grouped results + As a user + I want to search for contacts, wells, and springs + So that I can quickly find relevant information across multiple data types + + Background: + Given a functioning api + And the system has valid contact, well, and spring records in the database + + @positive @happy_path + Scenario: Retrieve mixed search results + When the user searches for "" + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include results grouped by: + | Contacts | Wells | Springs | + And each result should include a label, group, and properties + + @positive @happy_path + Scenario: Retrieve contact results + When the user searches for "" + Then the system should return a 200 status code + And the response should include a "Contacts" group + And each contact result should include: + #| TODO: use correct field name syntax | + | id | first_name | last_name | email | phone | address | associated_things | + + @positive @happy_path + Scenario: Retrieve well results + When the user searches for "" + Then the system should return a 200 status code + And the response should include a "Wells" group + And each well result should include: + #| TODO: use correct field name syntax | + | thing_type | id | name | alternate site name | contact first name | contact last name | contact id | county | + + @positive @happy_path + Scenario: Retrieve spring results + When the user searches for "" + Then the system should return a 200 status code + And the response should include a "Springs" group + And each spring result should include: + | thing_type | id | \ No newline at end of file diff --git a/tests/features/sensor-notes.feature b/tests/features/sensor-notes.feature new file mode 100644 index 000000000..b44ed8578 --- /dev/null +++ b/tests/features/sensor-notes.feature @@ -0,0 +1,39 @@ +# Created by jakeross at 10/21/25 +@backend @BDMS-199 +Feature: Retrieve sensor notes + As a user + I want to retrieve sensor notes for a given well name + So that I can view important information about the well's deployed sensors + Background: + Given a functioning api + And the system has valid well and location data in the database + +# @positive @happy_path +# Scenario: Request sensor notes for an existing well +# When the user requests the sensor for well 1 +# Then the system should return a 200 status code +# And the system should return a response in JSON format +# And the response should include notes +# And the notes should be a non-empty string + + @positive + Scenario: Request sensor notes by sensor ID + When the user requests the sensor with ID 1 + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include notes + And the notes should be a non-empty string + + @negative + Scenario: Request sensor notes for a non-existing sensor ID + When the user requests the sensor with ID 9999 + Then the system should return a 404 status code + And the system should return a response in JSON format + And the response should include an error message indicating the sensor was not found + +# @negative @sad_path +# Scenario: Request sensor notes for a non-existing well +# When the user requests the sensor for well 9999 +# Then the system should return a 404 status code +# And the system should return a response in JSON format +# And the response should include an error message indicating the well was not found \ No newline at end of file diff --git a/tests/features/thing-query-parameters.feature b/tests/features/thing-query-parameters.feature new file mode 100644 index 000000000..b0d3d250a --- /dev/null +++ b/tests/features/thing-query-parameters.feature @@ -0,0 +1,24 @@ +# Created by jakeross at 11/2/25 +@backend @BDMS-218 @production +Feature: Thing query paramaters + Use query parameters to filter things + Background: + Given a functioning api + And the system has valid well and location data in the database + + @positive @happy_path + Scenario: Filter things by type + When the user requests things with type "water well" + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include at least one thing + And the response should only include things of type "water well" + + @positive @happy_path + Scenario: + When the user requests things with type "spring" + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include at least one thing + And the response should only include things of type "spring" + diff --git a/tests/features/thing-type-path-parameters.feature b/tests/features/thing-type-path-parameters.feature new file mode 100644 index 000000000..f744e4728 --- /dev/null +++ b/tests/features/thing-type-path-parameters.feature @@ -0,0 +1,24 @@ +# Created by jakeross at 11/2/25 +@backend @BDMS-218 @production +Feature: Thing type path paramaters + Use path parameters to filter things + Background: + Given a functioning api + And the system has valid well and location data in the database + + @positive @happy_path + Scenario: Get all Water Well Things + When the user requests things with type "water well" + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include at least one thing + And the response should only include things of type "water well" + + @positive @happy_path + Scenario: + When the user requests things with type "spring" + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should include at least one thing + And the response should only include things of type "spring" + diff --git a/tests/features/transducer-data-response.feature b/tests/features/transducer-data-response.feature new file mode 100644 index 000000000..998503f04 --- /dev/null +++ b/tests/features/transducer-data-response.feature @@ -0,0 +1,30 @@ +# Created by jakeross at 11/4/25 +@backend @production +Feature: Transducer Data Response + This feature tests the API's ability to retrieve transducer data for wells. + Background: + Given a functioning api + And the system has valid well and transducer data in the database + + @positive @happy_path + Scenario: Retrieve transducer data for an existing well + When the user requests transducer data for a well + Then the system should return a 200 status code + And the system should return a response in JSON format + + And the response should be paginated + And each page should be an array of transducer data + And each transducer data entry should include a timestamp, value, status + And the timestamp should be in ISO 8601 format + And the value should be a numeric type + And the status should be one of "approved", "not reviewed" + + + + @negative @sad_path + Scenario: Retrieve transducer data for a non-existing well + When the user requests transducer data for a non-existing well + Then the system should return a 200 status code + And the system should return a response in JSON format + And the response should be paginated + And the items should be an empty list \ No newline at end of file diff --git a/tests/features/well-additional-information.feature b/tests/features/well-additional-information.feature new file mode 100644 index 000000000..b4d1ad08f --- /dev/null +++ b/tests/features/well-additional-information.feature @@ -0,0 +1,40 @@ +@backend @BDMS-227 +Feature: Retrieve additional well information by well ID + As a hydrogeologist or data specialist + I want to view additional well attributes with specific physical and operational characteristics + So that I have all necessary well data to confidently complete fieldwork + + Background: + Given a functioning api + And the system has valid well and location data in the database + + Scenario: Retrieve additional well information for an existing well + When the user retrieves the well by ID via path parameter + Then the system should return a 200 status code + And the system should return a response in JSON format + And null values in the response should be represented as JSON null (not placeholder strings) + + # Permissions / Operational OK flags + And the response should include whether repeat measurement permission is granted for the well + And the response should include whether sampling permission is granted for the well + And the response should include whether datalogger installation permission is granted for the well + + # Well Construction Information + And the response should include the completion date of the well + And the response should include the source of the completion information + And the response should include the driller name + And the response should include the construction method + And the response should include the source of the construction information + + # Additional Well Physical Properties + And the response should include the casing diameter in inches + And the response should include the casing depth in feet below ground surface + And the response should include the casing materials + And the response should include the well pump type (previously well_type field) + And the response should include the well pump depth in feet (new field) + And the response should include whether the well is open and suitable for a datalogger + + # Aquifer / Geology Information + And the response should include the formation as the formation zone of well completion + And the response should include the aquifer class code to classify the aquifer into aquifer system. + And the response should include the aquifer type as the type of aquifers penetrated by the well diff --git a/tests/features/well-core-information.feature b/tests/features/well-core-information.feature new file mode 100644 index 000000000..a1d9598e0 --- /dev/null +++ b/tests/features/well-core-information.feature @@ -0,0 +1,50 @@ +@backend @BDMS-221 +Feature: Retrieve core well information by well ID + As a hydrogeologist or data specialist + I want to view clearly labeled core physical attributes and identifiers for the well in a well information page section + so that I can assess key well characteristics at a glance + + Background: + Given a functioning api + And the system has valid well and location data in the database + + Scenario: Retrieve core well information for an existing well + When the user retrieves the well by ID via path parameter + Then the system should return a 200 status code + And the system should return a response in JSON format + And null values in the response should be represented as JSON null (not placeholder strings) + + # Well names and projects + And the response should include the well name (point ID) (i.e. NM-1234) + And the response should include the project(s) or group(s) associated with the well + + # Well Purpose and Status and Monitoring Status + And the response should include the purpose of the well (current use) + And the response should include the well hole status of the well as the status of the hole in the ground (from previous Status field) + And the response should include the monitoring frequency (new field) + And the response should include whether the well is currently being monitored with status text if applicable (from previous MonitoringStatus field) + + # Data Lifecycle and Public Visibility + # NEEDS USER RESEARCH - keep both under release_status for now? (previously PublicRelease) + And the response should include the release status of the well record + + # Well Physical Properties + And the response should include the hole depth in feet + And the response should include the well depth in feet + And the response should include the source of the well depth information + + # Measuring Point Information + And the response should include the description of the measuring point + And the response should include the measuring point height in feet + + # Location Information + # GeoJSON spec format RFC 7946 (Aug 2016) requires coordinates to be decimal degrees in WGS84 + And the response should include location information in GeoJSON spec format RFC 7946 + And the response should include a geometry object with type "Point" and coordinates array [longitude, latitude, elevation] + And the response should include the elevation in feet with vertical datum NAVD88 in the properties + And the response should include the elevation method (i.e. interpolated from digital elevation model) in the properties + And the response should include the UTM coordinates with datum NAD83 in the properties + + # Alternate Identifiers + And the response should include any alternate IDs for the well like the NMBGMR site_name (i.e. John Smith Well), USGS site number, or the OSE well ID and OSE well tag ID + diff --git a/tests/features/well-notes.feature b/tests/features/well-notes.feature new file mode 100644 index 000000000..5cb5a502b --- /dev/null +++ b/tests/features/well-notes.feature @@ -0,0 +1,29 @@ +# Created by jakeross at 10/21/25 +@backend @BDMS-199 @BDMS-233 @production +Feature: Retrieve well notes by well ID + As a user + I want to retrieve well notes for a given well + So that I can understand all necessary context about the well using info not captured in structured fields + Background: + Given a functioning api + And the system has valid well and location data in the database + + @positive @happy_path + Scenario: Retrieve well notes for an existing well + When the user retrieves the well by ID via path parameter + Then the system should return a 200 status code + And the system should return a response in JSON format + And null values in the response should be represented as JSON null (not placeholder strings) + And the response should include location notes (i.e. driving directions and geographic well location notes) + And the response should include construction notes (i.e. pump notes and other construction notes) + And the response should include general well notes (catch all notes field) + And the response should include measuring notes (notes about measuring/visiting the well, on Access form) + And the response should include water notes (i.e. water bearing zone information and other info from ose reports) + And the notes should be a non-empty string + + @negative @sad_path + Scenario: Retrieve well notes for a non-existing well + When the user retrieves the well 9999 + Then the system should return a 404 status code + And the system should return a response in JSON format + And the response should include an error message indicating the well was not found \ No newline at end of file From a922d13820ad798c601ba06630bc0885116c4eaa Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 10:09:56 -0700 Subject: [PATCH 2/8] refactor: move well_construction_notes to polymorphic notes table This field used to be standalone, but with the creation and implementation of the polymorphic notes table it has been moved there --- db/thing.py | 10 ++++++---- schemas/thing.py | 2 +- tests/conftest.py | 1 - tests/features/environment.py | 2 +- tests/features/steps/well-notes.py | 8 +++----- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/db/thing.py b/db/thing.py index 92c7bd942..8ace92c4c 100644 --- a/db/thing.py +++ b/db/thing.py @@ -15,7 +15,7 @@ # =============================================================================== from typing import List, TYPE_CHECKING from datetime import date -from sqlalchemy import Integer, ForeignKey, String, Column, Float, Text, Date +from sqlalchemy import Integer, ForeignKey, String, Column, Float, Date from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType @@ -117,8 +117,6 @@ class Thing( comment="Depth of the well casing from ground surface to the bottom of the casing (in feet).", ) - well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) - well_completion_date: Mapped[date] = mapped_column( nullable=True, comment="the date the well was completed if known" ) @@ -348,7 +346,7 @@ class Thing( ) # Full-text search vector - search_vector = Column(TSVectorType("name", "well_construction_notes")) + search_vector = Column(TSVectorType("name")) @property def current_location(self): @@ -378,6 +376,10 @@ def general_notes(self): def measuring_notes(self): return self._get_notes("Measuring") + @property + def construction_notes(self): + return self._get_notes("Construction") + @property def well_status(self) -> str | None: """ diff --git a/schemas/thing.py b/schemas/thing.py index 7a7982494..b82f55138 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -234,7 +234,6 @@ class WellResponse(BaseThingResponse): well_casing_depth: float | None = None well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] - well_construction_notes: str | None = None well_completion_date: PastOrTodayDate | None well_completion_date_source: str | None well_driller_name: str | None @@ -252,6 +251,7 @@ class WellResponse(BaseThingResponse): water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None + construction_notes: list[NoteResponse] | None = None permissions: list[PermissionHistoryResponse] formation_completion_code: FormationCode | None diff --git a/tests/conftest.py b/tests/conftest.py index 022171ed0..cd27b3cea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,6 @@ def water_well_thing(location): release_status="draft", well_depth=10, hole_depth=10, - well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, ) diff --git a/tests/features/environment.py b/tests/features/environment.py index da4d3a00e..1d655a4da 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -92,7 +92,6 @@ def add_well(context, session, location, name_num): release_status="draft", well_depth=10, hole_depth=10, - well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, well_completion_date="2013-05-15", @@ -117,6 +116,7 @@ def add_well(context, session, location, name_num): ("General", "well notes"), ("Water", "water notes"), ("Measuring", "measuring notes"), + ("Construction", "construction notes"), ): n = well.add_note(c, nt) session.add(n) diff --git a/tests/features/steps/well-notes.py b/tests/features/steps/well-notes.py index ffd692234..66520e6ab 100644 --- a/tests/features/steps/well-notes.py +++ b/tests/features/steps/well-notes.py @@ -49,11 +49,9 @@ def step_impl(context): ) def step_impl(context): data = context.response.json() - assert ( - "well_construction_notes" in data - ), "Response does not include construction notes" - assert data["well_construction_notes"] is not None, "Construction notes is null" - context.notes["construction"] = data["well_construction_notes"] + assert "construction_notes" in data, "Response does not include construction notes" + assert data["construction_notes"] is not None, "Construction notes is null" + context.notes["construction"] = data["construction_notes"] @then("the response should include general well notes (catch all notes field)") From d1f9cd73dabd20fca3b9f2df5870f584d013c845 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 10:47:04 -0700 Subject: [PATCH 3/8] fix: use "General" for general notes in thing model It used to be called "Other" but has been changed to "General" --- db/thing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/thing.py b/db/thing.py index 8ace92c4c..91afaba78 100644 --- a/db/thing.py +++ b/db/thing.py @@ -370,7 +370,7 @@ def water_notes(self): @property def general_notes(self): - return self._get_notes("Other") + return self._get_notes("General") @property def measuring_notes(self): From f7087b9df53d3038b59e2ec5de9eb064a305fa1d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 10:50:28 -0700 Subject: [PATCH 4/8] refactor: return specific note types rather than all notes at once The feature file requests specific note types. By returning all notes at once, duplicate entries will be returned. Furthermore, some note types only apply to specific thing types. So, general notes are returned for all thing types, but then for each thing type specific notes will be returned --- schemas/thing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index b82f55138..3b0bd19e5 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -195,13 +195,11 @@ class BaseThingResponse(BaseResponseModel): thing_type: str current_location: LocationGeoJSONResponse first_visit_date: PastOrTodayDate | None - # The new relationship to the polymorphic Notes table - notes: List[NoteResponse] = [] - groups: list[GroupResponse] = [] monitoring_status: str | None links: list[ThingIdLinkResponse] = Field(default=[], alias="alternate_ids") monitoring_frequencies: list[MonitoringFrequencyResponse] = [] + general_notes: list[NoteResponse] | None = None @field_validator("monitoring_frequencies", mode="before") def remove_records_with_end_date(cls, monitoring_frequencies): @@ -250,7 +248,7 @@ class WellResponse(BaseThingResponse): aquifers: list[dict] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None - general_notes: list[NoteResponse] | None = None + construction_notes: list[NoteResponse] | None = None permissions: list[PermissionHistoryResponse] formation_completion_code: FormationCode | None From efb216b269fc1a80c02347c8b0f179b69bb09880 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 10:56:25 -0700 Subject: [PATCH 5/8] refactor: update well notes test for general notes --- tests/features/steps/well-notes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/features/steps/well-notes.py b/tests/features/steps/well-notes.py index 66520e6ab..9e20e84f3 100644 --- a/tests/features/steps/well-notes.py +++ b/tests/features/steps/well-notes.py @@ -57,9 +57,9 @@ def step_impl(context): @then("the response should include general well notes (catch all notes field)") def step_impl(context): data = context.response.json() - assert "notes" in data, "Response does not include notes" - assert data["notes"] is not None, "Notes is null" - context.notes["general"] = data["notes"] + assert "general_notes" in data, "Response does not include notes" + assert data["general_notes"] is not None, "Notes is null" + context.notes["general"] = data["general_notes"] @then( From 3fc559b2e150c076edd33310233e8813b1023b29 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 13:35:21 -0700 Subject: [PATCH 6/8] fix: remove well construction notes from CreateWell these notes are no longer a field in the thing table and need to be handled separately. they can be added back later on and handled by the polymorphic notes table, but for the time being they impede the transfer process --- schemas/thing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schemas/thing.py b/schemas/thing.py index 3b0bd19e5..f25c1d691 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -122,7 +122,6 @@ class CreateWell(CreateBaseThing, ValidateWell): hole_depth: float | None = Field( default=None, gt=0, description="Hole depth in feet" ) - well_construction_notes: str | None = None well_casing_diameter: float | None = Field( default=None, gt=0, description="Well casing diameter in inches" ) From cd2699f94421591fe8083d551a723dfde86186e8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 13:42:58 -0700 Subject: [PATCH 7/8] refactor: transfer well and location notes to polymorphic notes table Add relevant notes as enumerated in well-notes.feature and remove well_construction_notes from the well transfer as they are now stored in the polymorphic notes table with note_type "Construction" --- tests/test_transfer_legacy_dates.py | 20 ++++++++++---------- transfers/thing_transfer.py | 11 +++++++++-- transfers/util.py | 11 +++++++---- transfers/waterlevels_transfer.py | 2 ++ transfers/well_transfer.py | 24 +++++++++++++++++++----- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/tests/test_transfer_legacy_dates.py b/tests/test_transfer_legacy_dates.py index 985214fbb..f871acb3b 100644 --- a/tests/test_transfer_legacy_dates.py +++ b/tests/test_transfer_legacy_dates.py @@ -21,7 +21,7 @@ 2. Location.nma_site_date is populated from CSV SiteDate if not null (read-only post-migration) """ import datetime -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import patch import pandas as pd import pytest @@ -71,7 +71,7 @@ def test_make_location_with_both_ampapi_dates(mock_lexicon_mapper): elevations = {} # Call make_location - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Verify nma_date_created is set from DateCreated assert location.nma_date_created is not None @@ -106,7 +106,7 @@ def test_make_location_with_only_date_created(mock_lexicon_mapper): ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Verify nma_date_created is set assert location.nma_date_created == datetime.date(2014, 4, 3) @@ -136,7 +136,7 @@ def test_make_location_with_site_date_later_than_date_created(mock_lexicon_mappe ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Both dates should be preserved as-is, regardless of order assert location.nma_date_created == datetime.date(2010, 1, 15) @@ -164,7 +164,7 @@ def test_make_location_with_very_old_site_date(mock_lexicon_mapper): ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Verify very old date is preserved assert location.nma_site_date == datetime.date(1954, 5, 1) @@ -196,7 +196,7 @@ def test_make_location_ampapi_dates_are_date_not_datetime(mock_lexicon_mapper): ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Verify they are date objects (not datetime) assert isinstance(location.nma_date_created, datetime.date) @@ -231,7 +231,7 @@ def test_make_location_ampapi_dates_independent_of_created_at(mock_lexicon_mappe ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # created_at should be None during transfer (auto-set by AutoBaseMixin on save) assert location.created_at is None @@ -271,7 +271,7 @@ def test_make_location_with_no_ampapi_dates(mock_lexicon_mapper): ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Both AMPAPI date fields should be null assert location.nma_date_created is None @@ -299,7 +299,7 @@ def test_make_location_with_empty_string_dates(mock_lexicon_mapper): ) elevations = {} - location, elevation_method = make_location(row, elevations) + location, elevation_method, location_notes = make_location(row, elevations) # Both AMPAPI date fields should be null (empty strings are falsy) assert location.nma_date_created is None @@ -336,7 +336,7 @@ def create_test_row(i, has_site_date): for i in range(100): row = create_test_row(i, has_site_date=(i < 9)) - location, _ = make_location(row, elevations) + location, _, _ = make_location(row, elevations) # Count coverage if location.nma_date_created is not None: diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 3469fbc53..6cfcea4c8 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== import time - +from pandas import isna from pydantic import ValidationError from sqlalchemy.orm import Session @@ -57,9 +57,16 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - session.commit() try: - location, elevation_method = make_location(row, cached_elevations) + location, elevation_method, location_notes = make_location( + row, cached_elevations + ) session.add(location) session.flush() + for note_type, note_content in location_notes.items(): + if not isna(note_content): + location_note = location.add_note(note_content, note_type) + session.add(location_note) + data_provenances = make_location_data_provenance( row, location, elevation_method ) diff --git a/transfers/util.py b/transfers/util.py index d1bc5d053..4faf5c91a 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -400,7 +400,7 @@ def get_groundwater_parameter_id() -> int: def make_location(row: pd.Series, elevations: dict) -> tuple: """ - Returns a tuple of location data and the elevation method + Returns a tuple of location data, the elevation method, and notes """ point = Point(row.Easting, row.Northing) @@ -439,6 +439,11 @@ def make_location(row: pd.Series, elevations: dict) -> tuple: f"LU_AltitudeMethod:{row.AltitudeMethod.strip()}" ) + notes = { + "Coordinate": row.CoordinateNotes, + "General": row.LocationNotes, + } + # Extract AMPAPI date fields (Date type, not DateTime) nma_date_created = None if row.DateCreated: @@ -455,13 +460,11 @@ def make_location(row: pd.Series, elevations: dict) -> tuple: point=transformed_point.wkt, elevation=z, release_status="public" if row.PublicRelease else "private", - nma_coordinate_notes=row.CoordinateNotes, - nma_notes_location=row.LocationNotes, nma_date_created=nma_date_created, nma_site_date=nma_site_date, ) - return location, elevation_method + return location, elevation_method, notes def make_location_data_provenance( diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 270592a66..ec612fbf0 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -235,6 +235,8 @@ def _get_groundwater_level_reason(self, row) -> str: if pd.isna(glv): return None + if glv == "X?": + glv = "X" glv = lexicon_mapper.map_value(f"LU_LevelStatus:{glv}") if glv == "Water level not affected by status": glv = "Water level not affected" diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index d92f2ece6..5d2309579 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -316,10 +316,12 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): location = None try: - location, elevation_method = make_location(row, self._cached_elevations) + location, elevation_method, location_notes = make_location( + row, self._cached_elevations + ) session.add(location) session.commit() - self._added_locations[row.PointID] = elevation_method + self._added_locations[row.PointID] = elevation_method, location_notes except Exception as e: self._capture_error(row.PointID, str(e), str(e), "Location") logger.critical(f"Error making location for {row.PointID}: {e}") @@ -346,7 +348,6 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): first_visit_date=first_visit_date, hole_depth=row.HoleDepth, well_depth=row.WellDepth, - well_construction_notes=row.ConstructionNotes, well_casing_diameter=( row.CasingDiameter * 12 if row.CasingDiameter else None ), @@ -596,11 +597,24 @@ def _after_hook(self, session): step_start_time = time.time() row = self.cleaned_df[self.cleaned_df["PointID"] == well.name].iloc[0] if notna(row.Notes): - note = well.add_note(row.Notes, "Other") + note = well.add_note(row.Notes, "General") + objs.append(note) + if row.ConstructionNotes: + note = well.add_note(row.ConstructionNotes, "Construction") + objs.append(note) + if row.WaterNotes: + note = well.add_note(row.WaterNotes, "Water") objs.append(note) location = well.current_location - elevation_method = self._added_locations[row.PointID] + elevation_method, location_notes = self._added_locations[row.PointID] + for note_type, note_content in location_notes.items(): + if not isna(note_content): + location_note = location.add_note(note_content, note_type) + objs.append(location_note) + logger.info( + f"Added note of type {note_type} for current location of well {well.name}" + ) data_provenances = make_location_data_provenance( row, location, elevation_method ) From 821a2e71868d109c58c6e4847f96cab3ece97777 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Dec 2025 13:48:09 -0700 Subject: [PATCH 8/8] feat: add Coordinate as note type to lexicon --- core/lexicon.json | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lexicon.json b/core/lexicon.json index 43542275f..2a9b3e63c 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1179,6 +1179,7 @@ {"categories": ["note_type"], "term": "General", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, + {"categories": ["note_type"], "term": "Coordinate", "definition": "Notes about a location's coordinates"}, {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"},