BDMS-408: add water level csv feature

api/search.py

@@ -164,7 +164,10 @@ def make_spring_response(thing: Thing) -> dict:
 def _get_asset_results(session: Session, q: str, limit: int) -> list[dict]:
     vector = Asset.search_vector
     query = search(
-        select(Asset).join(AssetThingAssociation).join(Thing),
+        select(Asset)
+        .join(AssetThingAssociation)
+        .join(Thing)
+        .options(selectinload(Asset.things)),
         q,
         vector=vector,
         limit=limit,
@@ -176,7 +179,11 @@ def _get_asset_results(session: Session, q: str, limit: int) -> list[dict]:
             "label": a.name,
             "group": "Assets",
             "properties": {
-                "things": [t.name for t in a.things],
+                "id": a.id,
+                "things": [
+                    {"label": t.name, "id": t.id, "thing_type": t.thing_type}
+                    for t in a.things
+                ],
                 "storage_service": a.storage_service,
                 "storage_path": a.storage_path,
                 "mime_type": a.mime_type,

core/lexicon.json

@@ -1168,7 +1168,7 @@
     {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."},
     {"categories": ["note_type"], "term": "General", "definition": "Other types of notes that do not fit into the predefined categories."},
     {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"},
-    {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"},
+    {"categories": ["note_type"], "term": "Sampling Procedure", "definition": "Notes about sampling procedures for all sample types, like water levels and water chemistry"},
     {"categories": ["note_type"], "term": "Coordinate", "definition": "Notes about a location's coordinates"},
     {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"},
     {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"},

db/thing.py

@@ -361,8 +361,8 @@ def general_notes(self):
         return self._get_notes("General")
 
     @property
-    def measuring_notes(self):
-        return self._get_notes("Measuring")
+    def sampling_procedure_notes(self):
+        return self._get_notes("Sampling Procedure")
 
     @property
     def construction_notes(self):

schemas/thing.py

@@ -196,7 +196,8 @@ class BaseThingResponse(BaseResponseModel):
     monitoring_status: str | None
     links: list[ThingIdLinkResponse] = Field(default=[], alias="alternate_ids")
     monitoring_frequencies: list[MonitoringFrequencyResponse] = []
-    general_notes: list[NoteResponse] | None = None
+    general_notes: list[NoteResponse] = []
+    sampling_procedure_notes: list[NoteResponse] = []
 
     @field_validator("monitoring_frequencies", mode="before")
     def remove_records_with_end_date(cls, monitoring_frequencies):
@@ -243,10 +244,8 @@ class WellResponse(BaseThingResponse):
     measuring_point_height_unit: str = "ft"
     measuring_point_description: str | None
     aquifers: list[dict] = []
-    water_notes: list[NoteResponse] | None = None
-    measuring_notes: list[NoteResponse] | None = None
-
-    construction_notes: list[NoteResponse] | None = None
+    water_notes: list[NoteResponse] = []
+    construction_notes: list[NoteResponse] = []
     permissions: list[PermissionHistoryResponse]
     formation_completion_code: FormationCode | None
 

tests/features/environment.py

@@ -115,7 +115,7 @@ def add_well(context, session, location, name_num):
     for nt, c in (
         ("General", "well notes"),
         ("Water", "water notes"),
-        ("Measuring", "measuring notes"),
+        ("Sampling Procedure", "sampling procedure notes"),
         ("Construction", "construction notes"),
     ):
         n = well.add_note(c, nt)

tests/features/steps/well-notes.py

@@ -63,13 +63,17 @@ def step_impl(context):
 
 
 @then(
-    "the response should include measuring notes (notes about measuring/visiting the well, on Access form)"
+    "the response should include sampling procedure notes (notes about sampling procedures for all sample types, like water levels and water chemistry)"
 )
 def step_impl(context):
     data = context.response.json()
-    assert "measuring_notes" in data, "Response does not include measuring notes"
-    assert data["measuring_notes"] is not None, "Measuring notes is null"
-    context.notes["measuring"] = data["measuring_notes"]
+    assert (
+        "sampling_procedure_notes" in data
+    ), "Response does not include sampling procedure notes"
+    assert (
+        data["sampling_procedure_notes"] is not None
+    ), "Sampling Procedure notes is null"
+    context.notes["sampling_procedure"] = data["sampling_procedure_notes"]
 
 
 @then(

tests/features/water-level-csv.feature

@@ -0,0 +1,132 @@
+@skip
+@backend
+@BDMS-TBD
+@production
+Feature: Bulk upload water level entries from CSV
+  As a hydrogeologist or data specialist
+  I want to upload a CSV file containing water level entry data for multiple wells
+  So that water level records can be created efficiently and accurately in the system
+
+  Background:
+    Given a functioning api
+    And valid lexicon values exist for:
+      | lexicon category      |
+      | sampler               |
+      | sample_method         |
+      | level_status          |
+      | data_quality          |
+
+  @positive @happy_path @BDMS-TBD
+  Scenario: Uploading a valid water level entry CSV containing required and optional fields
+    Given a valid CSV file for bulk water level entry upload
+    And my CSV file is encoded in UTF-8 and uses commas as separators
+    And my CSV file contains multiple rows of water level entry data
+    And the CSV includes required fields:
+      | required field name     |
+      | field_staff             |
+      | well_name_point_id      |
+      | field_event_date_time   |
+      | measurement_date_time   |
+      | sampler                 |
+      | sample_method           |
+      | mp_height               |
+      | level_status            |
+      | depth_to_water_ft       |
+      | data_quality            |
+    And each "well_name_point_id" value matches an existing well
+    And "measurement_date_time" values are valid ISO 8601 timestamps with timezone offsets (e.g. "2025-02-15T10:30:00-08:00")
+    And the CSV includes optional fields when available:
+      | optional field name     |
+      | water_level_notes       |
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 201 Created status code
+    And the system should return a response in JSON format
+    And the response includes a summary containing:
+      | summary_field              | value |
+      | total_rows_processed       | 2 |
+      | total_rows_imported        | 2 |
+      | validation_errors_or_warnings | 0  |
+    And the response includes an array of created water level entry objects
+
+  @positive @validation @column_order @BDMS-TBD
+  Scenario: Upload succeeds when required columns are present but in a different order
+    Given my CSV file contains all required headers but in a different column order
+    And the CSV includes required fields:
+      | required field name     |
+      | well_name_point_id      |
+      | measurement_date_time   |
+      | sampler                 |
+      | sample_method           |
+      | mp_height               |
+      | level_status            |
+      | depth_to_water_ft       |
+      | data_quality            |
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 201 Created status code
+    And the system should return a response in JSON format
+    And all water level entries are imported
+
+  @positive @validation @extra_columns @BDMS-TBD
+  Scenario: Upload succeeds when CSV contains extra, unknown columns
+    Given my CSV file contains extra columns but is otherwise valid
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 201 Created status code
+    And the system should return a response in JSON format
+    And all water level entries are imported
+
+  ###########################################################################
+  # NEGATIVE VALIDATION SCENARIOS
+  ###########################################################################
+
+  @negative @validation @BDMS-TBD
+  Scenario: No water level entries are imported when any row fails validation
+    Given my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id"
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 422 Unprocessable Entity status code
+    And the system should return a response in JSON format
+    And the response includes a validation error for the row missing "well_name_point_id"
+    And no water level entries are imported
+
+  @negative @validation @required_fields @BDMS-TBD
+  Scenario Outline: Upload fails when a required field is missing
+    Given my CSV file contains a row missing the required "<required_field>" field
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 422 Unprocessable Entity status code
+    And the system should return a response in JSON format
+    And the response includes a validation error for the "<required_field>" field
+    And no water level entries are imported
+
+    Examples:
+      | required_field         |
+      | well_name_point_id     |
+      | measurement_date_time  |
+      | sampler                |
+      | sample_method          |
+      | mp_height              |
+      | level_status           |
+      | depth_to_water_ft      |
+      | data_quality           |
+
+  @negative @validation @date_formats @BDMS-TBD
+  Scenario: Upload fails due to invalid date formats
+    Given my CSV file contains invalid ISO 8601 date values in the "measurement_date_time" field
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 422 Unprocessable Entity status code
+    And the response includes validation errors identifying the invalid field and row
+    And no water level entries are imported
+
+  @negative @validation @numeric_fields @BDMS-TBD
+  Scenario: Upload fails due to invalid numeric fields
+    Given my CSV file contains values that cannot be parsed as numeric in numeric-required fields such as "mp_height" or "depth_to_water_ft"
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 422 Unprocessable Entity status code
+    And the response includes validation errors identifying the invalid field and row
+    And no water level entries are imported
+
+  @negative @validation @lexicon_values @BDMS-TBD
+  Scenario: Upload fails due to invalid lexicon values
+    Given my CSV file contains invalid lexicon values for "sampler", "sample_method", "level_status", or "data_quality"
+    When I upload the file to the bulk upload endpoint
+    Then the system returns a 422 Unprocessable Entity status code
+    And the response includes validation errors identifying the invalid field and row
+    And no water level entries are imported

tests/features/well-notes.feature

@@ -17,7 +17,7 @@ Feature: Retrieve well notes by well ID
     And the response should include location notes (i.e. driving directions and geographic well location notes)
     And the response should include construction notes (i.e. pump notes and other construction notes)
     And the response should include general well notes (catch all notes field)
-    And the response should include measuring notes (notes about measuring/visiting the well, on Access form)
+    And the response should include sampling procedure notes (notes about sampling procedures for all sample types, like water levels and water chemistry)
     And the response should include water notes (i.e. water bearing zone information and other info from ose reports)
     And the notes should be a non-empty string
 

tests/test_search.py

@@ -39,14 +39,16 @@ def override_dependencies_fixture():
     app.dependency_overrides = {}
 
 
-def test_search_api(water_well_thing, spring_thing, contact):
+def test_search_api(
+    water_well_thing, spring_thing, contact, asset_with_associated_thing
+):
     response = client.get("/search", params={"q": "Test"})
     assert response.status_code == 200
     data = response.json()
     assert isinstance(data, dict)
     items = data.get("items")
     assert isinstance(items, list)
-    assert len(items) == 3
+    assert len(items) == 4
 
     # Check the contacts returned
     contact_items = [item for item in items if item["group"] == "Contacts"]
@@ -62,6 +64,20 @@ def test_search_api(water_well_thing, spring_thing, contact):
         },
     ]
 
+    # Check the assets returned
+    asset_items = [item for item in items if item["group"] == "Assets"]
+    assert len(asset_items) == 1
+    asset_item = asset_items[0]
+    assert asset_item["label"] == asset_with_associated_thing.name
+    assert asset_item["properties"]["id"] == asset_with_associated_thing.id
+    assert asset_item["properties"]["things"] == [
+        {
+            "label": water_well_thing.name,
+            "id": water_well_thing.id,
+            "thing_type": water_well_thing.thing_type,
+        },
+    ]
+
 
 @pytest.mark.skip(reason="This test is not working .")
 def test_search_api2():