diff --git a/core/lexicon.json b/core/lexicon.json index dc02e30d5..47eae5751 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -570,6 +570,7 @@ {"categories": ["organization"], "term": "Winter Brothers", "definition": "Winter Brothers"}, {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, + {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, @@ -584,9 +585,6 @@ {"categories": ["collection_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["collection_method"], "term": "Transit, theodolite, or other survey method", "definition": "Transit, theodolite, or other survey method"}, {"categories": ["role"], "term": "Principal Investigator", "definition": "Principal Investigator"}, - {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, - {"categories": ["collection_method"], "term": "manual", "definition": "manual sampling"}, - {"categories": ["collection_method"], "term": "continuous", "definition": "continuous sampling"}, {"categories": ["role"], "term": "Owner", "definition": "Owner"}, {"categories": ["role"], "term": "Manager", "definition": "Manager"}, {"categories": ["role"], "term": "Operator", "definition": "Operator"}, diff --git a/db/__init__.py b/db/__init__.py index 9771aaa1e..af993e8de 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -41,6 +41,7 @@ from db.thing import * from db.transducer import * from db.measuring_point_history import * +from db.data_provenance import * from sqlalchemy import ( func, diff --git a/db/data_provenance.py b/db/data_provenance.py index 4673fbd25..06c468c8d 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -121,3 +121,24 @@ def data_provenance(cls): lazy="selectin", viewonly=True, ) + + def _get_data_provenance_attribute(self, field_name, attribute): + """ + Returns the specified attribute from the DataProvenance record + for the given field_name, or None if not found. + + Args: + field_name (str): The name of the field to look up provenance for. + attribute (str): The attribute of the DataProvenance record to return. + + Returns: + The value of the specified attribute, or None if no record found. + """ + data_provenance_records = self.data_provenance + record = next( + (r for r in data_provenance_records if r.field_name == field_name), None + ) + if record: + return getattr(record, attribute) + else: + return None diff --git a/db/location.py b/db/location.py index bdc189dd7..24674b1cc 100644 --- a/db/location.py +++ b/db/location.py @@ -30,8 +30,8 @@ from sqlalchemy.orm import relationship, Mapped, mapped_column from constants import SRID_WGS84 -from db.base import Base, AutoBaseMixin, ReleaseMixin, DataProvenanceMixin -from db.lexicon import lexicon_term +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.data_provenance import DataProvenanceMixin if TYPE_CHECKING: from db.thing import Thing @@ -58,10 +58,6 @@ class Location(Base, AutoBaseMixin, ReleaseMixin, DataProvenanceMixin): notes: Mapped[str] = mapped_column(Text, nullable=True) nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True) nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True) - elevation_accuracy: Mapped[float] = mapped_column(nullable=True) - elevation_method: Mapped[str] = lexicon_term(nullable=True) - coordinate_accuracy: Mapped[float] = mapped_column(nullable=True) - coordinate_method: Mapped[str] = lexicon_term(nullable=True) # --- Relationship Definitions --- thing_associations: Mapped[list["LocationThingAssociation"]] = relationship( @@ -83,6 +79,10 @@ def latlon(self): p = to_shape(point) return p.y, p.x + @property + def elevation_method(self) -> str | None: + return self._get_data_provenance_attribute("elevation", "collection_method") + class LocationThingAssociation(Base, AutoBaseMixin): location_id: Mapped[int] = mapped_column( diff --git a/db/thing.py b/db/thing.py index 31d357c7b..ec6fed151 100644 --- a/db/thing.py +++ b/db/thing.py @@ -27,11 +27,11 @@ Base, ReleaseMixin, PermissionMixin, - DataProvenanceMixin, ) from db.status_history import StatusHistoryMixin from db.measuring_point_history import MeasuringPointHistory -from services.util import retrieve_latest_polymorphic_table_record +from db.data_provenance import DataProvenanceMixin +from services.util import retrieve_latest_polymorphic_history_table_record if TYPE_CHECKING: from db.location import Location @@ -313,7 +313,7 @@ def well_status(self) -> str | None: Since status_history is eagerly loaded, this should not introduce N+1 query issues. """ - latest_status = retrieve_latest_polymorphic_table_record( + latest_status = retrieve_latest_polymorphic_history_table_record( self, "status_history", "Well Status" ) return latest_status.status_value if latest_status else None @@ -326,7 +326,7 @@ def monitoring_status(self) -> str | None: Since status_history is eagerly loaded, this should not introduce N+1 query issues. """ - latest_status = retrieve_latest_polymorphic_table_record( + latest_status = retrieve_latest_polymorphic_history_table_record( self, "status_history", "Monitoring Status" ) return latest_status.status_value if latest_status else None @@ -363,6 +363,10 @@ def measuring_point_description(self) -> str | None: else: return None + @property + def well_depth_source(self) -> str | None: + return self._get_data_provenance_attribute("well_depth", "origin_source") + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/run_bdd.sh b/run_bdd.sh index 1f30a4432..9fd1ae38d 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -59,13 +59,13 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} #uv run behave tests/features --tags=@backend #uv run behave tests/features/sensor-notes.feature --tags=@backend -uv run behave tests/features/transducer-data-response.feature +# uv run behave tests/features/transducer-data-response.feature #uv run behave tests/features/transducer-data-response.feature \ # tests/features/thing-type-path-parameters.feature \ # tests/features/thing-query-parameters.feature #uv run behave tests/features/well-inventory-csv.feature - +uv run behave tests/features/well-core-information.feature --capture echo "✅ BDD test run complete." diff --git a/schemas/location.py b/schemas/location.py index 195562084..69e083793 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -123,6 +123,9 @@ def populate_fields(cls, data: Any) -> Any: if not isinstance(data, dict): data_dict = {c.name: getattr(data, c.name) for c in data.__table__.columns} + # @property need to be added manually + data_dict["elevation_method"] = data.elevation_method + # add empty fields as necessary data_dict["geometry"] = {} data_dict["properties"] = {} diff --git a/schemas/thing.py b/schemas/thing.py index 032571408..10479c49d 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -184,6 +184,7 @@ class WellResponse(BaseThingResponse): well_purposes: list[WellPurpose] = [] well_depth: float | None = None well_depth_unit: str = "ft" + well_depth_source: str | None hole_depth: float | None = None hole_depth_unit: str = "ft" well_casing_diameter: float | None = None # in inches diff --git a/services/util.py b/services/util.py index c5edee30e..06c29a6ad 100644 --- a/services/util.py +++ b/services/util.py @@ -132,7 +132,7 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_latest_polymorphic_table_record( +def retrieve_latest_polymorphic_history_table_record( target_record: DeclarativeBase, polymorphic_relationship: str, polymorphic_type: str, @@ -142,6 +142,9 @@ def retrieve_latest_polymorphic_table_record( parent class has the correct mixin to support retrieval via an attribute. This requires end_date to be None + This function does not apply to the DataProvenance table since it is not + a history table. + Parameters: ---------- target_record : DeclarativeBase diff --git a/tests/features/environment.py b/tests/features/environment.py index 6bfb94ede..0cb28d945 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -33,6 +33,7 @@ WellPurpose, MeasuringPointHistory, MonitoringFrequencyHistory, + DataProvenance, ) from db.engine import session_ctx @@ -57,10 +58,10 @@ def add_location(context, session): point="POINT(-107.949533 33.809665)", elevation=2464.9, release_status="draft", - elevation_accuracy=100, - elevation_method="Survey-grade GPS", - coordinate_accuracy=50, - coordinate_method="GPS, uncorrected", + # elevation_accuracy=100, + # elevation_method="Survey-grade GPS", + # coordinate_accuracy=50, + # coordinate_method="GPS, uncorrected", ) session.add(loc) session.commit() @@ -294,6 +295,36 @@ def add_id_link( return id_link +@add_context_object_container("data_provenance") +def add_data_provenance( + context, + session, + target_id, + target_table, + field_name, + origin_source, + collection_method=None, + accuracy_value=None, + accuracy_unit=None, +): + data_provenance = DataProvenance( + field_name=field_name, + collection_method=collection_method, + target_id=target_id, + target_table=target_table, + origin_source=origin_source, + accuracy_value=accuracy_value, + accuracy_unit=accuracy_unit, + ) + + session.add(data_provenance) + session.commit() + session.refresh(data_provenance) + + context.objects["data_provenance"].append(data_provenance) + return data_provenance + + @add_context_object_container("transducer_observations") def add_transducer_observation(context, session, block, deployment_id, value): obs = TransducerObservation( @@ -428,6 +459,25 @@ def before_all(context): group = add_group(context, session, [well_1, well_2]) + elevation_method = add_data_provenance( + context, + session, + target_id=loc_1.id, + target_table="location", + field_name="elevation", + origin_source="Private geologist, consultant or univ associate", + collection_method="LiDAR DEM", + ) + + well_depth_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_depth", + origin_source="Other", + ) + for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) @@ -441,8 +491,9 @@ def before_all(context): session.commit() - # the well needs to be refreshed to get all the new relationships + # the following needs to be refreshed to get all the new relationships session.refresh(well_1) + session.refresh(loc_1) def after_all(context): diff --git a/tests/features/steps/common.py b/tests/features/steps/common.py index e724a6016..ccfe3b79f 100644 --- a/tests/features/steps/common.py +++ b/tests/features/steps/common.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from behave import then, given +from behave import then, given, when from starlette.testclient import TestClient from core.dependencies import ( @@ -65,6 +65,25 @@ def closure(): assert context.client is not None, "TestClient failed to initialize" +@when("the user retrieves the well by ID via path parameter") +def step_impl(context): + context.response = context.client.get( + f"thing/water-well/{context.objects['wells'][0].id}" + ) + context.water_well_data = context.response.json() + context.notes = {} + + +@then( + "null values in the response should be represented as JSON null (not placeholder strings)" +) +def step_impl(context): + data = context.response.json() + for k, v in data.items(): + if v == "": + assert v is None, f"Value for key {k} is an empty string but should be null" + + @then("I should receive a successful response") def step_impl(context): assert ( diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index c4f235135..b0adc8346 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -2,35 +2,18 @@ from services.util import ( transform_srid, convert_m_to_ft, - retrieve_latest_polymorphic_table_record, + retrieve_latest_polymorphic_history_table_record, ) -from behave import when, then +from behave import then from geoalchemy2.shape import to_shape -# TODO: move to commonly used step definitions -@when("the user retrieves the well by ID via path parameter") -def step_impl(context): - well_id = context.objects["wells"][0].id - context.response = context.client.get(f"/thing/water-well/{well_id}") - context.water_well_data = context.response.json() - - @then("the response should be in JSON format") def step_impl(context): assert context.response["Content-Type"] == "application/json" -@then( - "null values in the response should be represented as JSON null (not placeholder strings)" -) -def step_impl(context): - for key, value in context.water_well_data.items(): - if value is None: - assert value is None # JSON null is represented as None in Python - - # ------------------------------------------------------------------------------ # Well names and projects # ------------------------------------------------------------------------------ @@ -93,7 +76,7 @@ def step_impl(context): def step_impl(context): assert "well_status" in context.water_well_data - well_status_record = retrieve_latest_polymorphic_table_record( + well_status_record = retrieve_latest_polymorphic_history_table_record( context.objects["wells"][0], "status_history", "Well Status" ) assert context.water_well_data["well_status"] == well_status_record.status_value @@ -117,7 +100,7 @@ def step_impl(context): def step_impl(context): assert "monitoring_status" in context.water_well_data - monitoring_status_record = retrieve_latest_polymorphic_table_record( + monitoring_status_record = retrieve_latest_polymorphic_history_table_record( context.objects["wells"][0], "status_history", "Monitoring Status" ) assert ( @@ -168,15 +151,21 @@ def step_impl(context): assert context.water_well_data["well_depth_unit"] == "ft" -# TODO: this needs to be added to the model, schema, and test data @then("the response should include the source of the well depth information") def step_impl(context): assert "well_depth_source" in context.water_well_data - assert ( - context.water_well_data["well_depth_source"] - == context.objects["wells"][0].well_depth_source - ) + data_provenance_records = context.objects["data_provenance"] + well_depth_source_records = [ + r + for r in data_provenance_records + if r.field_name == "well_depth" + and r.target_table == "thing" + and r.target_id == context.objects["wells"][0].id + ] + well_depth_source = well_depth_source_records[0].origin_source + + assert context.water_well_data["well_depth_source"] == well_depth_source # ------------------------------------------------------------------------------ @@ -271,9 +260,19 @@ def step_impl(context): assert ( "elevation_method" in context.water_well_data["current_location"]["properties"] ) + + data_provenance_records = context.objects["data_provenance"] + elevation_method_records = [ + r + for r in data_provenance_records + if r.field_name == "elevation" + and r.target_table == "location" + and r.target_id == context.objects["locations"][0].id + ] + elevation_method = elevation_method_records[0].collection_method assert ( context.water_well_data["current_location"]["properties"]["elevation_method"] - == context.objects["locations"][0].elevation_method + == elevation_method ) diff --git a/tests/features/steps/well-notes.py b/tests/features/steps/well-notes.py index bb8943b8b..d5e4c75d2 100644 --- a/tests/features/steps/well-notes.py +++ b/tests/features/steps/well-notes.py @@ -33,24 +33,6 @@ def step_impl(context): assert note, f"{k} Note is empty" -@when("the user retrieves the well by ID via path parameter") -def step_impl(context): - context.response = context.client.get( - f"thing/water-well/{context.objects['wells'][0].id}" - ) - context.notes = {} - - -@then( - "null values in the response should be represented as JSON null (not placeholder strings)" -) -def step_impl(context): - data = context.response.json() - for k, v in data.items(): - if v == "": - assert v is None, f"Value for key {k} is an empty string but should be null" - - @then( "the response should include location notes (i.e. driving directions and geographic well location notes)" )