From 839c9240e18cb77e717f6d67095eb55beac65d1b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 15:17:02 -0600 Subject: [PATCH 01/37] refactor: type-hint observation table --- db/observation.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/db/observation.py b/db/observation.py index 1b49196ef..70d43c668 100644 --- a/db/observation.py +++ b/db/observation.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from datetime import datetime from sqlalchemy import ( ForeignKey, - Integer, - Float, DateTime, ) -from sqlalchemy.orm import mapped_column, relationship +from sqlalchemy.orm import mapped_column, relationship, Mapped from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -27,47 +26,45 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} - sample_id = mapped_column( - Integer, + # NM_Aquifer fields for audits + nma_pk_waterlevel: Mapped[str] = mapped_column(nullable=True) + + sample_id: Mapped[int] = mapped_column( ForeignKey("sample.id", ondelete="CASCADE"), nullable=False, ) - sensor_id = mapped_column( - Integer, + sensor_id: Mapped[int] = mapped_column( ForeignKey("sensor.id", ondelete="CASCADE"), nullable=False, ) - observation_datetime = mapped_column( + observation_datetime: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, doc="Timestamp of the observation" ) - observed_property = lexicon_term() - value = mapped_column( - Float, + observed_property: Mapped[str] = lexicon_term(nullable=False) + value: Mapped[float] = mapped_column( nullable=True, ) - unit = lexicon_term() + unit: Mapped[str] = lexicon_term(nullable=False) # groundwater - measuring_point_height = mapped_column( - Float, + measuring_point_height: Mapped[float] = mapped_column( nullable=True, doc="Height of the measuring point above the ground surface in ft", info={"unit": "ft"}, ) - level_status = lexicon_term() + level_status: Mapped[str] = lexicon_term(nullable=True) # geothermal - observation_depth = mapped_column( - Float, + observation_depth: Mapped[float] = mapped_column( nullable=True, info={"unit": "feet"}, doc="Depth of the geothermal observation in feet", ) - sensor = relationship("Sensor") - sample = relationship("Sample") + sensor: Mapped["Sensor"] = relationship("Sensor") # noqa: F821 + sample: Mapped["Sample"] = relationship("Sample") # noqa: F821 # ============= EOF ============================================= From fb8dbc560f27dc76f17018c18f71663dacf135d9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 15:30:05 -0600 Subject: [PATCH 02/37] feat: update waterlevels transfer for model revisions --- transfers/waterlevels_transfer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 53eff8d0c..aba14063e 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -78,11 +78,10 @@ def transfer_water_levels(session): # TODO: this needs to be resolved obs.sensor_id = 1 - # TODO: this needs to be implemented - # obs.nma_pk_observation = row.GlobalID + obs.nma_pk_waterlevels = row.GlobalID obs.sample = sample - obs.observation_datetime = dt + obs.observation_datetime = dt_utc obs.value = row.DepthToWater obs.measuring_point_height = row.MPHeight obs.observed_property = "groundwater level:groundwater level" From 4f90b5ecce364569b3092e07157c07af2c04574b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 16:11:58 -0600 Subject: [PATCH 03/37] feat: rename measurement_method to sample_method for new style --- core/lexicon.json | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 1cd498752..39d17e91e 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -288,29 +288,29 @@ {"categories": [{"name": "status", "description": null}], "term": "Destroyed, exists but not usable", "definition": "Destroyed, exists but not usable"}, {"categories": [{"name": "status", "description": null}], "term": "Inactive, exists but not used", "definition": "Inactive, exists but not used"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Airline measurement", "definition": "Airline measurement"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Analog or graphic recorder", "definition": "Analog or graphic recorder"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Calibrated airline measurement", "definition": "Calibrated airline measurement"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Differential GPS; especially applicable to surface expression of ground water", "definition": "Differential GPS; especially applicable to surface expression of ground water"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Estimated", "definition": "Estimated"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Transducer", "definition": "Transducer"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Pressure-gage measurement", "definition": "Pressure-gage measurement"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Calibrated pressure-gage measurement", "definition": "Calibrated pressure-gage measurement"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Interpreted from geophysical logs", "definition": "Interpreted from geophysical logs"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Manometer", "definition": "Manometer"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Non-recording gage", "definition": "Non-recording gage"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Observed (required for F, N, and W water level status)", "definition": "Observed (required for F, N, and W water level status)"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Sonic water level meter (acoustic pulse)", "definition": "Sonic water level meter (acoustic pulse)"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Reported, method not known", "definition": "Reported, method not known"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Steel-tape measurement", "definition": "Steel-tape measurement"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Electric tape measurement (E-probe)", "definition": "Electric tape measurement (E-probe)"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Unknown (for legacy data only; not for new data entry)", "definition": "Unknown (for legacy data only; not for new data entry)"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Calibrated electric tape; accuracy of equipment has been checked", "definition": "Calibrated electric tape; accuracy of equipment has been checked"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Calibrated electric cable", "definition": "Calibrated electric cable"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Uncalibrated electric cable", "definition": "Uncalibrated electric cable"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Continuous acoustic sounder", "definition": "Continuous acoustic sounder"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "Measurement not attempted", "definition": "Measurement not attempted"}, - {"categories": [{"name": "measurement_method", "description": null}], "term": "null placeholder", "definition": "null placeholder"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Airline measurement", "definition": "Airline measurement"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Analog or graphic recorder", "definition": "Analog or graphic recorder"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Calibrated airline measurement", "definition": "Calibrated airline measurement"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Differential GPS; especially applicable to surface expression of ground water", "definition": "Differential GPS; especially applicable to surface expression of ground water"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Estimated", "definition": "Estimated"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Transducer", "definition": "Transducer"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Pressure-gage measurement", "definition": "Pressure-gage measurement"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Calibrated pressure-gage measurement", "definition": "Calibrated pressure-gage measurement"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Interpreted from geophysical logs", "definition": "Interpreted from geophysical logs"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Manometer", "definition": "Manometer"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Non-recording gage", "definition": "Non-recording gage"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Observed (required for F, N, and W water level status)", "definition": "Observed (required for F, N, and W water level status)"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Sonic water level meter (acoustic pulse)", "definition": "Sonic water level meter (acoustic pulse)"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Reported, method not known", "definition": "Reported, method not known"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Steel-tape measurement", "definition": "Steel-tape measurement"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Electric tape measurement (E-probe)", "definition": "Electric tape measurement (E-probe)"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Unknown (for legacy data only; not for new data entry)", "definition": "Unknown (for legacy data only; not for new data entry)"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Calibrated electric tape; accuracy of equipment has been checked", "definition": "Calibrated electric tape; accuracy of equipment has been checked"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Calibrated electric cable", "definition": "Calibrated electric cable"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Uncalibrated electric cable", "definition": "Uncalibrated electric cable"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Continuous acoustic sounder", "definition": "Continuous acoustic sounder"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "Measurement not attempted", "definition": "Measurement not attempted"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "null placeholder", "definition": "null placeholder"}, {"categories": [{"name": "organization", "description": null}], "term": "USGS", "definition": "US Geological Survey"}, {"categories": [{"name": "organization", "description": null}], "term": "TWDB", "definition": "Texas Water Development Board"}, From 6dc2d7acf39e47aa265a3fb2b4c9e513d1a0ce36 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 11:23:14 -0600 Subject: [PATCH 04/37] refactor: use sample_type instead of observation_class --- api/observation.py | 8 +- core/lexicon.json | 252 ++++++++++++++++++--------------- db/observation.py | 2 +- db/sample.py | 31 ++-- schemas/observation.py | 22 +-- services/observation_helper.py | 45 +++--- tests/conftest.py | 166 ++++++++++++++-------- tests/test_observation.py | 83 +++++------ 8 files changed, 327 insertions(+), 282 deletions(-) diff --git a/api/observation.py b/api/observation.py index 27da81974..cb53befd5 100644 --- a/api/observation.py +++ b/api/observation.py @@ -43,7 +43,7 @@ from services.observation_helper import ( get_observations, observation_model_patcher, - get_observation_of_an_observation_class_by_id, + get_observation_of_a_sample_type_by_id, ) router = APIRouter(prefix="/observation", tags=["observation"]) @@ -177,7 +177,7 @@ async def get_groundwater_level_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> GroundwaterLevelObservationResponse: - return get_observation_of_an_observation_class_by_id( + return get_observation_of_a_sample_type_by_id( session=session, request=request, observation_id=observation_id, @@ -224,7 +224,7 @@ async def get_water_chemistry_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> WaterChemistryObservationResponse: - return get_observation_of_an_observation_class_by_id( + return get_observation_of_a_sample_type_by_id( session=session, request=request, observation_id=observation_id, @@ -269,7 +269,7 @@ async def get_geothermal_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> GeothermalObservationResponse: - return get_observation_of_an_observation_class_by_id( + return get_observation_of_a_sample_type_by_id( session=session, request=request, observation_id=observation_id ) diff --git a/core/lexicon.json b/core/lexicon.json index 39d17e91e..3b06b1e08 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -118,123 +118,123 @@ {"categories": [{"name": "unit", "description": null}], "term": "deg second", "definition": "degree second"}, {"categories": [{"name": "unit", "description": null}], "term": "deg minute", "definition": "degree minute"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "groundwater level:groundwater level", "definition": "groundwater level measurement" }, - {"categories": [{"name": "observed_property", "description": null}], "term": "geothermal:temperature", "definition": "Temperature measurement"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:pH", "definition": "pH"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Alkalinity, Total", "definition": "Alkalinity, Total"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Alkalinity as OH-", "definition": "Alkalinity as OH-"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Calcium", "definition": "Calcium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Calcium, total, unfiltered", "definition": "Calcium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chloride", "definition": "Chloride"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Carbonate", "definition": "Carbonate"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Conductivity, laboratory", "definition": "Conductivity, laboratory"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Bicarbonate", "definition": "Bicarbonate"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Hardness (CaCO3)", "definition": "Hardness (CaCO3)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Ion Balance", "definition": "Ion Balance"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Potassium", "definition": "Potassium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Potassium, total, unfiltered", "definition": "Potassium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Magnesium", "definition": "Magnesium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Magnesium, total, unfiltered", "definition": "Magnesium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sodium", "definition": "Sodium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sodium, total, unfiltered", "definition": "Sodium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sodium and Potassium combined", "definition": "Sodium and Potassium combined"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sulfate", "definition": "Sulfate"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total Anions", "definition": "Total Anions"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total Cations", "definition": "Total Cations"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total Dissolved Solids", "definition": "Total Dissolved Solids"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Tritium", "definition": "Tritium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Age of Water using dissolved gases", "definition": "Age of Water using dissolved gases"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Silver", "definition": "Silver"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Silver, total, unfiltered", "definition": "Silver, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Aluminum", "definition": "Aluminum"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Aluminum, total, unfiltered", "definition": "Aluminum, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Arsenic", "definition": "Arsenic"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Arsenic, total, unfiltered", "definition": "Arsenic, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Boron", "definition": "Boron"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Boron, total, unfiltered", "definition": "Boron, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Barium", "definition": "Barium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Barium, total, unfiltered", "definition": "Barium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Beryllium", "definition": "Beryllium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Beryllium, total, unfiltered", "definition": "Beryllium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Bromide", "definition": "Bromide"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:13C:12C ratio", "definition": "13C:12C ratio"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:14C content, pmc", "definition": "14C content, pmc"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Uncorrected C14 age", "definition": "Uncorrected C14 age"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Cadmium", "definition": "Cadmium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Cadmium, total, unfiltered", "definition": "Cadmium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chlorofluorocarbon-11 avg age", "definition": "Chlorofluorocarbon-11 avg age"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chlorofluorocarbon-113 avg age", "definition": "Chlorofluorocarbon-113 avg age"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chlorofluorocarbon-113/12 avg RATIO age", "definition": "Chlorofluorocarbon-113/12 avg RATIO age"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chlorofluorocarbon-12 avg age", "definition": "Chlorofluorocarbon-12 avg age"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Cobalt", "definition": "Cobalt"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Cobalt, total, unfiltered", "definition": "Cobalt, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chromium", "definition": "Chromium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Chromium, total, unfiltered", "definition": "Chromium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Copper", "definition": "Copper"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Copper, total, unfiltered", "definition": "Copper, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:delta O18 sulfate", "definition": "delta O18 sulfate"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sulfate 34 isotope ratio", "definition": "Sulfate 34 isotope ratio"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Fluoride", "definition": "Fluoride"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Iron", "definition": "Iron"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Iron, total, unfiltered", "definition": "Iron, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Deuterium:Hydrogen ratio", "definition": "Deuterium:Hydrogen ratio"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Mercury", "definition": "Mercury"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Mercury, total, unfiltered", "definition": "Mercury, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Lithium", "definition": "Lithium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Lithium, total, unfiltered", "definition": "Lithium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Manganese", "definition": "Manganese"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Manganese, total, unfiltered", "definition": "Manganese, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Molybdenum", "definition": "Molybdenum"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Molybdenum, total, unfiltered", "definition": "Molybdenum, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nickel", "definition": "Nickel"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nickel, total, unfiltered", "definition": "Nickel, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nitrite (as NO2)", "definition": "Nitrite (as NO2)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nitrite (as N)", "definition": "Nitrite (as N)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nitrate (as NO3)", "definition": "Nitrate (as NO3)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Nitrate (as N)", "definition": "Nitrate (as N)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:18O:16O ratio", "definition": "18O:16O ratio"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Lead", "definition": "Lead"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Lead, total, unfiltered", "definition": "Lead, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Phosphate", "definition": "Phosphate"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Antimony", "definition": "Antimony"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Antimony, total, unfiltered", "definition": "Antimony, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Selenium", "definition": "Selenium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Selenium, total, unfiltered", "definition": "Selenium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Sulfur hexafluoride", "definition": "Sulfur hexafluoride"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Silicon", "definition": "Silicon"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Silicon, total, unfiltered", "definition": "Silicon, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Silica", "definition": "Silica"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Tin", "definition": "Tin"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Tin, total, unfiltered", "definition": "Tin, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Strontium", "definition": "Strontium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Strontium, total, unfiltered", "definition": "Strontium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Strontium 87:86 ratio", "definition": "Strontium 87:86 ratio"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Thorium", "definition": "Thorium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Thorium, total, unfiltered", "definition": "Thorium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Titanium", "definition": "Titanium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Titanium, total, unfiltered", "definition": "Titanium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Thallium", "definition": "Thallium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Thallium, total, unfiltered", "definition": "Thallium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Uranium (total, by ICP-MS)", "definition": "Uranium (total, by ICP-MS)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Uranium, total, unfiltered", "definition": "Uranium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Vanadium", "definition": "Vanadium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Vanadium, total, unfiltered", "definition": "Vanadium, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Zinc", "definition": "Zinc"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Zinc, total, unfiltered", "definition": "Zinc, total, unfiltered"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Corrected C14 in years", "definition": "Corrected C14 in years"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Arsenite (arsenic species)", "definition": "Arsenite (arsenic species)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Arsenate (arsenic species)", "definition": "Arsenate (arsenic species)"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Cyanide", "definition": "Cyanide"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Estimated recharge temperature", "definition": "Estimated recharge temperature"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Hydrogen sulfide", "definition": "Hydrogen sulfide"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Ammonia", "definition": "Ammonia"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Ammonium", "definition": "Ammonium"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total nitrogen", "definition": "Total nitrogen"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total Kjeldahl nitrogen", "definition": "Total Kjeldahl nitrogen"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Dissolved organic carbon", "definition": "Dissolved organic carbon"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:Total organic carbon", "definition": "Total organic carbon"}, - {"categories": [{"name": "observed_property", "description": null}], "term": "water chemistry:delta C13 of dissolved inorganic carbon", "definition": "delta C13 of dissolved inorganic carbon"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "groundwater level", "definition": "groundwater level measurement" }, + {"categories": [{"name": "observed_property", "description": null}], "term": "temperature", "definition": "Temperature measurement"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "pH", "definition": "pH"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Alkalinity, Total", "definition": "Alkalinity, Total"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Alkalinity as OH-", "definition": "Alkalinity as OH-"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Calcium", "definition": "Calcium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Calcium, total, unfiltered", "definition": "Calcium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chloride", "definition": "Chloride"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Carbonate", "definition": "Carbonate"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Conductivity, laboratory", "definition": "Conductivity, laboratory"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Bicarbonate", "definition": "Bicarbonate"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Hardness (CaCO3)", "definition": "Hardness (CaCO3)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Ion Balance", "definition": "Ion Balance"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Potassium", "definition": "Potassium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Potassium, total, unfiltered", "definition": "Potassium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Magnesium", "definition": "Magnesium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Magnesium, total, unfiltered", "definition": "Magnesium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sodium", "definition": "Sodium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sodium, total, unfiltered", "definition": "Sodium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sodium and Potassium combined", "definition": "Sodium and Potassium combined"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sulfate", "definition": "Sulfate"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total Anions", "definition": "Total Anions"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total Cations", "definition": "Total Cations"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total Dissolved Solids", "definition": "Total Dissolved Solids"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Tritium", "definition": "Tritium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Age of Water using dissolved gases", "definition": "Age of Water using dissolved gases"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Silver", "definition": "Silver"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Silver, total, unfiltered", "definition": "Silver, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Aluminum", "definition": "Aluminum"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Aluminum, total, unfiltered", "definition": "Aluminum, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Arsenic", "definition": "Arsenic"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Arsenic, total, unfiltered", "definition": "Arsenic, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Boron", "definition": "Boron"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Boron, total, unfiltered", "definition": "Boron, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Barium", "definition": "Barium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Barium, total, unfiltered", "definition": "Barium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Beryllium", "definition": "Beryllium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Beryllium, total, unfiltered", "definition": "Beryllium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Bromide", "definition": "Bromide"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "13C:12C ratio", "definition": "13C:12C ratio"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "14C content, pmc", "definition": "14C content, pmc"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Uncorrected C14 age", "definition": "Uncorrected C14 age"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Cadmium", "definition": "Cadmium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Cadmium, total, unfiltered", "definition": "Cadmium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chlorofluorocarbon-11 avg age", "definition": "Chlorofluorocarbon-11 avg age"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chlorofluorocarbon-113 avg age", "definition": "Chlorofluorocarbon-113 avg age"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chlorofluorocarbon-113/12 avg RATIO age", "definition": "Chlorofluorocarbon-113/12 avg RATIO age"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chlorofluorocarbon-12 avg age", "definition": "Chlorofluorocarbon-12 avg age"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Cobalt", "definition": "Cobalt"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Cobalt, total, unfiltered", "definition": "Cobalt, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chromium", "definition": "Chromium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Chromium, total, unfiltered", "definition": "Chromium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Copper", "definition": "Copper"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Copper, total, unfiltered", "definition": "Copper, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "delta O18 sulfate", "definition": "delta O18 sulfate"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sulfate 34 isotope ratio", "definition": "Sulfate 34 isotope ratio"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Fluoride", "definition": "Fluoride"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Iron", "definition": "Iron"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Iron, total, unfiltered", "definition": "Iron, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Deuterium:Hydrogen ratio", "definition": "Deuterium:Hydrogen ratio"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Mercury", "definition": "Mercury"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Mercury, total, unfiltered", "definition": "Mercury, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Lithium", "definition": "Lithium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Lithium, total, unfiltered", "definition": "Lithium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Manganese", "definition": "Manganese"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Manganese, total, unfiltered", "definition": "Manganese, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Molybdenum", "definition": "Molybdenum"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Molybdenum, total, unfiltered", "definition": "Molybdenum, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nickel", "definition": "Nickel"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nickel, total, unfiltered", "definition": "Nickel, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nitrite (as NO2)", "definition": "Nitrite (as NO2)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nitrite (as N)", "definition": "Nitrite (as N)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nitrate (as NO3)", "definition": "Nitrate (as NO3)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Nitrate (as N)", "definition": "Nitrate (as N)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "18O:16O ratio", "definition": "18O:16O ratio"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Lead", "definition": "Lead"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Lead, total, unfiltered", "definition": "Lead, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Phosphate", "definition": "Phosphate"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Antimony", "definition": "Antimony"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Antimony, total, unfiltered", "definition": "Antimony, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Selenium", "definition": "Selenium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Selenium, total, unfiltered", "definition": "Selenium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Sulfur hexafluoride", "definition": "Sulfur hexafluoride"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Silicon", "definition": "Silicon"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Silicon, total, unfiltered", "definition": "Silicon, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Silica", "definition": "Silica"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Tin", "definition": "Tin"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Tin, total, unfiltered", "definition": "Tin, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Strontium", "definition": "Strontium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Strontium, total, unfiltered", "definition": "Strontium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Strontium 87:86 ratio", "definition": "Strontium 87:86 ratio"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Thorium", "definition": "Thorium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Thorium, total, unfiltered", "definition": "Thorium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Titanium", "definition": "Titanium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Titanium, total, unfiltered", "definition": "Titanium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Thallium", "definition": "Thallium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Thallium, total, unfiltered", "definition": "Thallium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Uranium (total, by ICP-MS)", "definition": "Uranium (total, by ICP-MS)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Uranium, total, unfiltered", "definition": "Uranium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Vanadium", "definition": "Vanadium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Vanadium, total, unfiltered", "definition": "Vanadium, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Zinc", "definition": "Zinc"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Zinc, total, unfiltered", "definition": "Zinc, total, unfiltered"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Corrected C14 in years", "definition": "Corrected C14 in years"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Arsenite (arsenic species)", "definition": "Arsenite (arsenic species)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Arsenate (arsenic species)", "definition": "Arsenate (arsenic species)"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Cyanide", "definition": "Cyanide"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Estimated recharge temperature", "definition": "Estimated recharge temperature"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Hydrogen sulfide", "definition": "Hydrogen sulfide"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Ammonia", "definition": "Ammonia"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Ammonium", "definition": "Ammonium"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total nitrogen", "definition": "Total nitrogen"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total Kjeldahl nitrogen", "definition": "Total Kjeldahl nitrogen"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Dissolved organic carbon", "definition": "Dissolved organic carbon"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "Total organic carbon", "definition": "Total organic carbon"}, + {"categories": [{"name": "observed_property", "description": null}], "term": "delta C13 of dissolved inorganic carbon", "definition": "delta C13 of dissolved inorganic carbon"}, {"categories": [{"name": "release_status", "description": null}], "term": "draft", "definition": "draft version"}, @@ -251,7 +251,23 @@ {"categories": [{"name": "relation", "description": null}], "term": "OSEPOD", "definition": "NM OSE 'Point of Diversion' ID"}, {"categories": [{"name": "relation", "description": null}], "term": "PLSS", "definition": "Public Land Survey System ID"}, - {"categories": [{"name": "sample_type", "description": null}], "term": "groundwater", "definition": "groundwater sample from a well"}, + {"categories": [{"name": "sample_type", "description": null}], "term": "groundwater level", "definition": "groundwater level"}, + {"categories": [{"name": "sample_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, + {"categories": [{"name": "sample_type", "description": null}], "term": "geothermal", "definition": "geothermal"}, + + {"categories": [{"name": "sample_matrix", "description": null}], "term": "groundwater", "definition": "groundwater"}, + + {"categories": [{"name": "thing_type", "description": null}], "term": "observation well", "definition": "a well used to monitor groundwater levels"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "piezometer", "definition": "a type of observation well that measures pressure head in the aquifer"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "monitoring well", "definition": "a well used to monitor groundwater quality or levels"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "production well", "definition": "a well used to extract groundwater for use"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "injection well", "definition": "a well used to inject water or other fluids into the ground"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "exploration well", "definition": "a well drilled to explore for groundwater or other resources"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "test well", "definition": "a well drilled to test the properties of the aquifer"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "abandoned well", "definition": "a well that is no longer in use and has been properly sealed"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "dry hole", "definition": "a well that did not produce water or other resources"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "artesian well", "definition": "a well that taps a confined aquifer where the water level is above the top of the aquifer"}, + {"categories": [{"name": "thing_type", "description": null}], "term": "dug well", "definition": "a shallow well dug by hand or with machinery, typically lined with stones or bricks"}, {"categories": [{"name": "thing_type", "description": null}], "term": "water well", "definition": "a hole drill into the ground to access groundwater"}, {"categories": [{"name": "thing_type", "description": null}], "term": "spring", "definition": "a natural discharge of groundwater at the surface"}, diff --git a/db/observation.py b/db/observation.py index 70d43c668..15138d4d1 100644 --- a/db/observation.py +++ b/db/observation.py @@ -35,7 +35,7 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): ) sensor_id: Mapped[int] = mapped_column( ForeignKey("sensor.id", ondelete="CASCADE"), - nullable=False, + nullable=True, ) observation_datetime: Mapped[datetime] = mapped_column( diff --git a/db/sample.py b/db/sample.py index 1e6c749b8..4e7a2a4e8 100644 --- a/db/sample.py +++ b/db/sample.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from sqlalchemy import DateTime, ForeignKey, UniqueConstraint, Float +from sqlalchemy import DateTime, ForeignKey, UniqueConstraint from sqlalchemy.orm import mapped_column, relationship, Mapped # import models from classes that are defined in separate files -from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term from db.thing import Thing from db.sensor import Sensor @@ -52,39 +52,36 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Date and time of sample collection.", ) - # REFACTOR TODO: update with enum/restricted values - sample_matrix: Mapped[Optional[str]] = mapped_column( - comment="The material of the sample (e.g., 'gw', 'soil')." + sample_matrix: Mapped[str] = lexicon_term( + nullable=False, comment="The material of the sample (e.g., 'gw', 'soil')." ) - # REFACTOR TODO: update with enum/restricted values - sample_method: Mapped[Optional[str]] = mapped_column( - comment="Method used to collect the sample." + sample_method: Mapped[str] = lexicon_term( + comment="Method used to collect the sample.", nullable=True ) field_sample_id: Mapped[str] = mapped_column( unique=True, nullable=False, comment="User-defined ID for field tracking." ) - # REFACTOR TODO: update with enum/restricted values - sampler_name: Mapped[Optional[str]] = mapped_column( + sampler_name: Mapped[str] = mapped_column( nullable=False, comment="Name of the person who collected the sample." ) - # REFACTOR TODO: update with enum/restricted values qc_sample: Mapped[str] = mapped_column( default="Original", nullable=False, comment="Quality control sample type (e.g., 'Original', 'field dupe').", ) - sample_top: Mapped[Optional[float]] = mapped_column( - Float, comment="Top depth of a discrete sample interval." + sample_top: Mapped[float] = mapped_column( + nullable=True, comment="Top depth of a discrete sample interval." ) - sample_bottom: Mapped[Optional[float]] = mapped_column( - Float, comment="Bottom depth of a discrete sample interval." + sample_bottom: Mapped[float] = mapped_column( + nullable=True, comment="Bottom depth of a discrete sample interval." ) duplicate_sample_number: Mapped[int] = mapped_column( default=0, comment="Identifier for duplicate samples (0 = original sample, not a duplicate, 1 = dup no.1, 2 = dup no.2, etc.).", ) - sample_type: Mapped[str] = mapped_column( - comment="The type of sample (e.g., 'geochemical', 'geothermal', 'groundwater')." + sample_type: Mapped[str] = lexicon_term( + nullable=False, + comment="The type of sample (e.g., 'geochemical', 'geothermal', 'groundwater').", ) # --- Relationship Definitions --- diff --git a/schemas/observation.py b/schemas/observation.py index a5269f91d..24b3b08d3 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -36,7 +36,6 @@ class ValidateObservation(BaseModel): - _observation_class: str observed_property: str observation_datetime: AwareDatetime @@ -55,21 +54,11 @@ def convert_observation_datetime_to_utc( return observation_datetime.astimezone(timezone.utc) return observation_datetime - @model_validator(mode="after") - def prepend_observed_property(self: Self) -> Self: - observed_property = self.observed_property - observation_class = self._observation_class - if observed_property is not None: - observation_class = self._observation_class - if not observed_property.startswith(f"{observation_class}:"): - self.observed_property = f"{observation_class}:{observed_property}" - return self - # -------- CREATE ---------- class CreateBaseObservation(BaseCreateModel, ValidateObservation): observation_datetime: Annotated[AwareDatetime, PastDatetime()] - sample_id: int | None = None + sample_id: int sensor_id: int observed_property: str release_status: str @@ -78,17 +67,15 @@ class CreateBaseObservation(BaseCreateModel, ValidateObservation): class CreateGroundwaterLevelObservation(CreateBaseObservation): - _observation_class: str = "groundwater level" measuring_point_height: float level_status: str class CreateWaterChemistryObservation(CreateBaseObservation): - _observation_class: str = "water chemistry" + pass class CreateGeothermalObservation(CreateBaseObservation): - _observation_class: str = "geothermal" observation_depth: float @@ -130,11 +117,6 @@ class BaseObservationResponse(BaseResponseModel): value: float | None unit: str - @field_validator("observed_property") - def remove_observed_property_prefix(cls, v: str) -> str: - colon_index = v.find(":") - return v[colon_index + 1 :] - class GroundwaterLevelObservationResponse(BaseObservationResponse): depth_to_water_bgs: float | None diff --git a/services/observation_helper.py b/services/observation_helper.py index 8f4561c32..2c74c0ae5 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -19,18 +19,18 @@ from services.query_helper import simple_get_by_id, order_sort_filter -def get_observation_class_from_request(request: Request) -> str: +def get_sample_type_from_request(request: Request) -> str: path = request.url.path path_components = path.split("/") if len(path_components) == 2: - # no observation class specified in path - observation_class_in_path = path_components[1] + # no sample type specified in path + sample_type_in_path = path_components[1] if len(path_components) >= 3: - # observation class specified in path - observation_class_in_path = path_components[2] + # sample type specified in path + sample_type_in_path = path_components[2] - observation_class = observation_class_in_path.replace("-", " ") - return observation_class + sample_type = sample_type_in_path.replace("-", " ") + return sample_type def get_observations( @@ -53,11 +53,13 @@ def get_observations( """ Retrieve all observations """ - observation_class = get_observation_class_from_request(request) + sample_table_is_joined = False + sample_type = get_sample_type_from_request(request) sql = select(Observation) if thing_id is not None: - sql = sql.join(Sample) + sample_table_is_joined = True + sql = sql.join(Sample, Sample.id == Observation.sample_id) sql = sql.where(Sample.thing_id == thing_id) if sample_id is not None: sql = sql.where(Observation.sample_id == sample_id) @@ -70,8 +72,10 @@ def get_observations( sql = sql.where(Observation.observation_datetime <= end_time) # root of path is /observation - if observation_class != "observation": - sql = sql.where(Observation.observed_property.like(f"{observation_class}:%")) + if sample_type != "observation": + if sample_table_is_joined is False: + sql = sql.join(Sample, Sample.id == Observation.sample_id) + sql = sql.where(Sample.sample_type == sample_type) sql = order_sort_filter(sql, Observation, sort, order, filter_) @@ -81,16 +85,13 @@ def get_observations( return paginate(query=sql, conn=session) -def verify_observed_property_corresponds_with_observation_class( +def verify_observed_property_corresponds_with_sample_type( observation: Observation, request: Request ): - observation_class = get_observation_class_from_request(request) + requested_sample_type = get_sample_type_from_request(request) + actual_sample_type = observation.sample.sample_type - observed_property = observation.observed_property - colon_index = observed_property.find(":") - actual_observation_class = observed_property[:colon_index] - - if actual_observation_class != observation_class: + if actual_sample_type != requested_sample_type: raise PydanticStyleException( status_code=HTTP_404_NOT_FOUND, detail=[ @@ -98,13 +99,13 @@ def verify_observed_property_corresponds_with_observation_class( "loc": ["path", "observation_id"], "type": "value_error", "input": {"observation_id": observation.id}, - "msg": f"Observation with ID {observation.id} is not a {observation_class} observation. It is a {actual_observation_class} observation.", + "msg": f"Observation with ID {observation.id} is not a {requested_sample_type} observation. It is a {actual_sample_type} observation.", } ], ) -def get_observation_of_an_observation_class_by_id( +def get_observation_of_a_sample_type_by_id( session: Session, request: Request, observation_id: int ) -> Observation: """ @@ -112,7 +113,7 @@ def get_observation_of_an_observation_class_by_id( """ observation = simple_get_by_id(session, Observation, observation_id) - verify_observed_property_corresponds_with_observation_class(observation, request) + verify_observed_property_corresponds_with_sample_type(observation, request) return observation @@ -130,7 +131,7 @@ def observation_model_patcher( # simple_get_by_id raises HTTP_404_NOT_FOUND if the item is not found observation = simple_get_by_id(session, Observation, observation_id) - verify_observed_property_corresponds_with_observation_class(observation, request) + verify_observed_property_corresponds_with_sample_type(observation, request) for key, value in payload.model_dump(exclude_unset=True).items(): setattr(observation, key, value) diff --git a/tests/conftest.py b/tests/conftest.py index 2e14dbdc2..aec663d74 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,54 +219,6 @@ def second_sensor(): session.commit() -@pytest.fixture(scope="session") -def sample(water_well_thing, sensor): - with session_ctx() as session: - sample = Sample( - sample_date="2025-01-01T00:00:00Z", - thing_id=water_well_thing.id, - sample_type="groundwater", - sampler_name="Test Sampler", - release_status="draft", - field_sample_id=f"FS-{uuid.uuid4()}", - qc_sample="Original", - sensor_id=sensor.id, - sample_matrix="water", - sample_method="manual", - duplicate_sample_number=0, - sample_top=None, - sample_bottom=None, - ) - session.add(sample) - session.commit() - yield sample - - -@pytest.fixture(scope="function") -def second_sample(water_well_thing, sensor): - with session_ctx() as session: - sample = Sample( - thing_id=water_well_thing.id, - sample_type="groundwater", - field_sample_id="FS-9999999", - sample_date="2025-01-01T00:00:00Z", - release_status="draft", - sampler_name="Test Sampler", - qc_sample="Duplicate", - sensor_id=sensor.id, - sample_matrix="water", - sample_method="manual", - duplicate_sample_number=3, - sample_top=2, - sample_bottom=3, - ) - session.add(sample) - session.commit() - yield sample - session.delete(sample) - session.commit() - - @pytest.fixture(scope="session") def contact(water_well_thing): with session_ctx() as session: @@ -507,13 +459,107 @@ def second_asset(): @pytest.fixture(scope="session") -def groundwater_level_observation(sensor, sample): +def groundwater_level_sample(water_well_thing, sensor): + with session_ctx() as session: + sample = Sample( + sample_date="2025-01-01T00:00:00Z", + thing_id=water_well_thing.id, + sample_type="groundwater level", + sampler_name="Test Sampler", + release_status="draft", + field_sample_id=f"FS-{uuid.uuid4()}", + qc_sample="Original", + sensor_id=sensor.id, + sample_matrix="groundwater", + sample_method="manual", + duplicate_sample_number=0, + sample_top=None, + sample_bottom=None, + ) + session.add(sample) + session.commit() + yield sample + + +@pytest.fixture(scope="session") +def water_chemistry_sample(water_well_thing, sensor): + with session_ctx() as session: + sample = Sample( + sample_date="2025-01-01T00:00:00Z", + thing_id=water_well_thing.id, + sample_type="water chemistry", + sampler_name="Test Sampler", + release_status="draft", + field_sample_id=f"FS-{uuid.uuid4()}", + qc_sample="Original", + sensor_id=sensor.id, + sample_matrix="groundwater", + sample_method="manual", + duplicate_sample_number=0, + sample_top=None, + sample_bottom=None, + ) + session.add(sample) + session.commit() + yield sample + + +@pytest.fixture(scope="session") +def geothermal_sample(water_well_thing, sensor): + with session_ctx() as session: + sample = Sample( + sample_date="2025-01-01T00:00:00Z", + thing_id=water_well_thing.id, + sample_type="geothermal", + sampler_name="Test Sampler", + release_status="draft", + field_sample_id=f"FS-{uuid.uuid4()}", + qc_sample="Original", + sensor_id=sensor.id, + sample_matrix="groundwater", + sample_method="manual", + duplicate_sample_number=0, + sample_top=None, + sample_bottom=None, + ) + session.add(sample) + session.commit() + yield sample + + +@pytest.fixture(scope="function") +def second_sample(water_well_thing, sensor): + with session_ctx() as session: + sample = Sample( + thing_id=water_well_thing.id, + sample_type="groundwater level", + field_sample_id="FS-9999999", + sample_date="2025-01-01T00:00:00Z", + release_status="draft", + sampler_name="Test Sampler", + qc_sample="Duplicate", + sensor_id=sensor.id, + sample_matrix="groundwater", + sample_method="manual", + duplicate_sample_number=3, + sample_top=2, + sample_bottom=3, + ) + session.add(sample) + session.commit() + yield sample + session.delete(sample) + session.commit() + + +@pytest.fixture(scope="session") +def groundwater_level_observation(sensor, groundwater_level_sample): with session_ctx() as session: observation = Observation( observation_datetime="2025-01-01T00:04:00Z", - sample_id=sample.id, + sample_id=groundwater_level_sample.id, sensor_id=sensor.id, - observed_property="groundwater level:groundwater level", + observed_property="groundwater level", release_status="draft", value=10.0, unit="ft", @@ -526,13 +572,13 @@ def groundwater_level_observation(sensor, sample): @pytest.fixture(scope="session") -def water_chemistry_observation(sensor, sample): +def water_chemistry_observation(sensor, water_chemistry_sample): with session_ctx() as session: observation = Observation( observation_datetime="2025-01-01T00:03:00Z", - sample_id=sample.id, + sample_id=water_chemistry_sample.id, sensor_id=sensor.id, - observed_property="water chemistry:pH", + observed_property="pH", release_status="draft", value=4.0, unit="dimensionless", @@ -543,13 +589,13 @@ def water_chemistry_observation(sensor, sample): @pytest.fixture(scope="session") -def geothermal_observation(sensor, sample): +def geothermal_observation(sensor, geothermal_sample): with session_ctx() as session: observation = Observation( observation_datetime="2025-01-01T00:02:00Z", - sample_id=sample.id, + sample_id=geothermal_sample.id, sensor_id=sensor.id, - observed_property="geothermal:temperature", + observed_property="temperature", release_status="draft", value=20.0, unit="deg C", @@ -561,13 +607,13 @@ def geothermal_observation(sensor, sample): @pytest.fixture(scope="function") -def observation_to_delete(sample, sensor): +def observation_to_delete(water_chemistry_sample, sensor): with session_ctx() as session: observation = Observation( observation_datetime="2019-01-01T00:03:00Z", - sample_id=sample.id, + sample_id=water_chemistry_sample.id, sensor_id=sensor.id, - observed_property="water chemistry:pH", + observed_property="pH", release_status="draft", value=4.0, unit="dimensionless", diff --git a/tests/test_observation.py b/tests/test_observation.py index 293b59023..4f9b56f69 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -42,13 +42,13 @@ def override_authentication_dependency_fixture(): # ============= Post tests ================= -def test_add_water_chemistry_observation(sample, sensor): +def test_add_water_chemistry_observation(water_chemistry_sample, sensor): payload = { "observation_datetime": "2025-01-01T00:00:00Z", "release_status": "draft", "value": 7.5, "unit": "dimensionless", - "sample_id": sample.id, + "sample_id": water_chemistry_sample.id, "sensor_id": sensor.id, "observed_property": "pH", } @@ -69,13 +69,13 @@ def test_add_water_chemistry_observation(sample, sensor): cleanup_post_test(Observation, data["id"]) -def test_add_groundwater_level_observation(sample, sensor): +def test_add_groundwater_level_observation(groundwater_level_sample, sensor): payload = { "observation_datetime": "2025-01-01T00:00:00Z", "release_status": "draft", "value": 101, "measuring_point_height": 53, - "sample_id": sample.id, + "sample_id": groundwater_level_sample.id, "sensor_id": sensor.id, "level_status": "Water level not affected by status", "observed_property": "groundwater level", @@ -102,13 +102,13 @@ def test_add_groundwater_level_observation(sample, sensor): cleanup_post_test(Observation, data["id"]) -def test_add_geothermal_observation(sample, sensor): +def test_add_geothermal_observation(geothermal_sample, sensor): payload = { "observation_datetime": "2025-01-01T00:00:00Z", "release_status": "draft", "observation_depth": 100, "value": 25.5, - "sample_id": sample.id, + "sample_id": geothermal_sample.id, "sensor_id": sensor.id, "observed_property": "temperature", "unit": "deg C", @@ -160,7 +160,7 @@ def test_patch_groundwater_level_observation_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_groundwater_level_observation_404_wrong_observation_class( +def test_patch_groundwater_level_observation_404_wrong_sample_type( water_chemistry_observation, geothermal_observation ): for obs in water_chemistry_observation, geothermal_observation: @@ -171,14 +171,14 @@ def test_patch_groundwater_level_observation_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "geothermal:temperature": - observation_class = "geothermal" + if obs.observed_property == "temperature": + sample_type = "geothermal" else: - observation_class = "water chemistry" + sample_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {observation_class} observation." + == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {sample_type} observation." ) @@ -206,7 +206,7 @@ def test_patch_water_chemistry_observation_404_not_found(water_chemistry_observa assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_water_chemistry_observation_404_wrong_observation_class( +def test_patch_water_chemistry_observation_404_wrong_sample_type( groundwater_level_observation, geothermal_observation ): for obs in groundwater_level_observation, geothermal_observation: @@ -215,14 +215,14 @@ def test_patch_water_chemistry_observation_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "geothermal:temperature": - observation_class = "geothermal" + if obs.observed_property == "temperature": + sample_type = "geothermal" else: - observation_class = "groundwater level" + sample_type = "groundwater level" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {observation_class} observation." + == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {sample_type} observation." ) @@ -248,7 +248,7 @@ def test_patch_geothermal_observation_404_not_found(geothermal_observation): assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_geothermal_observation_404_wrong_observation_class( +def test_patch_geothermal_observation_404_wrong_sample_type( groundwater_level_observation, water_chemistry_observation ): for obs in groundwater_level_observation, water_chemistry_observation: @@ -257,14 +257,14 @@ def test_patch_geothermal_observation_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "groundwater level:groundwater level": - observation_class = "groundwater level" + if obs.observed_property == "groundwater level": + sample_type = "groundwater level" else: - observation_class = "water chemistry" + sample_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {observation_class} observation." + == f"Observation with ID {obs.id} is not a geothermal observation. It is a {sample_type} observation." ) @@ -298,10 +298,10 @@ def test_get_observation_by_id( assert data["id"] == obs.id assert data["created_at"] == obs.created_at.isoformat().replace("+00:00", "Z") assert data["release_status"] == obs.release_status - if obs.observed_property == "groundwater level:groundwater level": + if obs.observed_property == "groundwater level": assert data["depth_to_water_bgs"] == obs.value - obs.measuring_point_height assert data["observation_depth"] is None - elif obs.observed_property == "geothermal:temperature": + elif obs.observed_property == "temperature": assert data["depth_to_water_bgs"] is None assert data["observation_depth"] == obs.observation_depth else: @@ -412,7 +412,7 @@ def test_get_groundwater_level_observation_by_id_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_groundwater_level_observation_by_id_404_wrong_observation_class( +def test_get_groundwater_level_observation_by_id_404_wrong_sample_type( water_chemistry_observation, geothermal_observation ): for obs in water_chemistry_observation, geothermal_observation: @@ -420,24 +420,27 @@ def test_get_groundwater_level_observation_by_id_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "geothermal:temperature": - actual_observation_class = "geothermal" + if obs.observed_property == "temperature": + actual_sample_type = "geothermal" else: - actual_observation_class = "water chemistry" + actual_sample_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {actual_observation_class} observation." + == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {actual_sample_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} assert data["detail"][0]["loc"] == ["path", "observation_id"] -def test_get_groundwater_observation_by_sample(sample): +def test_get_groundwater_observation_by_sample(groundwater_level_sample): response = client.get( "/observation/groundwater-level", - params={"sample_id": sample.id, "observed_property": "groundwater level"}, + params={ + "sample_id": groundwater_level_sample.id, + "observed_property": "groundwater level", + }, ) assert response.status_code == 200 data = response.json() @@ -566,7 +569,7 @@ def test_get_water_chemistry_observation_by_id_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_water_chemistry_observation_by_id_404_wrong_observation_class( +def test_get_water_chemistry_observation_by_id_404_wrong_sample_type( groundwater_level_observation, geothermal_observation ): for obs in groundwater_level_observation, geothermal_observation: @@ -574,14 +577,14 @@ def test_get_water_chemistry_observation_by_id_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "groundwater level:groundwater level": - actual_observation_class = "groundwater level" + if obs.observed_property == "groundwater level": + actual_sample_type = "groundwater level" else: - actual_observation_class = "geothermal" + actual_sample_type = "geothermal" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {actual_observation_class} observation." + == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {actual_sample_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} @@ -647,7 +650,7 @@ def test_get_geothermal_observation_by_id_404_not_found(geothermal_observation): assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_geothermal_observation_by_id_404_wrong_observation_class( +def test_get_geothermal_observation_by_id_404_wrong_sample_type( water_chemistry_observation, groundwater_level_observation ): for obs in water_chemistry_observation, groundwater_level_observation: @@ -655,14 +658,14 @@ def test_get_geothermal_observation_by_id_404_wrong_observation_class( assert response.status_code == 404 data = response.json() - if obs.observed_property == "groundwater level:groundwater level": - actual_observation_class = "groundwater level" + if obs.observed_property == "groundwater level": + actual_sample_type = "groundwater level" else: - actual_observation_class = "water chemistry" + actual_sample_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_observation_class} observation." + == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_sample_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} From d17bce644e231e7630ca68a8339a93add4c92428 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 11:59:16 -0600 Subject: [PATCH 05/37] note: note validations that need to be written --- api/observation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/observation.py b/api/observation.py index cb53befd5..9ebf5aca9 100644 --- a/api/observation.py +++ b/api/observation.py @@ -48,6 +48,13 @@ router = APIRouter(prefix="/observation", tags=["observation"]) +""" +TODO + +- add validation that the sample_id exists in the database before creating observation +- add validation that the activity_type of the sample corresponds with the endpoint where the observation is posted/patched +""" + # ============= Post ============================================= @router.post("/groundwater-level", status_code=HTTP_201_CREATED) From e259f72f8cb2821d58d31b388ab48b754be58aa8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 12:02:38 -0600 Subject: [PATCH 06/37] refactor: update tests for new sample fixtures --- api/sample.py | 1 + tests/test_sample.py | 133 ++++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 60 deletions(-) diff --git a/api/sample.py b/api/sample.py index a16b69d4e..87d53f498 100644 --- a/api/sample.py +++ b/api/sample.py @@ -45,6 +45,7 @@ def database_error_handler( Handle errors raised by the database when adding or updating a sample. """ error_message = error.orig.args[0]["M"] + print(error_message) if ( error_message == 'duplicate key value violates unique constraint "sample_field_sample_id_key"' diff --git a/tests/test_sample.py b/tests/test_sample.py index ba452d24c..58c7a95d7 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -59,14 +59,14 @@ def test_add_sample(spring_thing, sensor): """ payload = { "thing_id": spring_thing.id, - "sample_type": "groundwater", + "sample_type": "water chemistry", "field_sample_id": "FS-1234567", "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", "sampler_name": "Test Sampler", "qc_sample": "Duplicate", "sensor_id": sensor.id, - "sample_matrix": "water", + "sample_matrix": "groundwater", "sample_method": "manual", "duplicate_sample_number": 3, "sample_top": 2, @@ -98,20 +98,20 @@ def test_add_sample(spring_thing, sensor): cleanup_post_test(Sample, data["id"]) -def test_409_add_sample_invalid_field_sample_id(sample, spring_thing): +def test_409_add_sample_invalid_field_sample_id(water_chemistry_sample, spring_thing): """ Test adding a sample with an invalid field_sample_id. """ payload = { "thing_id": spring_thing.id, - "sample_type": "groundwater", - "field_sample_id": sample.field_sample_id, # This should already exist + "sample_type": "water chemistry", + "field_sample_id": water_chemistry_sample.field_sample_id, # This should already exist "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", "sampler_name": "Test Sampler", "qc_sample": "Duplicate", "sensor_id": None, - "sample_matrix": "water", + "sample_matrix": "groundwater", "sample_method": "manual", "duplicate_sample_number": 3, "sample_top": 2, @@ -126,10 +126,12 @@ def test_409_add_sample_invalid_field_sample_id(sample, spring_thing): assert data["detail"][0]["loc"] == ["body", "field_sample_id"] assert ( data["detail"][0]["msg"] - == f"Sample with field_sample_id {sample.field_sample_id} already exists." + == f"Sample with field_sample_id {water_chemistry_sample.field_sample_id} already exists." ) assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"field_sample_id": sample.field_sample_id} + assert data["detail"][0]["input"] == { + "field_sample_id": water_chemistry_sample.field_sample_id + } def test_409_add_sample_invalid_thing_id(): @@ -138,14 +140,14 @@ def test_409_add_sample_invalid_thing_id(): """ payload = { "thing_id": 9999999, - "sample_type": "groundwater", + "sample_type": "water chemistry", "field_sample_id": "FS-9999999", "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", "sampler_name": "Test Sampler", "qc_sample": "Duplicate", "sensor_id": None, - "sample_matrix": "water", + "sample_matrix": "groundwater", "sample_method": "manual", "duplicate_sample_number": 3, "sample_top": 2, @@ -167,7 +169,7 @@ def test_409_add_sample_invalid_thing_id(): # ============= Patch tests for samples ============================================= -def test_patch_sample(sample): +def test_patch_sample(water_chemistry_sample): """ Test updating a sample. """ @@ -177,21 +179,21 @@ def test_patch_sample(sample): "sample_date": "2025-01-02T00:00:00Z", "release_status": "private", } - response = client.patch(f"/sample/{sample.id}", json=payload) + response = client.patch(f"/sample/{water_chemistry_sample.id}", json=payload) assert response.status_code == 200 data = response.json() - assert data["id"] == sample.id + assert data["id"] == water_chemistry_sample.id assert data["sampler_name"] == payload["sampler_name"] assert data["sample_date"] == payload["sample_date"] assert data["sample_method"] == payload["sample_method"] assert data["release_status"] == payload["release_status"] # rollback after updating the sample - cleanup_patch_test(Sample, payload, sample) + cleanup_patch_test(Sample, payload, water_chemistry_sample) -def test_patch_sample_404_not_found(sample): +def test_patch_sample_404_not_found(water_chemistry_sample): """ Test updating a sample that does not exist """ @@ -207,12 +209,14 @@ def test_patch_sample_404_not_found(sample): assert data["detail"] == "Sample with ID 999 not found." -def test_409_patch_sample_invalid_field_sample_id(sample, second_sample): +def test_409_patch_sample_invalid_field_sample_id( + water_chemistry_sample, second_sample +): """ Test updating a sample with an invalid field_sample_id. """ payload = { - "field_sample_id": sample.field_sample_id, # This should already exist + "field_sample_id": water_chemistry_sample.field_sample_id, # This should already exist } response = client.patch( f"/sample/{second_sample.id}", @@ -226,10 +230,12 @@ def test_409_patch_sample_invalid_field_sample_id(sample, second_sample): == f"Sample with field_sample_id {payload['field_sample_id']} already exists." ) assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"field_sample_id": sample.field_sample_id} + assert data["detail"][0]["input"] == { + "field_sample_id": water_chemistry_sample.field_sample_id + } -def test_409_patch_sample_invalid_thing_id(sample): +def test_409_patch_sample_invalid_thing_id(water_chemistry_sample): """ Test updating a sample with an invalid thing_id. """ @@ -237,7 +243,7 @@ def test_409_patch_sample_invalid_thing_id(sample): "thing_id": 9999999, } response = client.patch( - f"/sample/{sample.id}", + f"/sample/{water_chemistry_sample.id}", json=payload, ) data = response.json() @@ -252,58 +258,65 @@ def test_409_patch_sample_invalid_thing_id(sample): # ============= Get tests for samples ============================================= -def test_get_samples(sample, water_well_thing): +def test_get_samples( + water_chemistry_sample, groundwater_level_sample, geothermal_sample +): """ Test retrieving samples """ response = client.get("/sample") assert response.status_code == 200 data = response.json() - assert len(data["items"]) == 1 - assert data["items"][0]["id"] == sample.id - assert data["items"][0]["created_at"] == sample.created_at.isoformat().replace( - "+00:00", "Z" - ) - assert data["items"][0]["thing"]["id"] == water_well_thing.id - assert data["items"][0]["sample_type"] == sample.sample_type - assert data["items"][0]["field_sample_id"] == sample.field_sample_id - assert data["items"][0]["sample_date"] == sample.sample_date - assert data["items"][0]["release_status"] == sample.release_status - assert data["items"][0]["sampler_name"] == sample.sampler_name - assert data["items"][0]["qc_sample"] == sample.qc_sample - assert data["items"][0]["sensor_id"] == sample.sensor_id - assert data["items"][0]["sample_matrix"] == sample.sample_matrix - assert data["items"][0]["sample_method"] == sample.sample_method - assert data["items"][0]["duplicate_sample_number"] == sample.duplicate_sample_number - assert data["items"][0]["sample_top"] == sample.sample_top - assert data["items"][0]["sample_bottom"] == sample.sample_bottom - - -def test_get_sample_by_id(sample, water_well_thing): + assert len(data["items"]) == 3 + + for item in data["items"]: + assert "id" in item + assert "created_at" in item + assert "thing" in item + assert "sample_type" in item + assert "field_sample_id" in item + assert "sample_date" in item + assert "release_status" in item + assert "sampler_name" in item + assert "qc_sample" in item + assert "sensor_id" in item + assert "sample_matrix" in item + assert "sample_method" in item + assert "duplicate_sample_number" in item + assert "sample_top" in item + assert "sample_bottom" in item + + +def test_get_sample_by_id(water_chemistry_sample, water_well_thing): """ Test retrieving a sample by its ID. """ - response = client.get(f"/sample/{sample.id}") + response = client.get(f"/sample/{water_chemistry_sample.id}") assert response.status_code == 200 data = response.json() - assert data["id"] == sample.id - assert data["created_at"] == sample.created_at.isoformat().replace("+00:00", "Z") + assert data["id"] == water_chemistry_sample.id + assert data["created_at"] == water_chemistry_sample.created_at.isoformat().replace( + "+00:00", "Z" + ) assert data["thing"]["id"] == water_well_thing.id - assert data["sample_type"] == sample.sample_type - assert data["field_sample_id"] == sample.field_sample_id - assert data["sample_date"] == sample.sample_date - assert data["release_status"] == sample.release_status - assert data["sampler_name"] == sample.sampler_name - assert data["qc_sample"] == sample.qc_sample - assert data["sensor_id"] == sample.sensor_id - assert data["sample_matrix"] == sample.sample_matrix - assert data["sample_method"] == sample.sample_method - assert data["duplicate_sample_number"] == sample.duplicate_sample_number - assert data["sample_top"] == sample.sample_top - assert data["sample_bottom"] == sample.sample_bottom - - -def test_get_sample_by_id_404_not_found(sample): + assert data["sample_type"] == water_chemistry_sample.sample_type + assert data["field_sample_id"] == water_chemistry_sample.field_sample_id + assert data["sample_date"] == water_chemistry_sample.sample_date + assert data["release_status"] == water_chemistry_sample.release_status + assert data["sampler_name"] == water_chemistry_sample.sampler_name + assert data["qc_sample"] == water_chemistry_sample.qc_sample + assert data["sensor_id"] == water_chemistry_sample.sensor_id + assert data["sample_matrix"] == water_chemistry_sample.sample_matrix + assert data["sample_method"] == water_chemistry_sample.sample_method + assert ( + data["duplicate_sample_number"] + == water_chemistry_sample.duplicate_sample_number + ) + assert data["sample_top"] == water_chemistry_sample.sample_top + assert data["sample_bottom"] == water_chemistry_sample.sample_bottom + + +def test_get_sample_by_id_404_not_found(water_chemistry_sample): """ Test retrieving a sample that does not exist. """ From 7ce91145b663bbf37883cbed90ee96a88a70cccd Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 12:08:21 -0600 Subject: [PATCH 07/37] refactor: rename sample_type to activity_type --- api/observation.py | 8 +++--- core/lexicon.json | 6 ++-- db/sample.py | 4 +-- schemas/sample.py | 6 ++-- services/observation_helper.py | 32 ++++++++++----------- tests/conftest.py | 8 +++--- tests/test_observation.py | 48 +++++++++++++++---------------- tests/test_sample.py | 12 ++++---- transfers/waterlevels_transfer.py | 19 ++++++++++-- 9 files changed, 79 insertions(+), 64 deletions(-) diff --git a/api/observation.py b/api/observation.py index 9ebf5aca9..9e5a442a7 100644 --- a/api/observation.py +++ b/api/observation.py @@ -43,7 +43,7 @@ from services.observation_helper import ( get_observations, observation_model_patcher, - get_observation_of_a_sample_type_by_id, + get_observation_of_an_activity_type_by_id, ) router = APIRouter(prefix="/observation", tags=["observation"]) @@ -184,7 +184,7 @@ async def get_groundwater_level_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> GroundwaterLevelObservationResponse: - return get_observation_of_a_sample_type_by_id( + return get_observation_of_an_activity_type_by_id( session=session, request=request, observation_id=observation_id, @@ -231,7 +231,7 @@ async def get_water_chemistry_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> WaterChemistryObservationResponse: - return get_observation_of_a_sample_type_by_id( + return get_observation_of_an_activity_type_by_id( session=session, request=request, observation_id=observation_id, @@ -276,7 +276,7 @@ async def get_geothermal_observation_by_id( user: amp_viewer_dependency, observation_id: int, ) -> GeothermalObservationResponse: - return get_observation_of_a_sample_type_by_id( + return get_observation_of_an_activity_type_by_id( session=session, request=request, observation_id=observation_id ) diff --git a/core/lexicon.json b/core/lexicon.json index 3b06b1e08..8305b4a59 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -251,9 +251,9 @@ {"categories": [{"name": "relation", "description": null}], "term": "OSEPOD", "definition": "NM OSE 'Point of Diversion' ID"}, {"categories": [{"name": "relation", "description": null}], "term": "PLSS", "definition": "Public Land Survey System ID"}, - {"categories": [{"name": "sample_type", "description": null}], "term": "groundwater level", "definition": "groundwater level"}, - {"categories": [{"name": "sample_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, - {"categories": [{"name": "sample_type", "description": null}], "term": "geothermal", "definition": "geothermal"}, + {"categories": [{"name": "activity_type", "description": null}], "term": "groundwater level", "definition": "groundwater level"}, + {"categories": [{"name": "activity_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, + {"categories": [{"name": "activity_type", "description": null}], "term": "geothermal", "definition": "geothermal"}, {"categories": [{"name": "sample_matrix", "description": null}], "term": "groundwater", "definition": "groundwater"}, diff --git a/db/sample.py b/db/sample.py index 4e7a2a4e8..a4486a8fc 100644 --- a/db/sample.py +++ b/db/sample.py @@ -79,9 +79,9 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): default=0, comment="Identifier for duplicate samples (0 = original sample, not a duplicate, 1 = dup no.1, 2 = dup no.2, etc.).", ) - sample_type: Mapped[str] = lexicon_term( + activity_type: Mapped[str] = lexicon_term( nullable=False, - comment="The type of sample (e.g., 'geochemical', 'geothermal', 'groundwater').", + comment="The type of sampling activity (e.g., 'geochemical', 'geothermal', 'groundwater level', 'water chemistry').", ) # --- Relationship Definitions --- diff --git a/schemas/sample.py b/schemas/sample.py index 8ed753deb..bdf5a4ab4 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -87,7 +87,7 @@ def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime: # -------- CREATE ---------- class CreateSample(BaseCreateModel, ValidateSample): thing_id: int - sample_type: str + activity_type: str field_sample_id: str sample_date: Annotated[AwareDatetime, PastDatetime()] sampler_name: str # REFACTOR TODO: update with enum/restricted values @@ -118,7 +118,7 @@ class UpdateSample(BaseUpdateModel, ValidateSample): """ thing_id: int | None = None # REFACTOR TODO: should users be able to change this? - sample_type: str | None = None + activity_type: str | None = None field_sample_id: str | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None sampler_name: str | None = None # REFACTOR TODO: update with enum/restricted values @@ -143,7 +143,7 @@ class UpdateSample(BaseUpdateModel, ValidateSample): # -------- RESPONSE ---------- class SampleResponse(BaseResponseModel): thing: ThingResponse - sample_type: str + activity_type: str field_sample_id: str sample_date: AwareDatetime release_status: str diff --git a/services/observation_helper.py b/services/observation_helper.py index 2c74c0ae5..eddb098fe 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -19,18 +19,18 @@ from services.query_helper import simple_get_by_id, order_sort_filter -def get_sample_type_from_request(request: Request) -> str: +def get_activity_type_from_request(request: Request) -> str: path = request.url.path path_components = path.split("/") if len(path_components) == 2: # no sample type specified in path - sample_type_in_path = path_components[1] + activity_type_in_path = path_components[1] if len(path_components) >= 3: # sample type specified in path - sample_type_in_path = path_components[2] + activity_type_in_path = path_components[2] - sample_type = sample_type_in_path.replace("-", " ") - return sample_type + activity_type = activity_type_in_path.replace("-", " ") + return activity_type def get_observations( @@ -54,7 +54,7 @@ def get_observations( Retrieve all observations """ sample_table_is_joined = False - sample_type = get_sample_type_from_request(request) + activity_type = get_activity_type_from_request(request) sql = select(Observation) if thing_id is not None: @@ -72,10 +72,10 @@ def get_observations( sql = sql.where(Observation.observation_datetime <= end_time) # root of path is /observation - if sample_type != "observation": + if activity_type != "observation": if sample_table_is_joined is False: sql = sql.join(Sample, Sample.id == Observation.sample_id) - sql = sql.where(Sample.sample_type == sample_type) + sql = sql.where(Sample.activity_type == activity_type) sql = order_sort_filter(sql, Observation, sort, order, filter_) @@ -85,13 +85,13 @@ def get_observations( return paginate(query=sql, conn=session) -def verify_observed_property_corresponds_with_sample_type( +def verify_observed_property_corresponds_with_activity_type( observation: Observation, request: Request ): - requested_sample_type = get_sample_type_from_request(request) - actual_sample_type = observation.sample.sample_type + requested_activity_type = get_activity_type_from_request(request) + actual_activity_type = observation.sample.activity_type - if actual_sample_type != requested_sample_type: + if actual_activity_type != requested_activity_type: raise PydanticStyleException( status_code=HTTP_404_NOT_FOUND, detail=[ @@ -99,13 +99,13 @@ def verify_observed_property_corresponds_with_sample_type( "loc": ["path", "observation_id"], "type": "value_error", "input": {"observation_id": observation.id}, - "msg": f"Observation with ID {observation.id} is not a {requested_sample_type} observation. It is a {actual_sample_type} observation.", + "msg": f"Observation with ID {observation.id} is not a {requested_activity_type} observation. It is a {actual_activity_type} observation.", } ], ) -def get_observation_of_a_sample_type_by_id( +def get_observation_of_an_activity_type_by_id( session: Session, request: Request, observation_id: int ) -> Observation: """ @@ -113,7 +113,7 @@ def get_observation_of_a_sample_type_by_id( """ observation = simple_get_by_id(session, Observation, observation_id) - verify_observed_property_corresponds_with_sample_type(observation, request) + verify_observed_property_corresponds_with_activity_type(observation, request) return observation @@ -131,7 +131,7 @@ def observation_model_patcher( # simple_get_by_id raises HTTP_404_NOT_FOUND if the item is not found observation = simple_get_by_id(session, Observation, observation_id) - verify_observed_property_corresponds_with_sample_type(observation, request) + verify_observed_property_corresponds_with_activity_type(observation, request) for key, value in payload.model_dump(exclude_unset=True).items(): setattr(observation, key, value) diff --git a/tests/conftest.py b/tests/conftest.py index aec663d74..5254df990 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -464,7 +464,7 @@ def groundwater_level_sample(water_well_thing, sensor): sample = Sample( sample_date="2025-01-01T00:00:00Z", thing_id=water_well_thing.id, - sample_type="groundwater level", + activity_type="groundwater level", sampler_name="Test Sampler", release_status="draft", field_sample_id=f"FS-{uuid.uuid4()}", @@ -487,7 +487,7 @@ def water_chemistry_sample(water_well_thing, sensor): sample = Sample( sample_date="2025-01-01T00:00:00Z", thing_id=water_well_thing.id, - sample_type="water chemistry", + activity_type="water chemistry", sampler_name="Test Sampler", release_status="draft", field_sample_id=f"FS-{uuid.uuid4()}", @@ -510,7 +510,7 @@ def geothermal_sample(water_well_thing, sensor): sample = Sample( sample_date="2025-01-01T00:00:00Z", thing_id=water_well_thing.id, - sample_type="geothermal", + activity_type="geothermal", sampler_name="Test Sampler", release_status="draft", field_sample_id=f"FS-{uuid.uuid4()}", @@ -532,7 +532,7 @@ def second_sample(water_well_thing, sensor): with session_ctx() as session: sample = Sample( thing_id=water_well_thing.id, - sample_type="groundwater level", + activity_type="groundwater level", field_sample_id="FS-9999999", sample_date="2025-01-01T00:00:00Z", release_status="draft", diff --git a/tests/test_observation.py b/tests/test_observation.py index 4f9b56f69..72ca9c825 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -160,7 +160,7 @@ def test_patch_groundwater_level_observation_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_groundwater_level_observation_404_wrong_sample_type( +def test_patch_groundwater_level_observation_404_wrong_activity_type( water_chemistry_observation, geothermal_observation ): for obs in water_chemistry_observation, geothermal_observation: @@ -172,13 +172,13 @@ def test_patch_groundwater_level_observation_404_wrong_sample_type( data = response.json() if obs.observed_property == "temperature": - sample_type = "geothermal" + activity_type = "geothermal" else: - sample_type = "water chemistry" + activity_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {sample_type} observation." + == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {activity_type} observation." ) @@ -206,7 +206,7 @@ def test_patch_water_chemistry_observation_404_not_found(water_chemistry_observa assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_water_chemistry_observation_404_wrong_sample_type( +def test_patch_water_chemistry_observation_404_wrong_activity_type( groundwater_level_observation, geothermal_observation ): for obs in groundwater_level_observation, geothermal_observation: @@ -216,13 +216,13 @@ def test_patch_water_chemistry_observation_404_wrong_sample_type( data = response.json() if obs.observed_property == "temperature": - sample_type = "geothermal" + activity_type = "geothermal" else: - sample_type = "groundwater level" + activity_type = "groundwater level" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {sample_type} observation." + == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {activity_type} observation." ) @@ -248,7 +248,7 @@ def test_patch_geothermal_observation_404_not_found(geothermal_observation): assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_geothermal_observation_404_wrong_sample_type( +def test_patch_geothermal_observation_404_wrong_activity_type( groundwater_level_observation, water_chemistry_observation ): for obs in groundwater_level_observation, water_chemistry_observation: @@ -258,13 +258,13 @@ def test_patch_geothermal_observation_404_wrong_sample_type( data = response.json() if obs.observed_property == "groundwater level": - sample_type = "groundwater level" + activity_type = "groundwater level" else: - sample_type = "water chemistry" + activity_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {sample_type} observation." + == f"Observation with ID {obs.id} is not a geothermal observation. It is a {activity_type} observation." ) @@ -412,7 +412,7 @@ def test_get_groundwater_level_observation_by_id_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_groundwater_level_observation_by_id_404_wrong_sample_type( +def test_get_groundwater_level_observation_by_id_404_wrong_activity_type( water_chemistry_observation, geothermal_observation ): for obs in water_chemistry_observation, geothermal_observation: @@ -421,13 +421,13 @@ def test_get_groundwater_level_observation_by_id_404_wrong_sample_type( data = response.json() if obs.observed_property == "temperature": - actual_sample_type = "geothermal" + actual_activity_type = "geothermal" else: - actual_sample_type = "water chemistry" + actual_activity_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {actual_sample_type} observation." + == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {actual_activity_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} @@ -569,7 +569,7 @@ def test_get_water_chemistry_observation_by_id_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_water_chemistry_observation_by_id_404_wrong_sample_type( +def test_get_water_chemistry_observation_by_id_404_wrong_activity_type( groundwater_level_observation, geothermal_observation ): for obs in groundwater_level_observation, geothermal_observation: @@ -578,13 +578,13 @@ def test_get_water_chemistry_observation_by_id_404_wrong_sample_type( data = response.json() if obs.observed_property == "groundwater level": - actual_sample_type = "groundwater level" + actual_activity_type = "groundwater level" else: - actual_sample_type = "geothermal" + actual_activity_type = "geothermal" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {actual_sample_type} observation." + == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {actual_activity_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} @@ -650,7 +650,7 @@ def test_get_geothermal_observation_by_id_404_not_found(geothermal_observation): assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_geothermal_observation_by_id_404_wrong_sample_type( +def test_get_geothermal_observation_by_id_404_wrong_activity_type( water_chemistry_observation, groundwater_level_observation ): for obs in water_chemistry_observation, groundwater_level_observation: @@ -659,13 +659,13 @@ def test_get_geothermal_observation_by_id_404_wrong_sample_type( data = response.json() if obs.observed_property == "groundwater level": - actual_sample_type = "groundwater level" + actual_activity_type = "groundwater level" else: - actual_sample_type = "water chemistry" + actual_activity_type = "water chemistry" assert ( data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_sample_type} observation." + == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_activity_type} observation." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"observation_id": obs.id} diff --git a/tests/test_sample.py b/tests/test_sample.py index 58c7a95d7..8b32ff200 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -59,7 +59,7 @@ def test_add_sample(spring_thing, sensor): """ payload = { "thing_id": spring_thing.id, - "sample_type": "water chemistry", + "activity_type": "water chemistry", "field_sample_id": "FS-1234567", "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", @@ -81,7 +81,7 @@ def test_add_sample(spring_thing, sensor): assert "id" in data assert "created_at" in data assert data["thing"]["id"] == spring_thing.id - assert data["sample_type"] == payload["sample_type"] + assert data["activity_type"] == payload["activity_type"] assert data["field_sample_id"] == payload["field_sample_id"] assert data["sample_date"] == payload["sample_date"] assert data["release_status"] == payload["release_status"] @@ -104,7 +104,7 @@ def test_409_add_sample_invalid_field_sample_id(water_chemistry_sample, spring_t """ payload = { "thing_id": spring_thing.id, - "sample_type": "water chemistry", + "activity_type": "water chemistry", "field_sample_id": water_chemistry_sample.field_sample_id, # This should already exist "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", @@ -140,7 +140,7 @@ def test_409_add_sample_invalid_thing_id(): """ payload = { "thing_id": 9999999, - "sample_type": "water chemistry", + "activity_type": "water chemistry", "field_sample_id": "FS-9999999", "sample_date": "2025-01-01T00:00:00Z", "release_status": "draft", @@ -273,7 +273,7 @@ def test_get_samples( assert "id" in item assert "created_at" in item assert "thing" in item - assert "sample_type" in item + assert "activity_type" in item assert "field_sample_id" in item assert "sample_date" in item assert "release_status" in item @@ -299,7 +299,7 @@ def test_get_sample_by_id(water_chemistry_sample, water_well_thing): "+00:00", "Z" ) assert data["thing"]["id"] == water_well_thing.id - assert data["sample_type"] == water_chemistry_sample.sample_type + assert data["activity_type"] == water_chemistry_sample.activity_type assert data["field_sample_id"] == water_chemistry_sample.field_sample_id assert data["sample_date"] == water_chemistry_sample.sample_date assert data["release_status"] == water_chemistry_sample.release_status diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index aba14063e..cc9a5052f 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -25,6 +25,7 @@ logger, read_csv, convert_mt_to_utc, + lu_to_lexicon_map, ) @@ -65,8 +66,22 @@ def transfer_water_levels(session): continue sample = Sample() - sample.sampler_name = "unknown" - sample.sample_type = "groundwater level" + + if pd.isna(row.MeasuredBy): + sampler_name = "Unknown" + else: + sampler_name = row.MeasuredBy + + sample.activity_type = "groundwater level" + + if not pd.isna(row.MeasurementMethod): + sample_method = lu_to_lexicon_map[ + f"LU_MeasurementMethod:{row.MeasurementMethod}" + ] + else: + sample_method = "null placeholder" + + sample_matrix = "groundwater" sample.field_sample_id = str(uuid.uuid4()) sample.sample_date = dt_utc From 5613e9d353cd80a42d85c06d8f0ecb075a8af520 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 16:41:32 -0600 Subject: [PATCH 08/37] refactor: update qc_sample lexicon --- core/lexicon.json | 12 +++++++----- db/observation.py | 2 +- db/sample.py | 8 +++++++- schemas/observation.py | 6 +++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 8305b4a59..994f8c19c 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1,7 +1,7 @@ [ - {"categories": [{"name": "qc_sample", "description": null}], "term": "original", "definition": ""}, - {"categories": [{"name": "qc_sample", "description": null}], "term": "duplicate", "definition": ""}, + {"categories": [{"name": "qc_sample", "description": null}], "term": "Original", "definition": ""}, + {"categories": [{"name": "qc_sample", "description": null}], "term": "Duplicate", "definition": ""}, {"categories" : [{"name": "vertical_datum", "description": null}], "term": "NAVD88", "definition": "North American Vertical Datum of 1988"}, {"categories" : [{"name": "vertical_datum", "description": null}], "term": "NGVD29", "definition": "National Geodetic Vertical Datum of 1929"}, @@ -444,9 +444,11 @@ {"categories": [{"name": "casing_material", "description": null}], "term": "Steel", "definition": "Steel"}, {"categories": [{"name": "casing_material", "description": null}], "term": "Concrete", "definition": "Concrete"}, - {"categories": [{"name": "quality_control_status", "description": null}], "term": "Provisional", "definition": "Provisional quality control status"}, - {"categories": [{"name": "quality_control_status", "description": null}], "term": "Approved", "definition": "Approved quality control status"}, - {"categories": [{"name": "quality_control_status", "description": null}], "term": "Rejected", "definition": "Rejected quality control status"}, + {"categories": [{"name": "quality_flag", "description": null}], "term": "Good", "definition": "The measurement was collected and analyzed according to standard procedures and passed all QA/QC checks."}, + {"categories": [{"name": "quality_flag", "description": null}], "term": "Questionable", "definition": "The measurement is suspect due to a known issue during collection or analysis, but it may still be usable."}, + {"categories": [{"name": "quality_flag", "description": null}], "term": "Estimated", "definition": "The value is not a direct measurement but an estimate derived from other data or models."}, + {"categories": [{"name": "quality_flag", "description": null}], "term": "Rejected", "definition": "Rejected"}, + {"categories": [{"name": "drilling_fluid", "description": null}], "term": "mud", "definition": "drilling mud"}, diff --git a/db/observation.py b/db/observation.py index 15138d4d1..ef951c912 100644 --- a/db/observation.py +++ b/db/observation.py @@ -27,7 +27,7 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} # NM_Aquifer fields for audits - nma_pk_waterlevel: Mapped[str] = mapped_column(nullable=True) + nma_pk_waterlevels: Mapped[str] = mapped_column(nullable=True) sample_id: Mapped[int] = mapped_column( ForeignKey("sample.id", ondelete="CASCADE"), diff --git a/db/sample.py b/db/sample.py index a4486a8fc..8e6c56f8f 100644 --- a/db/sample.py +++ b/db/sample.py @@ -41,9 +41,11 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Foreign key to the Thing (e.g., sampling location) table.", ) + # nullable because sample can be collected by steel tape sensor_id: Mapped[Optional[int]] = mapped_column( ForeignKey("sensor.id"), comment="Foreign key for the specific equipment used.", + nullable=True, ) # Sample Attributes @@ -64,10 +66,12 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): sampler_name: Mapped[str] = mapped_column( nullable=False, comment="Name of the person who collected the sample." ) + # TODO: is qc_sample required if we have duplicate_sample_number? are split samples recorded, or not? could be a user research question + # to guide development and future-proof qc_sample: Mapped[str] = mapped_column( default="Original", nullable=False, - comment="Quality control sample type (e.g., 'Original', 'field dupe').", + comment="Quality control sample type (e.g., 'Original', 'Split', 'Field duplicate').", ) sample_top: Mapped[float] = mapped_column( nullable=True, comment="Top depth of a discrete sample interval." @@ -75,6 +79,8 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): sample_bottom: Mapped[float] = mapped_column( nullable=True, comment="Bottom depth of a discrete sample interval." ) + # TODO: is qc_sample required if we have duplicate_sample_number? are split samples recorded, or not? could be a user research question + # to guide development and future-proof duplicate_sample_number: Mapped[int] = mapped_column( default=0, comment="Identifier for duplicate samples (0 = original sample, not a duplicate, 1 = dup no.1, 2 = dup no.2, etc.).", diff --git a/schemas/observation.py b/schemas/observation.py index 24b3b08d3..f308a4535 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -93,17 +93,15 @@ class UpdateBaseObservation(BaseUpdateModel, ValidateObservation): class UpdateGroundwaterLevelObservation(UpdateBaseObservation): - _observation_class: str = "groundwater level" measuring_point_height: float | None = None level_status: str | None = None class UpdateWaterChemistryObservation(UpdateBaseObservation): - _observation_class: str = "water chemistry" + pass class UpdateGeothermalObservation(UpdateBaseObservation): - _observation_class: str = "geothermal" observation_depth: float | None = None @@ -150,5 +148,7 @@ class ObservationResponse( Combines groundwater level and geothermal observation responses. """ + pass + # ============= EOF ============================================= From fb3765f54391e33352c67b543bf3286223180d99 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 16 Sep 2025 16:49:02 -0600 Subject: [PATCH 09/37] feat: update wl observation and sample transfers --- transfers/waterlevels_transfer.py | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index cc9a5052f..85391c1c3 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -81,28 +81,42 @@ def transfer_water_levels(session): else: sample_method = "null placeholder" - sample_matrix = "groundwater" - - sample.field_sample_id = str(uuid.uuid4()) - sample.sample_date = dt_utc - sample.thing = thing + sample = Sample( + sampler_name=sampler_name, + sample_date=dt_utc, + sample_matrix="groundwater", + field_sample_id=str(uuid.uuid4()), + thing=thing, + sample_method=sample_method, + qc_sample="Original", + sample_top=None, + sample_bottom=None, + duplicate_sample_number=0, + activity_type="groundwater level", + ) session.add(sample) - obs = Observation() - - # TODO: this needs to be resolved - obs.sensor_id = 1 - - obs.nma_pk_waterlevels = row.GlobalID + # TODO: update for auto-collectors in the Sensor table, like e-probes + sensor_id = None - obs.sample = sample - obs.observation_datetime = dt_utc - obs.value = row.DepthToWater - obs.measuring_point_height = row.MPHeight - obs.observed_property = "groundwater level:groundwater level" - obs.unit = "ft" - - session.add(obs) + if not pd.isna(row.LevelStatus): + level_status = lu_to_lexicon_map[f"LU_LevelStatus:{row.LevelStatus}"] + else: + level_status = None + + observation = Observation( + sensor_id=sensor_id, + sample=sample, + nma_pk_waterlevels=row.GlobalID, + value=row.DepthToWater, + measuring_point_height=row.MPHeight, + observed_property="groundwater level", + unit="ft", + level_status=level_status, + observation_datetime=dt_utc, + ) + + session.add(observation) session.commit() From 16997b4269eec8bfd620f483fc204ad265a3f260 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 17 Sep 2025 13:52:55 -0600 Subject: [PATCH 10/37] note: add developers note to model_patcher --- services/crud_helper.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/crud_helper.py b/services/crud_helper.py index daa4bbf3d..6ef4d80e4 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -52,6 +52,13 @@ def model_patcher( # simple_get_by_id raises HTTP_404_NOT_FOUND if the item is not found item = simple_get_by_id(session, model, item_id) + """ + Developer's notes + + exclude_unset ensures that fields that are not set in the payload do not + update record fields to None + """ + for key, value in payload.model_dump(exclude_unset=True).items(): setattr(item, key, value) From 04759def739b2503c0bc2a08e463d94031cab113 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 17 Sep 2025 16:58:31 -0600 Subject: [PATCH 11/37] WIP: sample, field, observation updates --- api/sample.py | 31 ++---------- core/lexicon.json | 28 ++++++----- db/__init__.py | 1 + db/field.py | 89 ++++++++++++++++++++++++++++++++++ db/sample.py | 104 ++++++++++++++-------------------------- db/sensor.py | 4 +- db/thing.py | 6 +-- main.py | 6 --- schemas/sample.py | 105 ++++++++++++++-------------------------- tests/conftest.py | 109 +++++++++++++++++++----------------------- tests/test_sample.py | 48 ++++++++----------- transfers/transfer.py | 11 +++++ 12 files changed, 265 insertions(+), 277 deletions(-) create mode 100644 db/field.py diff --git a/api/sample.py b/api/sample.py index 87d53f498..4871ab76b 100644 --- a/api/sample.py +++ b/api/sample.py @@ -48,23 +48,13 @@ def database_error_handler( print(error_message) if ( error_message - == 'duplicate key value violates unique constraint "sample_field_sample_id_key"' + == 'duplicate key value violates unique constraint "sample_sample_name_key"' ): detail = { - "loc": ["body", "field_sample_id"], - "msg": f"Sample with field_sample_id {payload.field_sample_id} already exists.", + "loc": ["body", "sample_name"], + "msg": f"Sample with sample_name {payload.sample_name} already exists.", "type": "value_error", - "input": {"field_sample_id": payload.field_sample_id}, - } - elif ( - error_message - == 'insert or update on table "sample" violates foreign key constraint "sample_thing_id_fkey"' - ): - detail = { - "loc": ["body", "thing_id"], - "msg": f"Thing with ID {payload.thing_id} does not exist.", - "type": "value_error", - "input": {"thing_id": payload.thing_id}, + "input": {"sample_name": payload.sample_name}, } raise PydanticStyleException(status_code=HTTP_409_CONFLICT, detail=[detail]) @@ -95,19 +85,6 @@ async def update_sample( """ Endpoint to update a sample. """ - - """ - Development notes: - - What do we do if the field is nullable and the schema defaults to None? - If that occurs, then we update the field to None, which may not have - been the intension of the user. We could set some string to indicate - DO NOT UPDATE. Perhaps coordination between the front and backends? - - - This is handled by the `model_patcher` function, which excludes unset fields from - the update. - """ try: return model_patcher(session, Sample, sample_id, sample_data, user=user) except (IntegrityError, ProgrammingError) as e: diff --git a/core/lexicon.json b/core/lexicon.json index 994f8c19c..aae2cee38 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1,7 +1,11 @@ [ + {"categories": [{"name": "qc_type", "description": null}], "term": "Normal", "definition": "The primary environmental sample collected from the well, spring, or soil boring."}, + {"categories": [{"name": "qc_type", "description": null}], "term": "Duplicate", "definition": "A second, independent sample collected at the same location, at the same time, and in the same manner as the normal sample. This sample is sent to the primary laboratory."}, + {"categories": [{"name": "qc_type", "description": null}], "term": "Split", "definition": "A subsample of a primary environmental sample that is sent to a separate, independent laboratory for analysis."}, + {"categories": [{"name": "qc_type", "description": null}], "term": "Field Blank", "definition": "A sample of certified pure water that is taken to the field, opened, and processed through the same sampling procedure as a normal sample (e.g., poured into a sample bottle)."}, + {"categories": [{"name": "qc_type", "description": null}], "term": "Trip Blank", "definition": "A sample of certified pure water that is prepared in the lab, taken to the field, and brought back to the lab without ever being opened."}, + {"categories": [{"name": "qc_type", "description": null}], "term": "Equipment Blank", "definition": "A sample of certified pure water that is run through the sampling equipment (like a pump and tubing) before the normal sample is collected."}, - {"categories": [{"name": "qc_sample", "description": null}], "term": "Original", "definition": ""}, - {"categories": [{"name": "qc_sample", "description": null}], "term": "Duplicate", "definition": ""}, {"categories" : [{"name": "vertical_datum", "description": null}], "term": "NAVD88", "definition": "North American Vertical Datum of 1988"}, {"categories" : [{"name": "vertical_datum", "description": null}], "term": "NGVD29", "definition": "National Geodetic Vertical Datum of 1929"}, @@ -23,19 +27,12 @@ {"categories": [{"name": "elevation_method", "description": null}], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": [{"name": "elevation_method", "description": null}], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": [{"name": "elevation_method", "description": null}, - {"name": "collection_method", "description": null}, + {"name": "sample_method", "description": null}, {"name": "coordinate_method", "description": null}, {"name": "current_use", "description": null}, {"name": "status", "description": null}, {"name": "organization", "description": null}], "term": "Unknown", "definition": "Unknown"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "bailer", "definition": "bailer"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "faucet at well head", "definition": "faucet at well head"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "faucet or outlet at house", "definition": "faucet or outlet at house"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "grab sample", "definition": "grab sample"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "pump", "definition": "pump"}, - {"categories": [{"name": "collection_method", "description": null}], "term": "thief sampler", "definition": "thief sampler"}, - {"categories": [{"name": "construction_method", "description": null}], "term": "Air-rotary", "definition": "Air-rotary"}, {"categories": [{"name": "construction_method", "description": null}], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": [{"name": "construction_method", "description": null}], "term": "Cable-tool", "definition": "Cable-tool"}, @@ -253,9 +250,9 @@ {"categories": [{"name": "activity_type", "description": null}], "term": "groundwater level", "definition": "groundwater level"}, {"categories": [{"name": "activity_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, - {"categories": [{"name": "activity_type", "description": null}], "term": "geothermal", "definition": "geothermal"}, - {"categories": [{"name": "sample_matrix", "description": null}], "term": "groundwater", "definition": "groundwater"}, + {"categories": [{"name": "sample_matrix", "description": null}], "term": "water", "definition": "water"}, + {"categories": [{"name": "sample_matrix", "description": null}], "term": "soil", "definition": "soil"}, {"categories": [{"name": "thing_type", "description": null}], "term": "observation well", "definition": "a well used to monitor groundwater levels"}, {"categories": [{"name": "thing_type", "description": null}], "term": "piezometer", "definition": "a type of observation well that measures pressure head in the aquifer"}, @@ -327,6 +324,13 @@ {"categories": [{"name": "sample_method", "description": null}], "term": "Continuous acoustic sounder", "definition": "Continuous acoustic sounder"}, {"categories": [{"name": "sample_method", "description": null}], "term": "Measurement not attempted", "definition": "Measurement not attempted"}, {"categories": [{"name": "sample_method", "description": null}], "term": "null placeholder", "definition": "null placeholder"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "bailer", "definition": "bailer"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "faucet at well head", "definition": "faucet at well head"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "faucet or outlet at house", "definition": "faucet or outlet at house"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "grab sample", "definition": "grab sample"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "pump", "definition": "pump"}, + {"categories": [{"name": "sample_method", "description": null}], "term": "thief sampler", "definition": "thief sampler"}, + {"categories": [{"name": "organization", "description": null}], "term": "USGS", "definition": "US Geological Survey"}, {"categories": [{"name": "organization", "description": null}], "term": "TWDB", "definition": "Texas Water Development Board"}, diff --git a/db/__init__.py b/db/__init__.py index 6d71baf61..b75475493 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -24,6 +24,7 @@ from db.contact import * from db.geochronology import * from db.geothermal import * +from db.field import * from db.group import * from db.lexicon import * from db.location import * diff --git a/db/field.py b/db/field.py new file mode 100644 index 000000000..5ed386a1a --- /dev/null +++ b/db/field.py @@ -0,0 +1,89 @@ +from datetime import datetime +from sqlalchemy import DateTime, ForeignKey +from sqlalchemy.orm import mapped_column, relationship, Mapped + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + + +class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): + """ + This table serves as the master log for all field visits. Each + record in this table represents a single, continuous collection event at a + specific Thing (e.g., a well) by a specific person on a specific date. + + This table's purpose is to store event-level metadata that is true for the + entire visit, such as the date, time, and the person responsible. It acts as + the parent container for all activities performed and all samples collected + during that single visit. + """ + + # Foreign Keys + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), + nullable=False, + comment="Foreign key to the Thing (e.g., sampling location) table.", + ) + + # Columns + # TODO: do we want to have a list of all present at the field event, or is it enough to capture the event_lead_name and sampler_name(s)? (AMP user research) + event_date: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + comment="Date and time of the field event.", + ) + event_lead_name: Mapped[str] = mapped_column( + nullable=False, comment="The name of the person leading the field event" + ) + # TODO: ask AMP if they care about this field. Is it needed? user research + collecting_organization: Mapped[str] = lexicon_term( + nullable=False, + comment="The organization that is collecting and storing the samples from the field event", + ) + notes: Mapped[str] = mapped_column( + nullable=True, + comment="Notes or comments about the field event.", + ) + # Relationships + thing: Mapped["Thing"] = relationship(back_populates="field_events") # noqa: F821 + field_activities: Mapped[list["FieldActivity"]] = relationship( + back_populates="field_event" + ) + + +class FieldActivity(Base, AutoBaseMixin, ReleaseMixin): + """ + This table serves as a log of the specific, distinct tasks + performed during a single `FieldEvent`. Its purpose is to correctly model + the one-to-many relationship where a single field visit can have multiple + objectives (e.g., collecting a water level and also collecting a water + sample for the lab). + + Each record in this table represents one type of work, such as + 'groundwater level', 'geochemical', or 'water chemistry'. By linking a + Sample record to a specific FieldActivity, the schema creates a clear and + unambiguous chain of custody, ensuring that every observation can be traced + back to the precise task that generated it. + """ + + # Foreign Keys + field_event_id: Mapped[int] = mapped_column( + ForeignKey("field_event.id", ondelete="CASCADE"), + nullable=False, + comment="Foreign key to the FieldEvent table.", + ) + + # Columns + activity_type: Mapped[str] = lexicon_term( + nullable=False, + comment="The type of activity performed during the field event (e.g., 'groundwater level', 'water chemistry', 'geothermal').", + ) + notes: Mapped[str] = mapped_column( + nullable=True, + comment="Notes or comments about the field activity.", + ) + + # Relationships + field_event: Mapped["FieldEvent"] = relationship(back_populates="field_activities") + samples: Mapped[list["Sample"]] = relationship( # noqa: F821 + back_populates="field_activities" + ) diff --git a/db/sample.py b/db/sample.py index 8e6c56f8f..8582eebd2 100644 --- a/db/sample.py +++ b/db/sample.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from sqlalchemy import DateTime, ForeignKey, UniqueConstraint +from sqlalchemy import DateTime, ForeignKey from sqlalchemy.orm import mapped_column, relationship, Mapped # import models from classes that are defined in separate files from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term -from db.thing import Thing from db.sensor import Sensor from typing import Optional @@ -28,19 +27,26 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): """ - Defines the Sample table, which stores data for individual - sampling events. + This table serves as an inventory of all the individual items collected or + generated by a FieldActivity. Each record represents a single, discrete + sample, which can be either a physical object (like a bottle of water or a + jar of soil) or a "virtual" object (representing the act of taking a + measurement, like a water level). + + Its purpose is to store the specific properties of each discrete sample, + such as its unique sample name (the label on the bottle), its matrix, and + any relevant physical measurements. """ # __table_name__ is inherited from AutoBaseMixin. - # --- Column Definitions --- - # Foreign Keys - thing_id: Mapped[int] = mapped_column( - ForeignKey("thing.id", ondelete="CASCADE"), - nullable=False, - comment="Foreign key to the Thing (e.g., sampling location) table.", + # --- Foreign Key Definitions --- + field_activity_id: Mapped[int] = mapped_column( + ForeignKey("field_activity.id"), nullable=False ) + + # --- Column Definitions --- + # nullable because sample can be collected by steel tape sensor_id: Mapped[Optional[int]] = mapped_column( ForeignKey("sensor.id"), @@ -54,79 +60,39 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Date and time of sample collection.", ) + sample_name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The unique identifier physically written on the sample container or used in field logs. Use for tracking/auditing purposes.", + ) sample_matrix: Mapped[str] = lexicon_term( nullable=False, comment="The material of the sample (e.g., 'gw', 'soil')." ) sample_method: Mapped[str] = lexicon_term( - comment="Method used to collect the sample.", nullable=True - ) - field_sample_id: Mapped[str] = mapped_column( - unique=True, nullable=False, comment="User-defined ID for field tracking." + comment="Method used to collect the sample.", nullable=False ) sampler_name: Mapped[str] = mapped_column( - nullable=False, comment="Name of the person who collected the sample." - ) - # TODO: is qc_sample required if we have duplicate_sample_number? are split samples recorded, or not? could be a user research question - # to guide development and future-proof - qc_sample: Mapped[str] = mapped_column( - default="Original", nullable=False, - comment="Quality control sample type (e.g., 'Original', 'Split', 'Field duplicate').", + comment="Name of the person who collected the sample. This may or may not be the person who lead the event (see FieldEvent table)", ) - sample_top: Mapped[float] = mapped_column( - nullable=True, comment="Top depth of a discrete sample interval." - ) - sample_bottom: Mapped[float] = mapped_column( - nullable=True, comment="Bottom depth of a discrete sample interval." + qc_type: Mapped[str] = mapped_column( + default="Normal", + nullable=False, + comment="Quality control sample type (e.g., 'Normal', 'Split', 'Field duplicate').", ) - # TODO: is qc_sample required if we have duplicate_sample_number? are split samples recorded, or not? could be a user research question - # to guide development and future-proof - duplicate_sample_number: Mapped[int] = mapped_column( - default=0, - comment="Identifier for duplicate samples (0 = original sample, not a duplicate, 1 = dup no.1, 2 = dup no.2, etc.).", + depth_top: Mapped[float] = mapped_column( + nullable=True, comment="Top depth of a discrete sample interval in ft." ) - activity_type: Mapped[str] = lexicon_term( - nullable=False, - comment="The type of sampling activity (e.g., 'geochemical', 'geothermal', 'groundwater level', 'water chemistry').", + depth_bottom: Mapped[float] = mapped_column( + nullable=True, comment="Bottom depth of a discrete sample interval in ft." ) + notes: Mapped[str] = mapped_column(nullable=True) # --- Relationship Definitions --- - thing: Mapped["Thing"] = relationship(back_populates="samples") - sensor: Mapped[Optional["Sensor"]] = relationship(back_populates="sample") - - # --- Table-level Arguments (e.g., Constraints) --- - # Unique samples should be based on the station_id, sample_date, sample_matrix, - # sample_top, sample_bottom, duplicate_sample, field_sample_id, and qc_sample fields. - __table_args__ = ( - UniqueConstraint( - "thing_id", - "sample_date", - "sample_matrix", - "sample_top", - "sample_bottom", - "duplicate_sample_number", - "field_sample_id", - "qc_sample", - name="uix_sample_uniqueness", - ), + field_activities: Mapped[list["FieldActivity"]] = relationship( # noqa: F821 + back_populates="samples" ) - - # ---Jake original code--- - # collection_timestamp = mapped_column(DateTime, nullable=False) - # collection_method = lexicon_term(nullable=False) - # - # thing_id = mapped_column( - # Integer, Foreign collection_timestamp = mapped_column(DateTime, nullable=False) - # collection_method = lexicon_term(nullable=False) - # - # thing_id = mapped_column( - # Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False - # ) - # thing = relationship("Thing")Key("thing.id", ondelete="CASCADE"), nullable=False - # ) - # thing = relationship("Thing") - - # wells = association_proxy("author_associations", "author") + sensor: Mapped["Sensor"] = relationship(back_populates="samples") # noqa: F821 # ============= EOF ============================================= diff --git a/db/sensor.py b/db/sensor.py index 1b7b52166..ada548d0e 100644 --- a/db/sensor.py +++ b/db/sensor.py @@ -40,11 +40,9 @@ class Sensor(Base, AutoBaseMixin, ReleaseMixin): recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) notes: Mapped[str] = mapped_column(String(50), nullable=True) - sample = relationship( + samples: Mapped[list["Sample"]] = relationship( # noqa: F821 "Sample", back_populates="sensor", - cascade="all, delete-orphan", - uselist=False, ) diff --git a/db/thing.py b/db/thing.py index b6117b9d0..a2ab09e89 100644 --- a/db/thing.py +++ b/db/thing.py @@ -18,8 +18,6 @@ from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType -from uuid import UUID - from db import lexicon_term from db.base import AutoBaseMixin, Base, ReleaseMixin @@ -82,8 +80,8 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): ) ) - samples = relationship( - "Sample", back_populates="thing", cascade="all, delete-orphan", uselist=True + field_events = relationship( + "FieldEvent", back_populates="thing", cascade="all, delete-orphan", uselist=True ) diff --git a/main.py b/main.py index 7a4f3cbcb..915d7460b 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,6 @@ import os import sentry_sdk from dotenv import load_dotenv -from starlette.middleware.base import BaseHTTPMiddleware -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware load_dotenv() @@ -44,8 +42,6 @@ from api.lexicon import router as lexicon_router -from api.geothermal import router as geothermal_router -from api.geochronology import router as geochronology_router from api.publication import router as publication_router from api.author import router as author_router from api.asset import router as asset_router @@ -55,9 +51,7 @@ app.include_router(asset_router) app.include_router(author_router) app.include_router(contact_router) -app.include_router(geochronology_router) app.include_router(geospatial_router) -app.include_router(geothermal_router) app.include_router(group_router) app.include_router(lexicon_router) app.include_router(location_router) diff --git a/schemas/sample.py b/schemas/sample.py index bdf5a4ab4..ed311cc52 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -54,22 +54,22 @@ class ValidateSample(BaseModel): # return sample_bottom sample_date: AwareDatetime | None = None - sample_top: float | None = None - sample_bottom: float | None = None + depth_top: float | None = None + depth_bottom: float | None = None @model_validator(mode="after") def validate_top_and_bottom(self) -> Self: """ - Validate that sample_top and sample_bottom are both defined or both None. + Validate that depth_top and depth_bottom are both defined or both None. """ - sample_top = getattr(self, "sample_top", None) - sample_bottom = getattr(self, "sample_bottom", None) + depth_top = getattr(self, "depth_top", None) + depth_bottom = getattr(self, "depth_bottom", None) - if (sample_top is not None and sample_bottom is None) or ( - sample_top is None and sample_bottom is not None + if (depth_top is not None and depth_bottom is None) or ( + depth_top is None and depth_bottom is not None ): raise ValueError( - "Sample top and bottom must both be defined or both must be None." + "Depth top and bottom must both be defined or both must be None." ) return self @@ -86,78 +86,45 @@ def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime: # -------- CREATE ---------- class CreateSample(BaseCreateModel, ValidateSample): - thing_id: int - activity_type: str - field_sample_id: str - sample_date: Annotated[AwareDatetime, PastDatetime()] - sampler_name: str # REFACTOR TODO: update with enum/restricted values - qc_sample: str = "Original" - + field_activity_id: int sensor_id: int | None = None - sample_matrix: str | None = ( - None # REFACTOR TODO: update with enum/restricted values - ) - sample_method: str | None = ( - None # REFACTOR TODO: update with enum/restricted values - ) - - duplicate_sample_number: int | None = 0 - - # REFACTOR TODO: update with numeric restrictions? Are negative values below ground and positive above? - # for example: wells below, rain above, and soil/rock could be at ground surface - sample_top: float | None = None - sample_bottom: float | None = None + sample_date: Annotated[AwareDatetime, PastDatetime()] + sample_name: str + sample_matrix: str + sample_method: str + sampler_name: str + qc_type: str + depth_top: float | None = None + depth_bottom: float | None = None # -------- UPDATE ---------- class UpdateSample(BaseUpdateModel, ValidateSample): - """ - Development notes: - - setting = None makes the field optional, but if it is defined it must be of that type. - """ - - thing_id: int | None = None # REFACTOR TODO: should users be able to change this? - activity_type: str | None = None - field_sample_id: str | None = None + thing: ThingResponse + field_activity_id: int | None = None + sensor_id: int | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None - sampler_name: str | None = None # REFACTOR TODO: update with enum/restricted values - qc_sample: str | None = None - - sensor_id: int | None = None # REFACTOR TODO: should users be able to change this? - sample_matrix: str | None = ( - None # REFACTOR TODO: update with enum/restricted values - ) - sample_method: str | None = ( - None # REFACTOR TODO: update with enum/restricted values - ) - - duplicate_sample_number: int | None = None - - # REFACTOR TODO: update with numeric restrictions? Are negative values below ground and positive above? - # for example: wells below, rain above, and soil/rock could be at ground surface - sample_top: float | None = None - sample_bottom: float | None = None + sample_name: str | None = None + sample_matrix: str | None = None + sample_method: str | None = None + sampler_name: str | None = None + qc_type: str | None = None + depth_top: float | None = None + depth_bottom: float | None = None # -------- RESPONSE ---------- class SampleResponse(BaseResponseModel): - thing: ThingResponse - activity_type: str - field_sample_id: str - sample_date: AwareDatetime - release_status: str - sampler_name: str - qc_sample: str - + field_activity_id: int sensor_id: int | None - sample_matrix: str | None - sample_method: str | None - - duplicate_sample_number: int | None - - sample_top: float | None - sample_bottom: float | None + sample_date: Annotated[AwareDatetime, PastDatetime()] + sample_name: str + sample_matrix: str + sample_method: str + sampler_name: str + qc_type: str + depth_top: float | None + depth_bottom: float | None # ============= EOF ============================================= diff --git a/tests/conftest.py b/tests/conftest.py index 5254df990..6d456cac5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -459,97 +459,86 @@ def second_asset(): @pytest.fixture(scope="session") -def groundwater_level_sample(water_well_thing, sensor): +def field_event(water_well_thing): with session_ctx() as session: - sample = Sample( - sample_date="2025-01-01T00:00:00Z", + field_event = FieldEvent( thing_id=water_well_thing.id, + event_date="2025-01-01T00:00:00Z", + event_lead_name="Sesame Mango", + collecting_organization="NMBGMR", + notes="field event fixture notes", + ) + session.add(field_event) + session.commit() + yield field_event + + +@pytest.fixture(scope="session") +def groundwater_level_field_activity(field_event): + with session_ctx() as session: + field_activity = FieldActivity( + field_event_id=field_event.id, activity_type="groundwater level", - sampler_name="Test Sampler", - release_status="draft", - field_sample_id=f"FS-{uuid.uuid4()}", - qc_sample="Original", - sensor_id=sensor.id, - sample_matrix="groundwater", - sample_method="manual", - duplicate_sample_number=0, - sample_top=None, - sample_bottom=None, + notes="field activity fixture notes", ) - session.add(sample) + session.add(field_activity) session.commit() - yield sample + yield field_activity @pytest.fixture(scope="session") -def water_chemistry_sample(water_well_thing, sensor): +def water_chemistry_field_activity(field_event): with session_ctx() as session: - sample = Sample( - sample_date="2025-01-01T00:00:00Z", - thing_id=water_well_thing.id, + field_activity = FieldActivity( + field_event_id=field_event.id, activity_type="water chemistry", - sampler_name="Test Sampler", - release_status="draft", - field_sample_id=f"FS-{uuid.uuid4()}", - qc_sample="Original", - sensor_id=sensor.id, - sample_matrix="groundwater", - sample_method="manual", - duplicate_sample_number=0, - sample_top=None, - sample_bottom=None, + notes="field activity fixture notes", ) - session.add(sample) + session.add(field_activity) session.commit() - yield sample + yield field_activity @pytest.fixture(scope="session") -def geothermal_sample(water_well_thing, sensor): +def groundwater_level_sample(groundwater_level_field_activity, sensor): with session_ctx() as session: sample = Sample( - sample_date="2025-01-01T00:00:00Z", - thing_id=water_well_thing.id, - activity_type="geothermal", - sampler_name="Test Sampler", - release_status="draft", - field_sample_id=f"FS-{uuid.uuid4()}", - qc_sample="Original", + field_activity_id=groundwater_level_field_activity.id, sensor_id=sensor.id, - sample_matrix="groundwater", - sample_method="manual", - duplicate_sample_number=0, - sample_top=None, - sample_bottom=None, + sample_date="2025-01-01T12:00:00Z", + sample_name="groundwater level sample name", + sample_matrix="water", + sample_method="Steel-tape measurement", + sampler_name="Esme Patterson", + qc_type="Normal", + depth_top=None, + depth_bottom=None, + notes="groundwater level sample fixture notes", ) session.add(sample) session.commit() yield sample -@pytest.fixture(scope="function") -def second_sample(water_well_thing, sensor): +@pytest.fixture(scope="session") +def water_chemistry_sample(water_chemistry_field_activity, sensor): with session_ctx() as session: sample = Sample( - thing_id=water_well_thing.id, - activity_type="groundwater level", - field_sample_id="FS-9999999", - sample_date="2025-01-01T00:00:00Z", - release_status="draft", - sampler_name="Test Sampler", - qc_sample="Duplicate", + field_activity_id=water_chemistry_field_activity.id, sensor_id=sensor.id, - sample_matrix="groundwater", - sample_method="manual", - duplicate_sample_number=3, - sample_top=2, - sample_bottom=3, + sample_date="2025-01-01T13:00:00Z", + sample_name="water chemistry sample name", + sample_matrix="water", + sample_method="grab sample", + sampler_name="Esme Patterson", + qc_type="Normal", + depth_top=None, + depth_bottom=None, + notes="water chemistry sample fixture notes", ) session.add(sample) session.commit() yield sample - session.delete(sample) - session.commit() @pytest.fixture(scope="session") diff --git a/tests/test_sample.py b/tests/test_sample.py index 8b32ff200..d71c847bd 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -43,34 +43,31 @@ def override_dependencies_fixture(): def test_validate_sample_top_and_bottom(): for i in range(2): - sample_top = 10.0 if i == 0 else None - sample_bottom = 5.0 if i == 1 else None + depth_top = 10.0 if i == 0 else None + depth_bottom = 5.0 if i == 1 else None with pytest.raises( ValidationError, - match="Sample top and bottom must both be defined or both must be None.", + match="Depth top and bottom must both be defined or both must be None.", ): - ValidateSample(sample_top=sample_top, sample_bottom=sample_bottom) + ValidateSample(depth_top=depth_top, depth_bottom=depth_bottom) # ============= Post tests for samples ============================================= -def test_add_sample(spring_thing, sensor): +def test_add_sample(groundwater_level_field_activity, sensor): """ Test adding a sample. """ payload = { - "thing_id": spring_thing.id, - "activity_type": "water chemistry", - "field_sample_id": "FS-1234567", - "sample_date": "2025-01-01T00:00:00Z", - "release_status": "draft", - "sampler_name": "Test Sampler", - "qc_sample": "Duplicate", + "field_activity_id": groundwater_level_field_activity.id, "sensor_id": sensor.id, - "sample_matrix": "groundwater", - "sample_method": "manual", - "duplicate_sample_number": 3, - "sample_top": 2, - "sample_bottom": 3, + "sample_date": "2025-01-01T14:00:00Z", + "sample_name": "second groundwater level field activity name", + "sample_matrix": "water", + "sample_method": "grab sample", + "sampler_name": "Ptolemy I Soter", + "qc_type": "Normal", + "depth_top": None, + "depth_bottom": None, } response = client.post( "/sample", @@ -80,19 +77,16 @@ def test_add_sample(spring_thing, sensor): assert response.status_code == 201 assert "id" in data assert "created_at" in data - assert data["thing"]["id"] == spring_thing.id - assert data["activity_type"] == payload["activity_type"] - assert data["field_sample_id"] == payload["field_sample_id"] - assert data["sample_date"] == payload["sample_date"] - assert data["release_status"] == payload["release_status"] - assert data["sampler_name"] == payload["sampler_name"] - assert data["qc_sample"] == payload["qc_sample"] + assert data["field_activity_id"] == payload["field_activity_id"] assert data["sensor_id"] == payload["sensor_id"] + assert data["sample_date"] == payload["sample_date"] + assert data["sample_name"] == payload["sample_name"] assert data["sample_matrix"] == payload["sample_matrix"] assert data["sample_method"] == payload["sample_method"] - assert data["duplicate_sample_number"] == payload["duplicate_sample_number"] - assert data["sample_top"] == payload["sample_top"] - assert data["sample_bottom"] == payload["sample_bottom"] + assert data["sampler_name"] == payload["sampler_name"] + assert data["qc_type"] == payload["qc_type"] + assert data["depth_top"] == payload["depth_top"] + assert data["depth_bottom"] == payload["depth_bottom"] # cleanup after adding the sample cleanup_post_test(Sample, data["id"]) diff --git a/transfers/transfer.py b/transfers/transfer.py index 5d5be3cbc..d71c1c869 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -131,6 +131,17 @@ def main_transfer(): message("TRANSFERRING WATER LEVELS") transfer_water_levels(sess) + """ + Developer's notes + + When transfering water chemistry data use the qc_type field to indicate + normal/blanks/duplicates instead of what comes from LU_SampleType. Use + those values, however, to map to the standard qc_type fields if applicable + (i.e. not applicable when sample type is "Soil or rock sample" or + "Precipitation," but is applicable when sample type is "Equipment blank" + or "Field duplicate") + """ + if init or transfer_link_ids_flag: message("TRANSFERRING LINK IDS") transfer_link_ids(sess) From 0ee9b04b8816217036a6a3cb7f085622db03af81 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 09:18:49 -0600 Subject: [PATCH 12/37] feat: implement POST for new sample/field --- api/sample.py | 10 +++++++++- db/field.py | 2 +- db/sample.py | 2 +- schemas/field.py | 14 ++++++++++++++ schemas/sample.py | 4 +++- services/sample_helper.py | 27 +++++++++++++++++++++++++++ tests/conftest.py | 5 +++++ tests/test_sample.py | 6 +++++- 8 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 schemas/field.py create mode 100644 services/sample_helper.py diff --git a/api/sample.py b/api/sample.py index 4871ab76b..a722f01c2 100644 --- a/api/sample.py +++ b/api/sample.py @@ -69,7 +69,15 @@ async def add_sample( Endpoint to add a sample. """ try: - return model_adder(session, Sample, sample_data, user=user) + sample = model_adder(session, Sample, sample_data, user=user) + field_event = sample.field_activity.field_event + thing = field_event.thing + + # add related objects to the response for serialization by Pydantic + setattr(sample, "field_event", field_event) + setattr(sample, "thing", thing) + + return sample except (IntegrityError, ProgrammingError) as e: database_error_handler(sample_data, e) diff --git a/db/field.py b/db/field.py index 5ed386a1a..3efed4c47 100644 --- a/db/field.py +++ b/db/field.py @@ -85,5 +85,5 @@ class FieldActivity(Base, AutoBaseMixin, ReleaseMixin): # Relationships field_event: Mapped["FieldEvent"] = relationship(back_populates="field_activities") samples: Mapped[list["Sample"]] = relationship( # noqa: F821 - back_populates="field_activities" + back_populates="field_activity" ) diff --git a/db/sample.py b/db/sample.py index 8582eebd2..877d95af6 100644 --- a/db/sample.py +++ b/db/sample.py @@ -89,7 +89,7 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): notes: Mapped[str] = mapped_column(nullable=True) # --- Relationship Definitions --- - field_activities: Mapped[list["FieldActivity"]] = relationship( # noqa: F821 + field_activity: Mapped["FieldActivity"] = relationship( # noqa: F821 back_populates="samples" ) sensor: Mapped["Sensor"] = relationship(back_populates="samples") # noqa: F821 diff --git a/schemas/field.py b/schemas/field.py new file mode 100644 index 000000000..e72c3d254 --- /dev/null +++ b/schemas/field.py @@ -0,0 +1,14 @@ +from pydantic import AwareDatetime + +from schemas import BaseResponseModel + + +# RESPONSE --------------------------------------------------------------------- + + +class FieldEventResponse(BaseResponseModel): + thing_id: int + event_date: AwareDatetime + event_lead_name: str + collecting_organization: str | None + notes: str | None diff --git a/schemas/sample.py b/schemas/sample.py index ed311cc52..fc1f95282 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -26,6 +26,7 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.thing import ThingResponse +from schemas.field import FieldEventResponse """ REFACTOR TODO: can we use inheritance for commonly defined fields and then set them as optional @@ -100,7 +101,6 @@ class CreateSample(BaseCreateModel, ValidateSample): # -------- UPDATE ---------- class UpdateSample(BaseUpdateModel, ValidateSample): - thing: ThingResponse field_activity_id: int | None = None sensor_id: int | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None @@ -115,6 +115,8 @@ class UpdateSample(BaseUpdateModel, ValidateSample): # -------- RESPONSE ---------- class SampleResponse(BaseResponseModel): + thing: ThingResponse + field_event: FieldEventResponse field_activity_id: int sensor_id: int | None sample_date: Annotated[AwareDatetime, PastDatetime()] diff --git a/services/sample_helper.py b/services/sample_helper.py new file mode 100644 index 000000000..59eefa347 --- /dev/null +++ b/services/sample_helper.py @@ -0,0 +1,27 @@ +from fastapi_pagination.ext.sqlalchemy import paginate +from sqlalchemy import select +from sqlalchemy.orm import Session + +from db import Sample, Thing, FieldEvent, FieldActivity +from services.query_helper import order_sort_filter + + +def get_samples( + session: Session, + order: str | None = None, + sort: str | None = None, + filter_: str | None = None, +): + query = select(Sample, Thing, FieldEvent) + query = query.join(FieldActivity, Sample.field_activity_id == FieldActivity.id) + query = query.join(FieldEvent, FieldActivity.field_event_id == FieldEvent.id) + query = query.join(Thing, FieldEvent.thing_id == Thing.id) + + query = order_sort_filter(query, Sample, sort, order, filter_) + + return paginate(query, conn=session) + + +def get_sample_by_id(session: Session, sample_id: int) -> Sample | None: + query = select(Sample).where(Sample.id == sample_id) + return session.execute(query).scalar_one_or_none() diff --git a/tests/conftest.py b/tests/conftest.py index 6d456cac5..649e3d1f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -467,6 +467,7 @@ def field_event(water_well_thing): event_lead_name="Sesame Mango", collecting_organization="NMBGMR", notes="field event fixture notes", + release_status="draft", ) session.add(field_event) session.commit() @@ -480,6 +481,7 @@ def groundwater_level_field_activity(field_event): field_event_id=field_event.id, activity_type="groundwater level", notes="field activity fixture notes", + release_status="draft", ) session.add(field_activity) session.commit() @@ -493,6 +495,7 @@ def water_chemistry_field_activity(field_event): field_event_id=field_event.id, activity_type="water chemistry", notes="field activity fixture notes", + release_status="draft", ) session.add(field_activity) session.commit() @@ -514,6 +517,7 @@ def groundwater_level_sample(groundwater_level_field_activity, sensor): depth_top=None, depth_bottom=None, notes="groundwater level sample fixture notes", + release_status="draft", ) session.add(sample) session.commit() @@ -535,6 +539,7 @@ def water_chemistry_sample(water_chemistry_field_activity, sensor): depth_top=None, depth_bottom=None, notes="water chemistry sample fixture notes", + release_status="draft", ) session.add(sample) session.commit() diff --git a/tests/test_sample.py b/tests/test_sample.py index d71c847bd..eca2f0d75 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -53,7 +53,7 @@ def test_validate_sample_top_and_bottom(): # ============= Post tests for samples ============================================= -def test_add_sample(groundwater_level_field_activity, sensor): +def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): """ Test adding a sample. """ @@ -68,15 +68,19 @@ def test_add_sample(groundwater_level_field_activity, sensor): "qc_type": "Normal", "depth_top": None, "depth_bottom": None, + "release_status": "draft", } response = client.post( "/sample", json=payload, ) data = response.json() + assert response.status_code == 201 assert "id" in data assert "created_at" in data + assert data["thing"]["id"] == water_well_thing.id + assert data["field_event"]["id"] == groundwater_level_field_activity.field_event_id assert data["field_activity_id"] == payload["field_activity_id"] assert data["sensor_id"] == payload["sensor_id"] assert data["sample_date"] == payload["sample_date"] From 6044b683afc8470570541540aa768a6c65a628f3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 09:33:23 -0600 Subject: [PATCH 13/37] refactor: revise test_409_add_sample_ivnalid_sample_name for updates --- tests/test_sample.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_sample.py b/tests/test_sample.py index eca2f0d75..7cc0587b1 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -96,24 +96,24 @@ def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): cleanup_post_test(Sample, data["id"]) -def test_409_add_sample_invalid_field_sample_id(water_chemistry_sample, spring_thing): +def test_409_add_sample_invalid_sample_name( + groundwater_level_field_activity, groundwater_level_sample, sensor +): """ - Test adding a sample with an invalid field_sample_id. + Test that a 409 error is raised if a duplicate sample_name is in the payload """ payload = { - "thing_id": spring_thing.id, - "activity_type": "water chemistry", - "field_sample_id": water_chemistry_sample.field_sample_id, # This should already exist - "sample_date": "2025-01-01T00:00:00Z", + "field_activity_id": groundwater_level_field_activity.id, + "sensor_id": sensor.id, + "sample_date": "2025-01-01T14:00:00Z", + "sample_name": groundwater_level_sample.sample_name, + "sample_matrix": "water", + "sample_method": "grab sample", + "sampler_name": "Ptolemy I Soter", + "qc_type": "Normal", + "depth_top": None, + "depth_bottom": None, "release_status": "draft", - "sampler_name": "Test Sampler", - "qc_sample": "Duplicate", - "sensor_id": None, - "sample_matrix": "groundwater", - "sample_method": "manual", - "duplicate_sample_number": 3, - "sample_top": 2, - "sample_bottom": 3, } response = client.post( "/sample", @@ -121,20 +121,20 @@ def test_409_add_sample_invalid_field_sample_id(water_chemistry_sample, spring_t ) data = response.json() assert response.status_code == 409 - assert data["detail"][0]["loc"] == ["body", "field_sample_id"] + assert data["detail"][0]["loc"] == ["body", "sample_name"] assert ( data["detail"][0]["msg"] - == f"Sample with field_sample_id {water_chemistry_sample.field_sample_id} already exists." + == f"Sample with sample_name {groundwater_level_sample.sample_name} already exists." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == { - "field_sample_id": water_chemistry_sample.field_sample_id + "sample_name": groundwater_level_sample.sample_name } -def test_409_add_sample_invalid_thing_id(): +def test_409_add_sample_invalid_field_activity_id(): """ - Test adding a sample with an invalid thing_id. + Test adding a sample with an invalid field_activity_id. """ payload = { "thing_id": 9999999, From bf843f2467c8360c50babb363a0d19e0cbedea15 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 13:57:04 -0600 Subject: [PATCH 14/37] feat: enable many-to-many fieldevent/contact --- db/contact.py | 13 +++++++++++++ db/field.py | 43 ++++++++++++++++++++++++++++++++++++++++--- db/sample.py | 5 +++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/db/contact.py b/db/contact.py index bbff76178..5e985154f 100644 --- a/db/contact.py +++ b/db/contact.py @@ -71,6 +71,19 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) things = association_proxy("thing_associations", "thing") + # Proxy to directly access the FieldEvent objects in which this Contact participated. + # fmt: off + field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # noqa: F821 + # fmt: on + relationship( + "FieldEventContactAssociation", + back_populates="contact", + cascade="all, delete-orphan", + passive_deletes=True, + ) + ) + field_events = association_proxy("field_event_contact_associations", "field_event") + class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( diff --git a/db/field.py b/db/field.py index 3efed4c47..d77a08f5e 100644 --- a/db/field.py +++ b/db/field.py @@ -1,10 +1,34 @@ from datetime import datetime from sqlalchemy import DateTime, ForeignKey from sqlalchemy.orm import mapped_column, relationship, Mapped +from sqlalchemy.ext.associationproxy import association_proxy from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term +class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + This association table is to create a many-to-many relationship between + FieldEvent and Contact. These are participants in the field event. + """ + + # --- Foreign keys --- + field_event_id: Mapped[int] = mapped_column( + ForeignKey("field_event.id", ondelete="CASCADE"), + nullable=False, + comment="Foreign key to the FieldEvent table.", + ) + contact_id: Mapped[str] = mapped_column( + ForeignKey("contact.id", ondelete="CASCADE"), + nullable=False, + comment="Foreign key to the Contact table", + ) + + # --- Relationships --- + field_event: Mapped[list["FieldEvent"]] = relationship("FieldEvent") + contact: Mapped[list["Contact"]] = relationship("Contact") # noqa: F821 + + class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): """ This table serves as the master log for all field visits. Each @@ -25,7 +49,6 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): ) # Columns - # TODO: do we want to have a list of all present at the field event, or is it enough to capture the event_lead_name and sampler_name(s)? (AMP user research) event_date: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, @@ -34,20 +57,34 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): event_lead_name: Mapped[str] = mapped_column( nullable=False, comment="The name of the person leading the field event" ) - # TODO: ask AMP if they care about this field. Is it needed? user research collecting_organization: Mapped[str] = lexicon_term( nullable=False, + default="NMBGMR", # TODO: put default in schema comment="The organization that is collecting and storing the samples from the field event", ) notes: Mapped[str] = mapped_column( nullable=True, comment="Notes or comments about the field event.", ) - # Relationships + # --- Relationships --- thing: Mapped["Thing"] = relationship(back_populates="field_events") # noqa: F821 field_activities: Mapped[list["FieldActivity"]] = relationship( back_populates="field_event" ) + field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( + relationship( + "FieldEventContactAssociation", + back_populates="field_event", + cascade="all, delete-orphan", + passive_deletes=True, + ) + ) + + # --- Association Proxies --- + # Proxy to directly access the Contact objects participating in this event. + contacts: Mapped[list["Contact"]] = association_proxy( # noqa: F821 + "field_event_contact_associations", "contact" + ) class FieldActivity(Base, AutoBaseMixin, ReleaseMixin): diff --git a/db/sample.py b/db/sample.py index 877d95af6..3b5d506e3 100644 --- a/db/sample.py +++ b/db/sample.py @@ -15,6 +15,7 @@ # =============================================================================== from sqlalchemy import DateTime, ForeignKey from sqlalchemy.orm import mapped_column, relationship, Mapped +from sqlalchemy.ext.associationproxy import association_proxy # import models from classes that are defined in separate files from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -94,5 +95,9 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): ) sensor: Mapped["Sensor"] = relationship(back_populates="samples") # noqa: F821 + # association proxies to help keep code DRY + field_event = association_proxy("field_activity", "field_event") + thing = association_proxy("field_activity", "field_event.thing") + # ============= EOF ============================================= From 3dff2fad70a8c10d66b292c0747769535f46b008 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 13:57:23 -0600 Subject: [PATCH 15/37] feat: enable many-to-many fieldevent/contact --- db/contact.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/contact.py b/db/contact.py index 5e985154f..6f6d72597 100644 --- a/db/contact.py +++ b/db/contact.py @@ -73,9 +73,9 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): # Proxy to directly access the FieldEvent objects in which this Contact participated. # fmt: off - field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # noqa: F821 + field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # fmt: skip # noqa: F821 # fmt: on - relationship( + relationship( "FieldEventContactAssociation", back_populates="contact", cascade="all, delete-orphan", From ec550c06cc9ec44b92a42d77a8c19b13adf007ca Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 13:58:25 -0600 Subject: [PATCH 16/37] refactor: remove sample helper - use association proxies instead for DRY --- services/sample_helper.py | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 services/sample_helper.py diff --git a/services/sample_helper.py b/services/sample_helper.py deleted file mode 100644 index 59eefa347..000000000 --- a/services/sample_helper.py +++ /dev/null @@ -1,27 +0,0 @@ -from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy import select -from sqlalchemy.orm import Session - -from db import Sample, Thing, FieldEvent, FieldActivity -from services.query_helper import order_sort_filter - - -def get_samples( - session: Session, - order: str | None = None, - sort: str | None = None, - filter_: str | None = None, -): - query = select(Sample, Thing, FieldEvent) - query = query.join(FieldActivity, Sample.field_activity_id == FieldActivity.id) - query = query.join(FieldEvent, FieldActivity.field_event_id == FieldEvent.id) - query = query.join(Thing, FieldEvent.thing_id == Thing.id) - - query = order_sort_filter(query, Sample, sort, order, filter_) - - return paginate(query, conn=session) - - -def get_sample_by_id(session: Session, sample_id: int) -> Sample | None: - query = select(Sample).where(Sample.id == sample_id) - return session.execute(query).scalar_one_or_none() From bb7b26d3d0f77afe9b214d1369514155d0b5fdbe Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 13:58:47 -0600 Subject: [PATCH 17/37] note: remove old notes --- db/contact.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/contact.py b/db/contact.py index 6f6d72597..d2ac9466d 100644 --- a/db/contact.py +++ b/db/contact.py @@ -73,7 +73,7 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): # Proxy to directly access the FieldEvent objects in which this Contact participated. # fmt: off - field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # fmt: skip # noqa: F821 + field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # noqa: F821 # fmt: on relationship( "FieldEventContactAssociation", From 85d19f530233fe51108d573a299c6952191ce6f0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 14:07:33 -0600 Subject: [PATCH 18/37] refactor: update sample API and tests for revised schema --- api/sample.py | 26 ++++++++++++++++---------- tests/test_sample.py | 35 +++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/api/sample.py b/api/sample.py index a722f01c2..80c039a91 100644 --- a/api/sample.py +++ b/api/sample.py @@ -38,6 +38,11 @@ ) +# TODO: add the following database validation handlers +# invalid sample_id +# invalid lexicon terms + + def database_error_handler( payload: CreateSample | UpdateSample, error: IntegrityError | ProgrammingError ) -> None: @@ -45,7 +50,6 @@ def database_error_handler( Handle errors raised by the database when adding or updating a sample. """ error_message = error.orig.args[0]["M"] - print(error_message) if ( error_message == 'duplicate key value violates unique constraint "sample_sample_name_key"' @@ -56,6 +60,16 @@ def database_error_handler( "type": "value_error", "input": {"sample_name": payload.sample_name}, } + elif ( + error_message + == 'insert or update on table "sample" violates foreign key constraint "sample_field_activity_id_fkey"' + ): + detail = { + "loc": ["body", "field_activity_id"], + "msg": f"FieldActivity with ID {payload.field_activity_id} does not exist.", + "type": "value_error", + "input": {"field_activity_id": payload.field_activity_id}, + } raise PydanticStyleException(status_code=HTTP_409_CONFLICT, detail=[detail]) @@ -69,15 +83,7 @@ async def add_sample( Endpoint to add a sample. """ try: - sample = model_adder(session, Sample, sample_data, user=user) - field_event = sample.field_activity.field_event - thing = field_event.thing - - # add related objects to the response for serialization by Pydantic - setattr(sample, "field_event", field_event) - setattr(sample, "thing", thing) - - return sample + return model_adder(session, Sample, sample_data, user=user) except (IntegrityError, ProgrammingError) as e: database_error_handler(sample_data, e) diff --git a/tests/test_sample.py b/tests/test_sample.py index 7cc0587b1..8a55963f4 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -81,6 +81,7 @@ def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): assert "created_at" in data assert data["thing"]["id"] == water_well_thing.id assert data["field_event"]["id"] == groundwater_level_field_activity.field_event_id + assert data["field_activity"]["id"] == groundwater_level_field_activity.id assert data["field_activity_id"] == payload["field_activity_id"] assert data["sensor_id"] == payload["sensor_id"] assert data["sample_date"] == payload["sample_date"] @@ -132,24 +133,24 @@ def test_409_add_sample_invalid_sample_name( } -def test_409_add_sample_invalid_field_activity_id(): +def test_409_add_sample_invalid_field_activity_id( + groundwater_level_field_activity, groundwater_level_sample, sensor +): """ Test adding a sample with an invalid field_activity_id. """ payload = { - "thing_id": 9999999, - "activity_type": "water chemistry", - "field_sample_id": "FS-9999999", - "sample_date": "2025-01-01T00:00:00Z", + "field_activity_id": 999999, + "sensor_id": sensor.id, + "sample_date": "2025-01-01T14:00:00Z", + "sample_name": "yet another sample name", + "sample_matrix": "water", + "sample_method": "grab sample", + "sampler_name": "Ptolemy I Soter", + "qc_type": "Normal", + "depth_top": None, + "depth_bottom": None, "release_status": "draft", - "sampler_name": "Test Sampler", - "qc_sample": "Duplicate", - "sensor_id": None, - "sample_matrix": "groundwater", - "sample_method": "manual", - "duplicate_sample_number": 3, - "sample_top": 2, - "sample_bottom": 3, } response = client.post( "/sample", @@ -157,13 +158,15 @@ def test_409_add_sample_invalid_field_activity_id(): ) data = response.json() assert response.status_code == 409 - assert data["detail"][0]["loc"] == ["body", "thing_id"] + assert data["detail"][0]["loc"] == ["body", "field_activity_id"] assert ( data["detail"][0]["msg"] - == f"Thing with ID {payload['thing_id']} does not exist." + == f"FieldActivity with ID {payload['field_activity_id']} does not exist." ) assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"thing_id": payload["thing_id"]} + assert data["detail"][0]["input"] == { + "field_activity_id": payload["field_activity_id"] + } # ============= Patch tests for samples ============================================= From f94f51a133869fb950c95e59162e6d04330384db Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 14:31:29 -0600 Subject: [PATCH 19/37] note: note validations required for sample --- api/sample.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/sample.py b/api/sample.py index 80c039a91..8de9f741e 100644 --- a/api/sample.py +++ b/api/sample.py @@ -41,6 +41,7 @@ # TODO: add the following database validation handlers # invalid sample_id # invalid lexicon terms +# sample_date of the Sample model cannot be before the event_date of the FieldEvent model def database_error_handler( From fb167f76185796e26e32f4b4bd2c0edc69c1e26a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 14:54:32 -0600 Subject: [PATCH 20/37] refactor: update PATCH sample tests for revised schema --- tests/test_sample.py | 52 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/tests/test_sample.py b/tests/test_sample.py index 8a55963f4..dedf7c815 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -170,25 +170,31 @@ def test_409_add_sample_invalid_field_activity_id( # ============= Patch tests for samples ============================================= -def test_patch_sample(water_chemistry_sample): +def test_patch_sample( + water_chemistry_sample, second_sensor, groundwater_level_field_activity +): """ Test updating a sample. """ payload = { - "sampler_name": "test sample b", - "sample_method": "continuous", + "field_activity_id": groundwater_level_field_activity.id, + "sensor_id": second_sensor.id, "sample_date": "2025-01-02T00:00:00Z", + "sample_name": "patched sample name", + "sample_matrix": "soil", + "sample_method": "bailer", "release_status": "private", + "sampler_name": "test sample b", + "qc_type": "Split", + "depth_top": 10.0, + "depth_bottom": 20.0, } response = client.patch(f"/sample/{water_chemistry_sample.id}", json=payload) assert response.status_code == 200 data = response.json() - assert data["id"] == water_chemistry_sample.id - assert data["sampler_name"] == payload["sampler_name"] - assert data["sample_date"] == payload["sample_date"] - assert data["sample_method"] == payload["sample_method"] - assert data["release_status"] == payload["release_status"] + for key, value in payload.items(): + assert data[key] == value # rollback after updating the sample cleanup_patch_test(Sample, payload, water_chemistry_sample) @@ -210,38 +216,38 @@ def test_patch_sample_404_not_found(water_chemistry_sample): assert data["detail"] == "Sample with ID 999 not found." -def test_409_patch_sample_invalid_field_sample_id( - water_chemistry_sample, second_sample +def test_409_patch_sample_invalid_sample_name( + water_chemistry_sample, groundwater_level_sample ): """ - Test updating a sample with an invalid field_sample_id. + Test updating a sample with an invalid sample_name. """ payload = { - "field_sample_id": water_chemistry_sample.field_sample_id, # This should already exist + "sample_name": groundwater_level_sample.sample_name, # This should already exist } response = client.patch( - f"/sample/{second_sample.id}", + f"/sample/{water_chemistry_sample.id}", json=payload, ) data = response.json() assert response.status_code == 409 - assert data["detail"][0]["loc"] == ["body", "field_sample_id"] + assert data["detail"][0]["loc"] == ["body", "sample_name"] assert ( data["detail"][0]["msg"] - == f"Sample with field_sample_id {payload['field_sample_id']} already exists." + == f"Sample with sample_name {payload['sample_name']} already exists." ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == { - "field_sample_id": water_chemistry_sample.field_sample_id + "sample_name": groundwater_level_sample.sample_name } -def test_409_patch_sample_invalid_thing_id(water_chemistry_sample): +def test_409_patch_sample_invalid_field_activity_id(water_chemistry_sample): """ - Test updating a sample with an invalid thing_id. + Test updating a sample with an invalid field_activity_id. """ payload = { - "thing_id": 9999999, + "field_activity_id": 9999999, } response = client.patch( f"/sample/{water_chemistry_sample.id}", @@ -249,13 +255,15 @@ def test_409_patch_sample_invalid_thing_id(water_chemistry_sample): ) data = response.json() assert response.status_code == 409 - assert data["detail"][0]["loc"] == ["body", "thing_id"] + assert data["detail"][0]["loc"] == ["body", "field_activity_id"] assert ( data["detail"][0]["msg"] - == f"Thing with ID {payload['thing_id']} does not exist." + == f"FieldActivity with ID {payload['field_activity_id']} does not exist." ) assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"thing_id": payload["thing_id"]} + assert data["detail"][0]["input"] == { + "field_activity_id": payload["field_activity_id"] + } # ============= Get tests for samples ============================================= From 85d36ac2fd6235e110d712814c4c4ae87ef18480 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 15:07:46 -0600 Subject: [PATCH 21/37] feat: add field_activity to sample response for frontend use --- schemas/field.py | 5 +++++ schemas/sample.py | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/schemas/field.py b/schemas/field.py index e72c3d254..279b5a5eb 100644 --- a/schemas/field.py +++ b/schemas/field.py @@ -6,6 +6,11 @@ # RESPONSE --------------------------------------------------------------------- +class FieldActivityResponse(BaseResponseModel): + field_event_id: int + activity_type: str + + class FieldEventResponse(BaseResponseModel): thing_id: int event_date: AwareDatetime diff --git a/schemas/sample.py b/schemas/sample.py index fc1f95282..aaada80c6 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -26,7 +26,7 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.thing import ThingResponse -from schemas.field import FieldEventResponse +from schemas.field import FieldEventResponse, FieldActivityResponse """ REFACTOR TODO: can we use inheritance for commonly defined fields and then set them as optional @@ -101,7 +101,7 @@ class CreateSample(BaseCreateModel, ValidateSample): # -------- UPDATE ---------- class UpdateSample(BaseUpdateModel, ValidateSample): - field_activity_id: int | None = None + field_activity_id: int | None = None # TODO: should this be editable? sensor_id: int | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None sample_name: str | None = None @@ -115,8 +115,18 @@ class UpdateSample(BaseUpdateModel, ValidateSample): # -------- RESPONSE ---------- class SampleResponse(BaseResponseModel): + """ + Developer's note + + The frontend uses multiple fields for a thing, field_even, and field_activity, + which is why full ThingResponse, FieldEventResponse, and FieldActivityResponse + are returned. If the response becomes too large and slow, we can use + .model_dump() and exlude fields to reduce the size. + """ + thing: ThingResponse field_event: FieldEventResponse + field_activity: FieldActivityResponse field_activity_id: int sensor_id: int | None sample_date: Annotated[AwareDatetime, PastDatetime()] From 95c65c807cb1503823ac006174bb60ebd38983fe Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 15:16:44 -0600 Subject: [PATCH 22/37] note: note reusable utc converter function for schemas --- schemas/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/schemas/__init__.py b/schemas/__init__.py index 698dc02dd..9fdd22198 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -39,4 +39,14 @@ class BaseResponseModel(BaseModel): ) +# TODO: write function to convert any datetime field to UTC for use throughout +# for schema field_validators +# e.g. +# def convert_datetime_field_to_utc(dt_field): +# ... +# +# @field_validator("dt_field_name") +# def convert_to_utc(dt_field_name): +# return convert_datetime_field_to_utc(dt_field_name) + # ============= EOF ============================================= From 63623ff8647a8a99d081d17901268b40f5d06a18 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 15:17:57 -0600 Subject: [PATCH 23/37] refactor: remove outdate note --- schemas/sample.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/schemas/sample.py b/schemas/sample.py index aaada80c6..6be1360f6 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -28,11 +28,6 @@ from schemas.thing import ThingResponse from schemas.field import FieldEventResponse, FieldActivityResponse -""" -REFACTOR TODO: can we use inheritance for commonly defined fields and then set them as optional -or not between Create, Update, and Response schemas? -""" - # -------- VALIDATE ---------- class ValidateSample(BaseModel): From 2d12e78021ff3559909137c20510e2f9afe2fbe5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 18 Sep 2025 15:25:11 -0600 Subject: [PATCH 24/37] refactor: update all sample tests for revised schema --- schemas/sample.py | 3 ++ tests/conftest.py | 24 +++++++++++++++ tests/test_sample.py | 73 ++++++++++++++++++++++++-------------------- 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/schemas/sample.py b/schemas/sample.py index 6be1360f6..4d48fc400 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -90,6 +90,7 @@ class CreateSample(BaseCreateModel, ValidateSample): sample_method: str sampler_name: str qc_type: str + notes: str | None = None depth_top: float | None = None depth_bottom: float | None = None @@ -104,6 +105,7 @@ class UpdateSample(BaseUpdateModel, ValidateSample): sample_method: str | None = None sampler_name: str | None = None qc_type: str | None = None + notes: str | None = None depth_top: float | None = None depth_bottom: float | None = None @@ -130,6 +132,7 @@ class SampleResponse(BaseResponseModel): sample_method: str sampler_name: str qc_type: str + notes: str | None depth_top: float | None depth_bottom: float | None diff --git a/tests/conftest.py b/tests/conftest.py index 649e3d1f5..04917f088 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -546,6 +546,30 @@ def water_chemistry_sample(water_chemistry_field_activity, sensor): yield sample +@pytest.fixture(scope="function") +def sample_to_delete(water_chemistry_field_activity, sensor): + with session_ctx() as session: + sample = Sample( + field_activity_id=water_chemistry_field_activity.id, + sensor_id=sensor.id, + sample_date="2025-01-01T13:00:00Z", + sample_name="sample to delete", + sample_matrix="water", + sample_method="grab sample", + sampler_name="Esme Patterson", + qc_type="Normal", + depth_top=None, + depth_bottom=None, + notes="water chemistry sample fixture notes", + release_status="draft", + ) + session.add(sample) + session.commit() + yield sample + session.delete(sample) + session.commit() + + @pytest.fixture(scope="session") def groundwater_level_observation(sensor, groundwater_level_sample): with session_ctx() as session: diff --git a/tests/test_sample.py b/tests/test_sample.py index dedf7c815..3024e4c05 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -65,6 +65,7 @@ def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): "sample_matrix": "water", "sample_method": "grab sample", "sampler_name": "Ptolemy I Soter", + "notes": "posted notes", "qc_type": "Normal", "depth_top": None, "depth_bottom": None, @@ -89,6 +90,7 @@ def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): assert data["sample_matrix"] == payload["sample_matrix"] assert data["sample_method"] == payload["sample_method"] assert data["sampler_name"] == payload["sampler_name"] + assert data["notes"] == payload["notes"] assert data["qc_type"] == payload["qc_type"] assert data["depth_top"] == payload["depth_top"] assert data["depth_bottom"] == payload["depth_bottom"] @@ -186,6 +188,7 @@ def test_patch_sample( "release_status": "private", "sampler_name": "test sample b", "qc_type": "Split", + "notes": "patched notes", "depth_top": 10.0, "depth_bottom": 20.0, } @@ -267,36 +270,42 @@ def test_409_patch_sample_invalid_field_activity_id(water_chemistry_sample): # ============= Get tests for samples ============================================= -def test_get_samples( - water_chemistry_sample, groundwater_level_sample, geothermal_sample -): +def test_get_samples(water_chemistry_sample, groundwater_level_sample): """ Test retrieving samples """ response = client.get("/sample") assert response.status_code == 200 data = response.json() - assert len(data["items"]) == 3 + assert len(data["items"]) == 2 for item in data["items"]: assert "id" in item assert "created_at" in item - assert "thing" in item - assert "activity_type" in item - assert "field_sample_id" in item - assert "sample_date" in item assert "release_status" in item - assert "sampler_name" in item - assert "qc_sample" in item + assert "thing" in item + assert "field_event" in item + assert "field_activity" in item + assert "field_activity_id" in item assert "sensor_id" in item + assert "sample_date" in item + assert "sample_name" in item assert "sample_matrix" in item assert "sample_method" in item - assert "duplicate_sample_number" in item - assert "sample_top" in item - assert "sample_bottom" in item - - -def test_get_sample_by_id(water_chemistry_sample, water_well_thing): + assert "sampler_name" in item + assert "qc_type" in item + assert "depth_top" in item + assert "depth_bottom" in item + assert "notes" in item + + +def test_get_sample_by_id( + water_chemistry_sample, + water_chemistry_field_activity, + field_event, + water_well_thing, + sensor, +): """ Test retrieving a sample by its ID. """ @@ -308,21 +317,19 @@ def test_get_sample_by_id(water_chemistry_sample, water_well_thing): "+00:00", "Z" ) assert data["thing"]["id"] == water_well_thing.id - assert data["activity_type"] == water_chemistry_sample.activity_type - assert data["field_sample_id"] == water_chemistry_sample.field_sample_id + assert data["field_event"]["id"] == field_event.id + assert data["field_activity"]["id"] == water_chemistry_field_activity.id + assert data["field_activity_id"] == water_chemistry_field_activity.id + assert data["sensor_id"] == sensor.id assert data["sample_date"] == water_chemistry_sample.sample_date - assert data["release_status"] == water_chemistry_sample.release_status - assert data["sampler_name"] == water_chemistry_sample.sampler_name - assert data["qc_sample"] == water_chemistry_sample.qc_sample - assert data["sensor_id"] == water_chemistry_sample.sensor_id + assert data["sample_name"] == water_chemistry_sample.sample_name assert data["sample_matrix"] == water_chemistry_sample.sample_matrix assert data["sample_method"] == water_chemistry_sample.sample_method - assert ( - data["duplicate_sample_number"] - == water_chemistry_sample.duplicate_sample_number - ) - assert data["sample_top"] == water_chemistry_sample.sample_top - assert data["sample_bottom"] == water_chemistry_sample.sample_bottom + assert data["sampler_name"] == water_chemistry_sample.sampler_name + assert data["qc_type"] == water_chemistry_sample.qc_type + assert data["notes"] == water_chemistry_sample.notes + assert data["depth_top"] == water_chemistry_sample.depth_top + assert data["depth_bottom"] == water_chemistry_sample.depth_bottom def test_get_sample_by_id_404_not_found(water_chemistry_sample): @@ -338,18 +345,18 @@ def test_get_sample_by_id_404_not_found(water_chemistry_sample): # DELETE tests ================================================================= -def test_delete_sample(second_sample): - response = client.delete(f"/sample/{second_sample.id}") +def test_delete_sample(sample_to_delete): + response = client.delete(f"/sample/{sample_to_delete.id}") assert response.status_code == 204 # verify the sample is deleted - response = client.get(f"/sample/{second_sample.id}") + response = client.get(f"/sample/{sample_to_delete.id}") assert response.status_code == 404 data = response.json() - assert data["detail"] == f"Sample with ID {second_sample.id} not found." + assert data["detail"] == f"Sample with ID {sample_to_delete.id} not found." -def test_delete_sample_404_not_found(second_sample): +def test_delete_sample_404_not_found(sample_to_delete): bad_sample_id = 999999 response = client.delete(f"/sample/{bad_sample_id}") assert response.status_code == 404 From 2b6637a7b186c825e237aa05928d8dedd306a690 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 11:02:54 -0600 Subject: [PATCH 25/37] refactor: use joinedload to prevent N+1 issues with lazy loading --- api/sample.py | 14 +++++++++----- services/sample_helper.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 services/sample_helper.py diff --git a/api/sample.py b/api/sample.py index 8de9f741e..f26b9c6db 100644 --- a/api/sample.py +++ b/api/sample.py @@ -28,9 +28,10 @@ from db.sample import Sample from schemas import ResourceNotFoundResponse from schemas.sample import SampleResponse, CreateSample, UpdateSample -from services.query_helper import paginated_all_getter, simple_get_by_id +from services.query_helper import simple_get_by_id from services.crud_helper import model_patcher, model_deleter, model_adder from services.exceptions_helper import PydanticStyleException +from services.sample_helper import get_db_samples router = APIRouter( prefix="/sample", @@ -84,6 +85,8 @@ async def add_sample( Endpoint to add a sample. """ try: + # since this is only one instance N+1 is not a concern for + # FieldActivity, FieldEvent, and Thing return model_adder(session, Sample, sample_data, user=user) except (IntegrityError, ProgrammingError) as e: database_error_handler(sample_data, e) @@ -101,6 +104,8 @@ async def update_sample( Endpoint to update a sample. """ try: + # since this is only one instance N+1 is not a concern for + # FieldActivity, FieldEvent, and Thing return model_patcher(session, Sample, sample_id, sample_data, user=user) except (IntegrityError, ProgrammingError) as e: database_error_handler(sample_data, e) @@ -118,10 +123,7 @@ async def get_samples( """ Endpoint to retrieve samples. """ - - return paginated_all_getter( - session, Sample, sort=sort, order=order, filter_=filter_ - ) + return get_db_samples(session, sort=sort, order=order, filter_=filter_) @router.get("/{sample_id}", summary="Get Sample by ID") @@ -131,6 +133,8 @@ async def get_sample_by_id( """ Endpoint to retrieve a sample by its ID. """ + # since this is only one instance N+1 is not a concern + # FieldActivity, FieldEvent, and Thing return simple_get_by_id(session, Sample, sample_id) diff --git a/services/sample_helper.py b/services/sample_helper.py new file mode 100644 index 000000000..cdefdc859 --- /dev/null +++ b/services/sample_helper.py @@ -0,0 +1,23 @@ +from sqlalchemy.orm import Session, joinedload +from fastapi_pagination.ext.sqlalchemy import paginate + +from db import FieldEvent, FieldActivity, Sample +from services.query_helper import order_sort_filter + + +def get_db_samples( + session: Session, + order: str | None = None, + sort: str | None = None, + filter_: str | None = None, +): + query = session.query(Sample).options( + # Eagerly load related FieldActivity and FieldEvent to avoid N+1 problem + joinedload(Sample.field_activity) + .joinedload(FieldActivity.field_event) + .joinedload(FieldEvent.thing) + ) + + query = order_sort_filter(query, Sample, sort, order, filter_) + + return paginate(query) From 9dab2fb76fabb821781b37ab04b30a6335ea85c0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 11:36:40 -0600 Subject: [PATCH 26/37] refactor: revise observation models for revised schemas --- services/observation_helper.py | 26 +- tests/test_observation.py | 427 ++++++++++++++++----------------- 2 files changed, 232 insertions(+), 221 deletions(-) diff --git a/services/observation_helper.py b/services/observation_helper.py index eddb098fe..00c93e0af 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -8,7 +8,7 @@ from datetime import datetime from core.dependencies import session_dependency -from db import Observation, Sample +from db import Observation, Sample, FieldActivity, FieldEvent, Thing from schemas.observation import ( ObservationResponse, WaterChemistryObservationResponse, @@ -53,14 +53,17 @@ def get_observations( """ Retrieve all observations """ - sample_table_is_joined = False + activity_type_is_retrievable = False activity_type = get_activity_type_from_request(request) sql = select(Observation) if thing_id is not None: - sample_table_is_joined = True + activity_type_is_retrievable = True sql = sql.join(Sample, Sample.id == Observation.sample_id) - sql = sql.where(Sample.thing_id == thing_id) + sql = sql.join(FieldActivity, FieldActivity.id == Sample.field_activity_id) + sql = sql.join(FieldEvent, FieldEvent.id == FieldActivity.field_event_id) + sql = sql.join(Thing, Thing.id == FieldEvent.thing_id) + sql = sql.where(Thing.id == thing_id) if sample_id is not None: sql = sql.where(Observation.sample_id == sample_id) if sensor_id is not None: @@ -73,9 +76,10 @@ def get_observations( # root of path is /observation if activity_type != "observation": - if sample_table_is_joined is False: + if activity_type_is_retrievable is False: sql = sql.join(Sample, Sample.id == Observation.sample_id) - sql = sql.where(Sample.activity_type == activity_type) + sql = sql.join(FieldActivity, FieldActivity.id == Sample.field_activity_id) + sql = sql.where(FieldActivity.activity_type == activity_type) sql = order_sort_filter(sql, Observation, sort, order, filter_) @@ -88,8 +92,16 @@ def get_observations( def verify_observed_property_corresponds_with_activity_type( observation: Observation, request: Request ): + """ + Developer's notes & TODO + + This is only used when getting one observation by its ID, and when patching + a single observation. Since it uses lazy loads that shouldn't be much of an + issue, but if we notice performance problems getting the single record + should use joinedloads so everything is done in a single database query. + """ requested_activity_type = get_activity_type_from_request(request) - actual_activity_type = observation.sample.activity_type + actual_activity_type = observation.sample.field_activity.activity_type if actual_activity_type != requested_activity_type: raise PydanticStyleException( diff --git a/tests/test_observation.py b/tests/test_observation.py index 72ca9c825..335871b9f 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -102,33 +102,33 @@ def test_add_groundwater_level_observation(groundwater_level_sample, sensor): cleanup_post_test(Observation, data["id"]) -def test_add_geothermal_observation(geothermal_sample, sensor): - payload = { - "observation_datetime": "2025-01-01T00:00:00Z", - "release_status": "draft", - "observation_depth": 100, - "value": 25.5, - "sample_id": geothermal_sample.id, - "sensor_id": sensor.id, - "observed_property": "temperature", - "unit": "deg C", - } - response = client.post("/observation/geothermal", json=payload) - data = response.json() - assert response.status_code == 201 - - assert "id" in data - assert "created_at" in data - assert data["observation_datetime"] == payload["observation_datetime"] - assert data["release_status"] == payload["release_status"] - assert data["observation_depth"] == payload["observation_depth"] - assert data["value"] == payload["value"] - assert data["sample_id"] == payload["sample_id"] - assert data["sensor_id"] == payload["sensor_id"] - assert data["observed_property"] == payload["observed_property"] - assert data["unit"] == payload["unit"] - - cleanup_post_test(Observation, data["id"]) +# def test_add_geothermal_observation(geothermal_sample, sensor): +# payload = { +# "observation_datetime": "2025-01-01T00:00:00Z", +# "release_status": "draft", +# "observation_depth": 100, +# "value": 25.5, +# "sample_id": geothermal_sample.id, +# "sensor_id": sensor.id, +# "observed_property": "temperature", +# "unit": "deg C", +# } +# response = client.post("/observation/geothermal", json=payload) +# data = response.json() +# assert response.status_code == 201 + +# assert "id" in data +# assert "created_at" in data +# assert data["observation_datetime"] == payload["observation_datetime"] +# assert data["release_status"] == payload["release_status"] +# assert data["observation_depth"] == payload["observation_depth"] +# assert data["value"] == payload["value"] +# assert data["sample_id"] == payload["sample_id"] +# assert data["sensor_id"] == payload["sensor_id"] +# assert data["observed_property"] == payload["observed_property"] +# assert data["unit"] == payload["unit"] + +# cleanup_post_test(Observation, data["id"]) # PATCH tests ================================================================== @@ -161,25 +161,21 @@ def test_patch_groundwater_level_observation_404_not_found( def test_patch_groundwater_level_observation_404_wrong_activity_type( - water_chemistry_observation, geothermal_observation + water_chemistry_observation, ): - for obs in water_chemistry_observation, geothermal_observation: - payload = {"measuring_point_height": 3} - response = client.patch( - f"/observation/groundwater-level/{obs.id}", json=payload - ) - assert response.status_code == 404 - data = response.json() + payload = {"measuring_point_height": 3} + response = client.patch( + f"/observation/groundwater-level/{water_chemistry_observation.id}", json=payload + ) + assert response.status_code == 404 + data = response.json() - if obs.observed_property == "temperature": - activity_type = "geothermal" - else: - activity_type = "water chemistry" + actual_activity_type = "water chemistry" - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {activity_type} observation." - ) + assert ( + data["detail"][0]["msg"] + == f"Observation with ID {water_chemistry_observation.id} is not a groundwater level observation. It is a {actual_activity_type} observation." + ) def test_patch_water_chemistry_observation(water_chemistry_observation): @@ -207,89 +203,94 @@ def test_patch_water_chemistry_observation_404_not_found(water_chemistry_observa def test_patch_water_chemistry_observation_404_wrong_activity_type( - groundwater_level_observation, geothermal_observation + groundwater_level_observation, ): - for obs in groundwater_level_observation, geothermal_observation: - payload = {"value": 8} - response = client.patch(f"/observation/water-chemistry/{obs.id}", json=payload) - assert response.status_code == 404 - data = response.json() + payload = {"value": 8} + response = client.patch( + f"/observation/water-chemistry/{groundwater_level_observation.id}", json=payload + ) + assert response.status_code == 404 + data = response.json() - if obs.observed_property == "temperature": - activity_type = "geothermal" - else: - activity_type = "groundwater level" + actualy_activity_type = "groundwater level" - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {activity_type} observation." - ) + assert ( + data["detail"][0]["msg"] + == f"Observation with ID {groundwater_level_observation.id} is not a water chemistry observation. It is a {actualy_activity_type} observation." + ) -def test_patch_geothermal_observation(geothermal_observation): - payload = {"observation_depth": 4, "release_status": "private"} - response = client.patch( - f"/observation/geothermal/{geothermal_observation.id}", json=payload - ) - assert response.status_code == 200 - data = response.json() - assert data["observation_depth"] == payload["observation_depth"] - assert data["release_status"] == payload["release_status"] +# def test_patch_geothermal_observation(geothermal_observation): +# payload = {"observation_depth": 4, "release_status": "private"} +# response = client.patch( +# f"/observation/geothermal/{geothermal_observation.id}", json=payload +# ) +# assert response.status_code == 200 +# data = response.json() +# assert data["observation_depth"] == payload["observation_depth"] +# assert data["release_status"] == payload["release_status"] - cleanup_patch_test(Observation, payload, geothermal_observation) +# cleanup_patch_test(Observation, payload, geothermal_observation) -def test_patch_geothermal_observation_404_not_found(geothermal_observation): - bad_id = 999999 - payload = {"observation_depth": 8} - response = client.patch(f"/observation/geothermal/{bad_id}", json=payload) - assert response.status_code == 404 - data = response.json() - assert data["detail"] == f"Observation with ID {bad_id} not found." +# def test_patch_geothermal_observation_404_not_found(geothermal_observation): +# bad_id = 999999 +# payload = {"observation_depth": 8} +# response = client.patch(f"/observation/geothermal/{bad_id}", json=payload) +# assert response.status_code == 404 +# data = response.json() +# assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_patch_geothermal_observation_404_wrong_activity_type( - groundwater_level_observation, water_chemistry_observation -): - for obs in groundwater_level_observation, water_chemistry_observation: - payload = {"value": 8} - response = client.patch(f"/observation/geothermal/{obs.id}", json=payload) - assert response.status_code == 404 - data = response.json() +# def test_patch_geothermal_observation_404_wrong_activity_type( +# groundwater_level_observation, water_chemistry_observation +# ): +# for obs in groundwater_level_observation, water_chemistry_observation: +# payload = {"value": 8} +# response = client.patch(f"/observation/geothermal/{obs.id}", json=payload) +# assert response.status_code == 404 +# data = response.json() - if obs.observed_property == "groundwater level": - activity_type = "groundwater level" - else: - activity_type = "water chemistry" +# if obs.observed_property == "groundwater level": +# activity_type = "groundwater level" +# else: +# activity_type = "water chemistry" - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {activity_type} observation." - ) +# assert ( +# data["detail"][0]["msg"] +# == f"Observation with ID {obs.id} is not a geothermal observation. It is a {activity_type} observation." +# ) # ============= Get tests ================= def test_get_all_observations( - groundwater_level_observation, water_chemistry_observation, geothermal_observation + groundwater_level_observation, water_chemistry_observation ): response = client.get("/observation") assert response.status_code == 200 data = response.json() - assert data["total"] == 3 - assert data["items"][0]["id"] == groundwater_level_observation.id - assert data["items"][1]["id"] == water_chemistry_observation.id - assert data["items"][2]["id"] == geothermal_observation.id + + assert data["total"] == 2 + for item in data["items"]: + assert "id" in item + assert "created_at" in item + assert "release_status" in item + assert "sample_id" in item + assert "sensor_id" in item + assert "observation_datetime" in item + assert "observed_property" in item + assert "value" in item + assert "unit" in item def test_get_observation_by_id( - groundwater_level_observation, water_chemistry_observation, geothermal_observation + groundwater_level_observation, water_chemistry_observation ): for obs in ( groundwater_level_observation, water_chemistry_observation, - geothermal_observation, ): response = client.get(f"/observation/{obs.id}") assert response.status_code == 200 @@ -301,16 +302,13 @@ def test_get_observation_by_id( if obs.observed_property == "groundwater level": assert data["depth_to_water_bgs"] == obs.value - obs.measuring_point_height assert data["observation_depth"] is None - elif obs.observed_property == "temperature": - assert data["depth_to_water_bgs"] is None - assert data["observation_depth"] == obs.observation_depth else: assert data["depth_to_water_bgs"] is None assert data["observation_depth"] is None def test_get_observation_by_id_404_not_found( - groundwater_level_observation, water_chemistry_observation, geothermal_observation + groundwater_level_observation, water_chemistry_observation ): bad_id = 999999 response = client.get(f"/observation/{bad_id}") @@ -319,9 +317,7 @@ def test_get_observation_by_id_404_not_found( assert data["detail"] == f"Observation with ID {bad_id} not found." -def test_get_groundwater_level_observations( - groundwater_level_observation, water_chemistry_observation, geothermal_observation -): +def test_get_groundwater_level_observations(groundwater_level_observation): response = client.get("/observation/groundwater-level") assert response.status_code == 200 data = response.json() @@ -413,25 +409,25 @@ def test_get_groundwater_level_observation_by_id_404_not_found( def test_get_groundwater_level_observation_by_id_404_wrong_activity_type( - water_chemistry_observation, geothermal_observation + water_chemistry_observation, ): - for obs in water_chemistry_observation, geothermal_observation: - response = client.get(f"/observation/groundwater-level/{obs.id}") - assert response.status_code == 404 - data = response.json() + response = client.get( + f"/observation/groundwater-level/{water_chemistry_observation.id}" + ) + assert response.status_code == 404 + data = response.json() - if obs.observed_property == "temperature": - actual_activity_type = "geothermal" - else: - actual_activity_type = "water chemistry" + actual_activity_type = "water chemistry" - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a groundwater level observation. It is a {actual_activity_type} observation." - ) - assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"observation_id": obs.id} - assert data["detail"][0]["loc"] == ["path", "observation_id"] + assert ( + data["detail"][0]["msg"] + == f"Observation with ID {water_chemistry_observation.id} is not a groundwater level observation. It is a {actual_activity_type} observation." + ) + assert data["detail"][0]["type"] == "value_error" + assert data["detail"][0]["input"] == { + "observation_id": water_chemistry_observation.id + } + assert data["detail"][0]["loc"] == ["path", "observation_id"] def test_get_groundwater_observation_by_sample(groundwater_level_sample): @@ -570,106 +566,109 @@ def test_get_water_chemistry_observation_by_id_404_not_found( def test_get_water_chemistry_observation_by_id_404_wrong_activity_type( - groundwater_level_observation, geothermal_observation + groundwater_level_observation, ): - for obs in groundwater_level_observation, geothermal_observation: - response = client.get(f"/observation/water-chemistry/{obs.id}") - assert response.status_code == 404 - data = response.json() - - if obs.observed_property == "groundwater level": - actual_activity_type = "groundwater level" - else: - actual_activity_type = "geothermal" - - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a water chemistry observation. It is a {actual_activity_type} observation." - ) - assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"observation_id": obs.id} - assert data["detail"][0]["loc"] == ["path", "observation_id"] - - -def test_get_geothermal_observations(geothermal_observation): - response = client.get("/observation/geothermal") - assert response.status_code == 200 - data = response.json() - assert data["total"] == 1 - assert data["items"][0]["id"] == geothermal_observation.id - assert data["items"][0][ - "created_at" - ] == geothermal_observation.created_at.isoformat().replace("+00:00", "Z") - assert data["items"][0]["release_status"] == geothermal_observation.release_status - assert data["items"][0]["sample_id"] == geothermal_observation.sample_id - assert data["items"][0]["sensor_id"] == geothermal_observation.sensor_id - assert ( - data["items"][0]["observation_datetime"] - == geothermal_observation.observation_datetime - ) - colon_index = geothermal_observation.observed_property.find(":") - assert ( - data["items"][0]["observed_property"] - == geothermal_observation.observed_property[colon_index + 1 :] - ) - assert data["items"][0]["value"] == geothermal_observation.value - assert data["items"][0]["unit"] == geothermal_observation.unit - assert ( - data["items"][0]["observation_depth"] - == geothermal_observation.observation_depth - ) - - -def test_get_geothermal_observation_by_id(geothermal_observation): - response = client.get(f"/observation/geothermal/{geothermal_observation.id}") - assert response.status_code == 200 - data = response.json() - assert data["id"] == geothermal_observation.id - assert data["created_at"] == geothermal_observation.created_at.isoformat().replace( - "+00:00", "Z" - ) - assert data["release_status"] == geothermal_observation.release_status - assert data["sample_id"] == geothermal_observation.sample_id - assert data["sensor_id"] == geothermal_observation.sensor_id - assert data["observation_datetime"] == geothermal_observation.observation_datetime - colon_index = geothermal_observation.observed_property.find(":") - assert ( - data["observed_property"] - == geothermal_observation.observed_property[colon_index + 1 :] + response = client.get( + f"/observation/water-chemistry/{groundwater_level_observation.id}" ) - assert data["value"] == geothermal_observation.value - assert data["unit"] == geothermal_observation.unit - assert data["observation_depth"] == geothermal_observation.observation_depth - - -def test_get_geothermal_observation_by_id_404_not_found(geothermal_observation): - bad_id = 99999 - response = client.get(f"/observation/geothermal/{bad_id}") assert response.status_code == 404 data = response.json() - assert data["detail"] == f"Observation with ID {bad_id} not found." + if groundwater_level_observation.observed_property == "groundwater level": + actual_activity_type = "groundwater level" + else: + actual_activity_type = "geothermal" -def test_get_geothermal_observation_by_id_404_wrong_activity_type( - water_chemistry_observation, groundwater_level_observation -): - for obs in water_chemistry_observation, groundwater_level_observation: - response = client.get(f"/observation/geothermal/{obs.id}") - assert response.status_code == 404 - data = response.json() - - if obs.observed_property == "groundwater level": - actual_activity_type = "groundwater level" - else: - actual_activity_type = "water chemistry" - - assert ( - data["detail"][0]["msg"] - == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_activity_type} observation." - ) - assert data["detail"][0]["type"] == "value_error" - assert data["detail"][0]["input"] == {"observation_id": obs.id} - assert data["detail"][0]["loc"] == ["path", "observation_id"] + assert ( + data["detail"][0]["msg"] + == f"Observation with ID {groundwater_level_observation.id} is not a water chemistry observation. It is a {actual_activity_type} observation." + ) + assert data["detail"][0]["type"] == "value_error" + assert data["detail"][0]["input"] == { + "observation_id": groundwater_level_observation.id + } + assert data["detail"][0]["loc"] == ["path", "observation_id"] + + +# def test_get_geothermal_observations(geothermal_observation): +# response = client.get("/observation/geothermal") +# assert response.status_code == 200 +# data = response.json() +# assert data["total"] == 1 +# assert data["items"][0]["id"] == geothermal_observation.id +# assert data["items"][0][ +# "created_at" +# ] == geothermal_observation.created_at.isoformat().replace("+00:00", "Z") +# assert data["items"][0]["release_status"] == geothermal_observation.release_status +# assert data["items"][0]["sample_id"] == geothermal_observation.sample_id +# assert data["items"][0]["sensor_id"] == geothermal_observation.sensor_id +# assert ( +# data["items"][0]["observation_datetime"] +# == geothermal_observation.observation_datetime +# ) +# colon_index = geothermal_observation.observed_property.find(":") +# assert ( +# data["items"][0]["observed_property"] +# == geothermal_observation.observed_property[colon_index + 1 :] +# ) +# assert data["items"][0]["value"] == geothermal_observation.value +# assert data["items"][0]["unit"] == geothermal_observation.unit +# assert ( +# data["items"][0]["observation_depth"] +# == geothermal_observation.observation_depth +# ) + + +# def test_get_geothermal_observation_by_id(geothermal_observation): +# response = client.get(f"/observation/geothermal/{geothermal_observation.id}") +# assert response.status_code == 200 +# data = response.json() +# assert data["id"] == geothermal_observation.id +# assert data["created_at"] == geothermal_observation.created_at.isoformat().replace( +# "+00:00", "Z" +# ) +# assert data["release_status"] == geothermal_observation.release_status +# assert data["sample_id"] == geothermal_observation.sample_id +# assert data["sensor_id"] == geothermal_observation.sensor_id +# assert data["observation_datetime"] == geothermal_observation.observation_datetime +# colon_index = geothermal_observation.observed_property.find(":") +# assert ( +# data["observed_property"] +# == geothermal_observation.observed_property[colon_index + 1 :] +# ) +# assert data["value"] == geothermal_observation.value +# assert data["unit"] == geothermal_observation.unit +# assert data["observation_depth"] == geothermal_observation.observation_depth + + +# def test_get_geothermal_observation_by_id_404_not_found(geothermal_observation): +# bad_id = 99999 +# response = client.get(f"/observation/geothermal/{bad_id}") +# assert response.status_code == 404 +# data = response.json() +# assert data["detail"] == f"Observation with ID {bad_id} not found." + + +# def test_get_geothermal_observation_by_id_404_wrong_activity_type( +# water_chemistry_observation, groundwater_level_observation +# ): +# for obs in water_chemistry_observation, groundwater_level_observation: +# response = client.get(f"/observation/geothermal/{obs.id}") +# assert response.status_code == 404 +# data = response.json() + +# if obs.observed_property == "groundwater level": +# actual_activity_type = "groundwater level" +# else: +# actual_activity_type = "water chemistry" + +# assert ( +# data["detail"][0]["msg"] +# == f"Observation with ID {obs.id} is not a geothermal observation. It is a {actual_activity_type} observation." +# ) +# assert data["detail"][0]["type"] == "value_error" +# assert data["detail"][0]["input"] == {"observation_id": obs.id} +# assert data["detail"][0]["loc"] == ["path", "observation_id"] # JB's comment: I don't think that geographic filters are necessary for From c0d0e4aaddceb10b9e1d1f9aa6bb94f0cbabcabe Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 11:39:06 -0600 Subject: [PATCH 27/37] test: skip geochronology tests for now --- tests/test_geochronology.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_geochronology.py b/tests/test_geochronology.py index 241d96629..c6b32ce1f 100644 --- a/tests/test_geochronology.py +++ b/tests/test_geochronology.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== - +import pytest from tests import client +@pytest.mark.skip( + reason="Not implemented and may be fully deprecated for the observation table/router" +) def test_add_geochronology_age(): response = client.post( "/geochronology/age", From 8a5d0c3201e96c9ed97d2c799977e5cc5537f375 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 14:35:58 -0600 Subject: [PATCH 28/37] refactor: deprecate geothermal for now --- api/observation.py | 136 ++++++++++++++++----------------- schemas/observation.py | 14 ++-- services/observation_helper.py | 4 +- tests/test_observation.py | 2 - 4 files changed, 76 insertions(+), 80 deletions(-) diff --git a/api/observation.py b/api/observation.py index 9e5a442a7..74719912a 100644 --- a/api/observation.py +++ b/api/observation.py @@ -21,9 +21,7 @@ from core.dependencies import ( session_dependency, amp_admin_dependency, - admin_dependency, amp_viewer_dependency, - viewer_dependency, ) from db import Observation from schemas.observation import ( @@ -31,12 +29,12 @@ GroundwaterLevelObservationResponse, CreateWaterChemistryObservation, WaterChemistryObservationResponse, - CreateGeothermalObservation, - GeothermalObservationResponse, + # CreateGeothermalObservation, + # GeothermalObservationResponse, ObservationResponse, UpdateGroundwaterLevelObservation, UpdateWaterChemistryObservation, - UpdateGeothermalObservation, + # UpdateGeothermalObservation, ) from services.crud_helper import model_deleter, model_adder from services.query_helper import simple_get_by_id @@ -82,17 +80,17 @@ async def add_water_chemistry_observation( return model_adder(session, Observation, obs_data, user=user) -@router.post("/geothermal", status_code=HTTP_201_CREATED) -async def add_geothermal_observation( - obs_data: CreateGeothermalObservation, - session: session_dependency, - user: admin_dependency, -) -> GeothermalObservationResponse: - """ - Add a new geothermal observation to the database. - This endpoint is currently a placeholder and does not implement any functionality. - """ - return model_adder(session, Observation, obs_data, user=user) +# @router.post("/geothermal", status_code=HTTP_201_CREATED) +# async def add_geothermal_observation( +# obs_data: CreateGeothermalObservation, +# session: session_dependency, +# user: admin_dependency, +# ) -> GeothermalObservationResponse: +# """ +# Add a new geothermal observation to the database. +# This endpoint is currently a placeholder and does not implement any functionality. +# """ +# return model_adder(session, Observation, obs_data, user=user) # PATCH ======================================================================== @@ -126,18 +124,18 @@ async def update_water_chemistry_observation( return observation_model_patcher(session, request, observation_id, obs_data, user) -@router.patch("/geothermal/{observation_id}", status_code=HTTP_200_OK) -async def update_geothermal_observation( - observation_id: int, - obs_data: UpdateGeothermalObservation, - session: session_dependency, - user: admin_dependency, - request: Request, -) -> GeothermalObservationResponse: - """ - Update an existing geothermal observation in the database. - """ - return observation_model_patcher(session, request, observation_id, obs_data, user) +# @router.patch("/geothermal/{observation_id}", status_code=HTTP_200_OK) +# async def update_geothermal_observation( +# observation_id: int, +# obs_data: UpdateGeothermalObservation, +# session: session_dependency, +# user: admin_dependency, +# request: Request, +# ) -> GeothermalObservationResponse: +# """ +# Update an existing geothermal observation in the database. +# """ +# return observation_model_patcher(session, request, observation_id, obs_data, user) # ============= Get ============================================== @@ -238,47 +236,47 @@ async def get_water_chemistry_observation_by_id( ) -@router.get("/geothermal", summary="Get geothermal observations") -async def get_geothermal_observations( - request: Request, - session: session_dependency, - user: viewer_dependency, - thing_id: int | None = None, - sensor_id: int | None = None, - sample_id: int | None = None, - start_time: datetime | None = None, - end_time: datetime | None = None, - sort: str | None = None, - order: str | None = None, - filter_: str = Query(alias="filter", default=None), -) -> CustomPage[GeothermalObservationResponse]: - """ - Retrieve all geothermal observations from the database. - """ - return get_observations( - request=request, - session=session, - thing_id=thing_id, - sensor_id=sensor_id, - sample_id=sample_id, - start_time=start_time, - end_time=end_time, - sort=sort, - order=order, - filter_=filter_, - ) - - -@router.get("/geothermal/{observation_id}", summary="Get geothermal observation by ID") -async def get_geothermal_observation_by_id( - session: session_dependency, - request: Request, - user: amp_viewer_dependency, - observation_id: int, -) -> GeothermalObservationResponse: - return get_observation_of_an_activity_type_by_id( - session=session, request=request, observation_id=observation_id - ) +# @router.get("/geothermal", summary="Get geothermal observations") +# async def get_geothermal_observations( +# request: Request, +# session: session_dependency, +# user: viewer_dependency, +# thing_id: int | None = None, +# sensor_id: int | None = None, +# sample_id: int | None = None, +# start_time: datetime | None = None, +# end_time: datetime | None = None, +# sort: str | None = None, +# order: str | None = None, +# filter_: str = Query(alias="filter", default=None), +# ) -> CustomPage[GeothermalObservationResponse]: +# """ +# Retrieve all geothermal observations from the database. +# """ +# return get_observations( +# request=request, +# session=session, +# thing_id=thing_id, +# sensor_id=sensor_id, +# sample_id=sample_id, +# start_time=start_time, +# end_time=end_time, +# sort=sort, +# order=order, +# filter_=filter_, +# ) + + +# @router.get("/geothermal/{observation_id}", summary="Get geothermal observation by ID") +# async def get_geothermal_observation_by_id( +# session: session_dependency, +# request: Request, +# user: amp_viewer_dependency, +# observation_id: int, +# ) -> GeothermalObservationResponse: +# return get_observation_of_an_activity_type_by_id( +# session=session, request=request, observation_id=observation_id +# ) @router.get("", summary="Get all observations") diff --git a/schemas/observation.py b/schemas/observation.py index f308a4535..1e052fec3 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -75,8 +75,8 @@ class CreateWaterChemistryObservation(CreateBaseObservation): pass -class CreateGeothermalObservation(CreateBaseObservation): - observation_depth: float +# class CreateGeothermalObservation(CreateBaseObservation): +# observation_depth: float # -------- UPDATE ------------ @@ -101,8 +101,8 @@ class UpdateWaterChemistryObservation(UpdateBaseObservation): pass -class UpdateGeothermalObservation(UpdateBaseObservation): - observation_depth: float | None = None +# class UpdateGeothermalObservation(UpdateBaseObservation): +# observation_depth: float | None = None # -------- RESPONSE ---------- @@ -136,12 +136,12 @@ class WaterChemistryObservationResponse(BaseObservationResponse): pass -class GeothermalObservationResponse(BaseObservationResponse): - observation_depth: float | None +# class GeothermalObservationResponse(BaseObservationResponse): +# observation_depth: float | None class ObservationResponse( - GroundwaterLevelObservationResponse, GeothermalObservationResponse + GroundwaterLevelObservationResponse, WaterChemistryObservationResponse ): """ Response model for observations. diff --git a/services/observation_helper.py b/services/observation_helper.py index 00c93e0af..7d935c346 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -12,7 +12,7 @@ from schemas.observation import ( ObservationResponse, WaterChemistryObservationResponse, - GeothermalObservationResponse, + # GeothermalObservationResponse, GroundwaterLevelObservationResponse, ) from services.exceptions_helper import PydanticStyleException @@ -47,7 +47,7 @@ def get_observations( ) -> ( List[ObservationResponse] | List[WaterChemistryObservationResponse] - | List[GeothermalObservationResponse] + # | List[GeothermalObservationResponse] | List[GroundwaterLevelObservationResponse] ): """ diff --git a/tests/test_observation.py b/tests/test_observation.py index 335871b9f..f7c206699 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -301,10 +301,8 @@ def test_get_observation_by_id( assert data["release_status"] == obs.release_status if obs.observed_property == "groundwater level": assert data["depth_to_water_bgs"] == obs.value - obs.measuring_point_height - assert data["observation_depth"] is None else: assert data["depth_to_water_bgs"] is None - assert data["observation_depth"] is None def test_get_observation_by_id_404_not_found( From d967c5b92a8c3cc1c5083cbc81500f8c848b7dac Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 16:52:03 -0600 Subject: [PATCH 29/37] refactor: only enable event contacts to relate to a sample --- core/lexicon.json | 7 +++++++ db/field.py | 21 ++++++++++++++++----- db/sample.py | 18 +++++++++++------- schemas/field.py | 1 - schemas/sample.py | 8 +++++--- services/sample_helper.py | 7 +++++-- tests/conftest.py | 28 +++++++++++++++++++++------- tests/test_sample.py | 35 +++++++++++++++++++++++++---------- 8 files changed, 90 insertions(+), 35 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index aae2cee38..1c4980449 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -251,6 +251,13 @@ {"categories": [{"name": "activity_type", "description": null}], "term": "groundwater level", "definition": "groundwater level"}, {"categories": [{"name": "activity_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, + + {"categories": [{"name": "field_event_role", "description": null}], "term": "Lead", "definition": "the leader of the field event"}, + {"categories": [{"name": "field_event_role", "description": null}], "term": "Participant", "definition": "a person participating in the field event"}, + {"categories": [{"name": "field_event_role", "description": null}], "term": "Observer", "definition": "a person observing the field event"}, + {"categories": [{"name": "field_event_role", "description": null}], "term": "Visitor", "definition": "a person visiting the field event"}, + + {"categories": [{"name": "sample_matrix", "description": null}], "term": "water", "definition": "water"}, {"categories": [{"name": "sample_matrix", "description": null}], "term": "soil", "definition": "soil"}, diff --git a/db/field.py b/db/field.py index d77a08f5e..988ef3051 100644 --- a/db/field.py +++ b/db/field.py @@ -24,9 +24,23 @@ class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): comment="Foreign key to the Contact table", ) + # TODO: get AMP feedback on the roles + field_event_role: Mapped[str] = lexicon_term( + nullable=False, comment="Role of the contact in the field event" + ) + # --- Relationships --- - field_event: Mapped[list["FieldEvent"]] = relationship("FieldEvent") - contact: Mapped[list["Contact"]] = relationship("Contact") # noqa: F821 + field_event: Mapped["FieldEvent"] = relationship("FieldEvent") + contact: Mapped["Contact"] = relationship("Contact") # noqa: F821 + + # map associated contacts to samples to restrict the people who could have + # taken a sample to those present at the field event + samples: Mapped[list["Sample"]] = relationship( # noqa: F821 + "Sample", + back_populates="field_event_contact", + cascade="all, delete-orphan", + passive_deletes=True, + ) class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): @@ -54,9 +68,6 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Date and time of the field event.", ) - event_lead_name: Mapped[str] = mapped_column( - nullable=False, comment="The name of the person leading the field event" - ) collecting_organization: Mapped[str] = lexicon_term( nullable=False, default="NMBGMR", # TODO: put default in schema diff --git a/db/sample.py b/db/sample.py index 3b5d506e3..782e54a8b 100644 --- a/db/sample.py +++ b/db/sample.py @@ -46,8 +46,6 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): ForeignKey("field_activity.id"), nullable=False ) - # --- Column Definitions --- - # nullable because sample can be collected by steel tape sensor_id: Mapped[Optional[int]] = mapped_column( ForeignKey("sensor.id"), @@ -55,7 +53,11 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): nullable=True, ) - # Sample Attributes + field_event_contact_id: Mapped[Optional[str]] = mapped_column( + ForeignKey("field_event_contact_association.id") + ) + + # --- Columns --- sample_date: Mapped[datetime.datetime] = mapped_column( DateTime(timezone=True), nullable=False, @@ -72,10 +74,6 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): sample_method: Mapped[str] = lexicon_term( comment="Method used to collect the sample.", nullable=False ) - sampler_name: Mapped[str] = mapped_column( - nullable=False, - comment="Name of the person who collected the sample. This may or may not be the person who lead the event (see FieldEvent table)", - ) qc_type: Mapped[str] = mapped_column( default="Normal", nullable=False, @@ -94,10 +92,16 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): back_populates="samples" ) sensor: Mapped["Sensor"] = relationship(back_populates="samples") # noqa: F821 + # fmt: off + field_event_contact: Mapped["FieldEventContactAssociation"] = relationship( # noqa: F821 + back_populates="samples" + ) + # fmt: on # association proxies to help keep code DRY field_event = association_proxy("field_activity", "field_event") thing = association_proxy("field_activity", "field_event.thing") + contact = association_proxy("field_event_contact", "contact") # noqa: F821 # ============= EOF ============================================= diff --git a/schemas/field.py b/schemas/field.py index 279b5a5eb..43a042918 100644 --- a/schemas/field.py +++ b/schemas/field.py @@ -14,6 +14,5 @@ class FieldActivityResponse(BaseResponseModel): class FieldEventResponse(BaseResponseModel): thing_id: int event_date: AwareDatetime - event_lead_name: str collecting_organization: str | None notes: str | None diff --git a/schemas/sample.py b/schemas/sample.py index 4d48fc400..bd1f7468d 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -27,6 +27,7 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.thing import ThingResponse from schemas.field import FieldEventResponse, FieldActivityResponse +from schemas.contact import ContactResponse # -------- VALIDATE ---------- @@ -84,11 +85,11 @@ def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime: class CreateSample(BaseCreateModel, ValidateSample): field_activity_id: int sensor_id: int | None = None + field_event_contact_id: int sample_date: Annotated[AwareDatetime, PastDatetime()] sample_name: str sample_matrix: str sample_method: str - sampler_name: str qc_type: str notes: str | None = None depth_top: float | None = None @@ -99,11 +100,11 @@ class CreateSample(BaseCreateModel, ValidateSample): class UpdateSample(BaseUpdateModel, ValidateSample): field_activity_id: int | None = None # TODO: should this be editable? sensor_id: int | None = None + field_event_contact_id: int | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None sample_name: str | None = None sample_matrix: str | None = None sample_method: str | None = None - sampler_name: str | None = None qc_type: str | None = None notes: str | None = None depth_top: float | None = None @@ -124,13 +125,14 @@ class SampleResponse(BaseResponseModel): thing: ThingResponse field_event: FieldEventResponse field_activity: FieldActivityResponse + contact: ContactResponse field_activity_id: int + field_event_contact_id: int sensor_id: int | None sample_date: Annotated[AwareDatetime, PastDatetime()] sample_name: str sample_matrix: str sample_method: str - sampler_name: str qc_type: str notes: str | None depth_top: float | None diff --git a/services/sample_helper.py b/services/sample_helper.py index cdefdc859..dc02506e0 100644 --- a/services/sample_helper.py +++ b/services/sample_helper.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session, joinedload from fastapi_pagination.ext.sqlalchemy import paginate -from db import FieldEvent, FieldActivity, Sample +from db import FieldEvent, FieldActivity, FieldEventContactAssociation, Sample from services.query_helper import order_sort_filter @@ -15,7 +15,10 @@ def get_db_samples( # Eagerly load related FieldActivity and FieldEvent to avoid N+1 problem joinedload(Sample.field_activity) .joinedload(FieldActivity.field_event) - .joinedload(FieldEvent.thing) + .joinedload(FieldEvent.thing), + joinedload(Sample.field_event_contact).joinedload( + FieldEventContactAssociation.contact + ), # Eagerly load related Contact ) query = order_sort_filter(query, Sample, sort, order, filter_) diff --git a/tests/conftest.py b/tests/conftest.py index 04917f088..2eb928e23 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -464,7 +464,6 @@ def field_event(water_well_thing): field_event = FieldEvent( thing_id=water_well_thing.id, event_date="2025-01-01T00:00:00Z", - event_lead_name="Sesame Mango", collecting_organization="NMBGMR", notes="field event fixture notes", release_status="draft", @@ -474,6 +473,19 @@ def field_event(water_well_thing): yield field_event +@pytest.fixture(scope="session") +def field_event_contact(field_event, contact): + with session_ctx() as session: + field_event_contact = FieldEventContactAssociation( + field_event_id=field_event.id, + contact_id=contact.id, + field_event_role="Lead", + ) + session.add(field_event_contact) + session.commit() + yield field_event_contact + + @pytest.fixture(scope="session") def groundwater_level_field_activity(field_event): with session_ctx() as session: @@ -503,16 +515,18 @@ def water_chemistry_field_activity(field_event): @pytest.fixture(scope="session") -def groundwater_level_sample(groundwater_level_field_activity, sensor): +def groundwater_level_sample( + groundwater_level_field_activity, sensor, field_event_contact +): with session_ctx() as session: sample = Sample( field_activity_id=groundwater_level_field_activity.id, + field_event_contact_id=field_event_contact.id, sensor_id=sensor.id, sample_date="2025-01-01T12:00:00Z", sample_name="groundwater level sample name", sample_matrix="water", sample_method="Steel-tape measurement", - sampler_name="Esme Patterson", qc_type="Normal", depth_top=None, depth_bottom=None, @@ -525,16 +539,16 @@ def groundwater_level_sample(groundwater_level_field_activity, sensor): @pytest.fixture(scope="session") -def water_chemistry_sample(water_chemistry_field_activity, sensor): +def water_chemistry_sample(water_chemistry_field_activity, sensor, field_event_contact): with session_ctx() as session: sample = Sample( field_activity_id=water_chemistry_field_activity.id, sensor_id=sensor.id, + field_event_contact_id=field_event_contact.id, sample_date="2025-01-01T13:00:00Z", sample_name="water chemistry sample name", sample_matrix="water", sample_method="grab sample", - sampler_name="Esme Patterson", qc_type="Normal", depth_top=None, depth_bottom=None, @@ -547,16 +561,16 @@ def water_chemistry_sample(water_chemistry_field_activity, sensor): @pytest.fixture(scope="function") -def sample_to_delete(water_chemistry_field_activity, sensor): +def sample_to_delete(water_chemistry_field_activity, sensor, field_event_contact): with session_ctx() as session: sample = Sample( field_activity_id=water_chemistry_field_activity.id, sensor_id=sensor.id, + field_event_contact_id=field_event_contact.id, sample_date="2025-01-01T13:00:00Z", sample_name="sample to delete", sample_matrix="water", sample_method="grab sample", - sampler_name="Esme Patterson", qc_type="Normal", depth_top=None, depth_bottom=None, diff --git a/tests/test_sample.py b/tests/test_sample.py index 3024e4c05..a74ea44a0 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -53,18 +53,20 @@ def test_validate_sample_top_and_bottom(): # ============= Post tests for samples ============================================= -def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): +def test_add_sample( + groundwater_level_field_activity, water_well_thing, sensor, field_event_contact +): """ Test adding a sample. """ payload = { "field_activity_id": groundwater_level_field_activity.id, "sensor_id": sensor.id, + "field_event_contact_id": field_event_contact.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": "second groundwater level field activity name", "sample_matrix": "water", "sample_method": "grab sample", - "sampler_name": "Ptolemy I Soter", "notes": "posted notes", "qc_type": "Normal", "depth_top": None, @@ -84,35 +86,40 @@ def test_add_sample(groundwater_level_field_activity, water_well_thing, sensor): assert data["field_event"]["id"] == groundwater_level_field_activity.field_event_id assert data["field_activity"]["id"] == groundwater_level_field_activity.id assert data["field_activity_id"] == payload["field_activity_id"] + assert data["contact"]["id"] == field_event_contact.contact_id + assert data["field_event_contact_id"] == payload["field_event_contact_id"] assert data["sensor_id"] == payload["sensor_id"] assert data["sample_date"] == payload["sample_date"] assert data["sample_name"] == payload["sample_name"] assert data["sample_matrix"] == payload["sample_matrix"] assert data["sample_method"] == payload["sample_method"] - assert data["sampler_name"] == payload["sampler_name"] assert data["notes"] == payload["notes"] assert data["qc_type"] == payload["qc_type"] assert data["depth_top"] == payload["depth_top"] assert data["depth_bottom"] == payload["depth_bottom"] + assert data["release_status"] == payload["release_status"] # cleanup after adding the sample cleanup_post_test(Sample, data["id"]) def test_409_add_sample_invalid_sample_name( - groundwater_level_field_activity, groundwater_level_sample, sensor + groundwater_level_field_activity, + groundwater_level_sample, + sensor, + field_event_contact, ): """ Test that a 409 error is raised if a duplicate sample_name is in the payload """ payload = { "field_activity_id": groundwater_level_field_activity.id, + "field_event_contact_id": field_event_contact.id, "sensor_id": sensor.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": groundwater_level_sample.sample_name, "sample_matrix": "water", "sample_method": "grab sample", - "sampler_name": "Ptolemy I Soter", "qc_type": "Normal", "depth_top": None, "depth_bottom": None, @@ -136,7 +143,10 @@ def test_409_add_sample_invalid_sample_name( def test_409_add_sample_invalid_field_activity_id( - groundwater_level_field_activity, groundwater_level_sample, sensor + groundwater_level_field_activity, + groundwater_level_sample, + sensor, + field_event_contact, ): """ Test adding a sample with an invalid field_activity_id. @@ -144,11 +154,11 @@ def test_409_add_sample_invalid_field_activity_id( payload = { "field_activity_id": 999999, "sensor_id": sensor.id, + "field_event_contact_id": field_event_contact.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": "yet another sample name", "sample_matrix": "water", "sample_method": "grab sample", - "sampler_name": "Ptolemy I Soter", "qc_type": "Normal", "depth_top": None, "depth_bottom": None, @@ -159,6 +169,7 @@ def test_409_add_sample_invalid_field_activity_id( json=payload, ) data = response.json() + print(data) assert response.status_code == 409 assert data["detail"][0]["loc"] == ["body", "field_activity_id"] assert ( @@ -181,12 +192,12 @@ def test_patch_sample( payload = { "field_activity_id": groundwater_level_field_activity.id, "sensor_id": second_sensor.id, + # "field_event_contact_id": third_contact.id, "sample_date": "2025-01-02T00:00:00Z", "sample_name": "patched sample name", "sample_matrix": "soil", "sample_method": "bailer", "release_status": "private", - "sampler_name": "test sample b", "qc_type": "Split", "notes": "patched notes", "depth_top": 10.0, @@ -287,16 +298,18 @@ def test_get_samples(water_chemistry_sample, groundwater_level_sample): assert "field_event" in item assert "field_activity" in item assert "field_activity_id" in item + assert "contact" in item + assert "field_event_contact_id" in item assert "sensor_id" in item assert "sample_date" in item assert "sample_name" in item assert "sample_matrix" in item assert "sample_method" in item - assert "sampler_name" in item assert "qc_type" in item assert "depth_top" in item assert "depth_bottom" in item assert "notes" in item + assert "release_status" in item def test_get_sample_by_id( @@ -305,6 +318,7 @@ def test_get_sample_by_id( field_event, water_well_thing, sensor, + field_event_contact, ): """ Test retrieving a sample by its ID. @@ -320,16 +334,17 @@ def test_get_sample_by_id( assert data["field_event"]["id"] == field_event.id assert data["field_activity"]["id"] == water_chemistry_field_activity.id assert data["field_activity_id"] == water_chemistry_field_activity.id + assert data["field_event_contact_id"] == field_event_contact.id assert data["sensor_id"] == sensor.id assert data["sample_date"] == water_chemistry_sample.sample_date assert data["sample_name"] == water_chemistry_sample.sample_name assert data["sample_matrix"] == water_chemistry_sample.sample_matrix assert data["sample_method"] == water_chemistry_sample.sample_method - assert data["sampler_name"] == water_chemistry_sample.sampler_name assert data["qc_type"] == water_chemistry_sample.qc_type assert data["notes"] == water_chemistry_sample.notes assert data["depth_top"] == water_chemistry_sample.depth_top assert data["depth_bottom"] == water_chemistry_sample.depth_bottom + assert data["release_status"] == water_chemistry_sample.release_status def test_get_sample_by_id_404_not_found(water_chemistry_sample): From 23c45679b7f49c37a2be29f27cc530769df78193 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 19 Sep 2025 16:53:29 -0600 Subject: [PATCH 30/37] WIP: water level transfers --- transfers/waterlevels_transfer.py | 49 ++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 85391c1c3..2edf71df6 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -19,7 +19,7 @@ import pandas as pd -from db import Thing, Sample, Observation +from db import Thing, Sample, Observation, FieldEvent, FieldActivity from transfers.util import ( filter_to_valid_point_ids, logger, @@ -65,14 +65,40 @@ def transfer_water_levels(session): ) continue - sample = Sample() + release_status = "public" if row.PublicRelease else "private" + + """ + Developer's notes + + Assumes for manual water levels that the date/time of the water level + measurement is the same as the date/time of the field event. + """ + + if pd.isna(row.MeasuringAgency): + collecting_organization = "Unknown" + else: + collecting_organization = row.MeasuringAgency if pd.isna(row.MeasuredBy): sampler_name = "Unknown" else: sampler_name = row.MeasuredBy - sample.activity_type = "groundwater level" + field_event = FieldEvent( + thing=thing, + event_date=dt_utc, + collecting_organization=collecting_organization, + release_status=release_status, + ) + + session.add(field_event) + + field_activity = FieldActivity( + field_event=field_event, + activity_type="groundwater level", + release_status=release_status, + ) + session.add(field_activity) if not pd.isna(row.MeasurementMethod): sample_method = lu_to_lexicon_map[ @@ -82,21 +108,22 @@ def transfer_water_levels(session): sample_method = "null placeholder" sample = Sample( + field_activity=field_activity, sampler_name=sampler_name, sample_date=dt_utc, - sample_matrix="groundwater", - field_sample_id=str(uuid.uuid4()), - thing=thing, + sample_matrix="water", + sample_name=str( + uuid.uuid4() + ), # TODO: should this stay as-is for water levels? since there are no lab-assigned names sample_method=sample_method, - qc_sample="Original", - sample_top=None, - sample_bottom=None, - duplicate_sample_number=0, - activity_type="groundwater level", + qc_type="Normal", + depth_top=None, + depth_bottom=None, ) session.add(sample) # TODO: update for auto-collectors in the Sensor table, like e-probes + # update the deployment table here sensor_id = None if not pd.isna(row.LevelStatus): From 7ebd8b80d0e8ca855e511eb100a178a8bb0646e7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 22 Sep 2025 16:23:03 -0600 Subject: [PATCH 31/37] refactor: PR 141 feedback --- core/lexicon.json | 8 ++++---- db/field.py | 2 +- tests/conftest.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 1c4980449..41826dcb4 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -252,10 +252,10 @@ {"categories": [{"name": "activity_type", "description": null}], "term": "water chemistry", "definition": "water chemistry"}, - {"categories": [{"name": "field_event_role", "description": null}], "term": "Lead", "definition": "the leader of the field event"}, - {"categories": [{"name": "field_event_role", "description": null}], "term": "Participant", "definition": "a person participating in the field event"}, - {"categories": [{"name": "field_event_role", "description": null}], "term": "Observer", "definition": "a person observing the field event"}, - {"categories": [{"name": "field_event_role", "description": null}], "term": "Visitor", "definition": "a person visiting the field event"}, + {"categories": [{"name": "field_contact_role", "description": null}], "term": "Lead", "definition": "the leader of the field event"}, + {"categories": [{"name": "field_contact_role", "description": null}], "term": "Participant", "definition": "a person participating in the field event"}, + {"categories": [{"name": "field_contact_role", "description": null}], "term": "Observer", "definition": "a person observing the field event"}, + {"categories": [{"name": "field_contact_role", "description": null}], "term": "Visitor", "definition": "a person visiting the field event"}, {"categories": [{"name": "sample_matrix", "description": null}], "term": "water", "definition": "water"}, diff --git a/db/field.py b/db/field.py index 988ef3051..ff7b1b740 100644 --- a/db/field.py +++ b/db/field.py @@ -25,7 +25,7 @@ class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): ) # TODO: get AMP feedback on the roles - field_event_role: Mapped[str] = lexicon_term( + field_contact_role: Mapped[str] = lexicon_term( nullable=False, comment="Role of the contact in the field event" ) diff --git a/tests/conftest.py b/tests/conftest.py index 2eb928e23..4674e3b9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -479,7 +479,7 @@ def field_event_contact(field_event, contact): field_event_contact = FieldEventContactAssociation( field_event_id=field_event.id, contact_id=contact.id, - field_event_role="Lead", + field_contact_role="Lead", ) session.add(field_event_contact) session.commit() From 8a518e59ac522dc33c8a5172520ff73a94172285 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 22 Sep 2025 16:27:25 -0600 Subject: [PATCH 32/37] refactor: remove cascade behavior of field event contact association --- db/field.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db/field.py b/db/field.py index ff7b1b740..81d295512 100644 --- a/db/field.py +++ b/db/field.py @@ -38,7 +38,6 @@ class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): samples: Mapped[list["Sample"]] = relationship( # noqa: F821 "Sample", back_populates="field_event_contact", - cascade="all, delete-orphan", passive_deletes=True, ) From 9cf0efe0d008d019da2366389fd27cd6bef6c818 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 22 Sep 2025 16:33:42 -0600 Subject: [PATCH 33/37] fix: ensure correct relationships for FieldEventContactAssociation --- db/field.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/db/field.py b/db/field.py index 81d295512..a9d94503a 100644 --- a/db/field.py +++ b/db/field.py @@ -4,6 +4,7 @@ from sqlalchemy.ext.associationproxy import association_proxy from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term +from db.contact import Contact class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): @@ -30,8 +31,12 @@ class FieldEventContactAssociation(Base, AutoBaseMixin, ReleaseMixin): ) # --- Relationships --- - field_event: Mapped["FieldEvent"] = relationship("FieldEvent") - contact: Mapped["Contact"] = relationship("Contact") # noqa: F821 + field_event: Mapped["FieldEvent"] = relationship( + "FieldEvent", back_populates="field_event_contact_associations" + ) + contact: Mapped["Contact"] = relationship( # noqa: F821 + "Contact", back_populates="field_event_contact_associations" + ) # map associated contacts to samples to restrict the people who could have # taken a sample to those present at the field event From 61301efa9eb3d3c570626f562034b4f0ebd7d494 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 22 Sep 2025 16:36:33 -0600 Subject: [PATCH 34/37] refactor: update FieldEvent documentation per PR feedback --- db/field.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db/field.py b/db/field.py index a9d94503a..bacd1d747 100644 --- a/db/field.py +++ b/db/field.py @@ -57,6 +57,11 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): entire visit, such as the date, time, and the person responsible. It acts as the parent container for all activities performed and all samples collected during that single visit. + + Its purpose is to store the "where and when" of the event. + Information about who participated is managed in the + FieldEventContactAssociation table. Information about the "what" of the + event is managed in the FieldActivity and Sample tables. """ # Foreign Keys From 31df3ec3e56b4443309fafad38637fc464e069ff Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 23 Sep 2025 09:56:53 -0600 Subject: [PATCH 35/37] refactor: PR 141 feedback - update documentation --- db/field.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/field.py b/db/field.py index bacd1d747..d3b07c937 100644 --- a/db/field.py +++ b/db/field.py @@ -80,7 +80,7 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): collecting_organization: Mapped[str] = lexicon_term( nullable=False, default="NMBGMR", # TODO: put default in schema - comment="The organization that is collecting and storing the samples from the field event", + comment="The organization that is collecting the samples from the field event", ) notes: Mapped[str] = mapped_column( nullable=True, From b4ae588ff464ad1e0a0f8377e1023a85023fa07c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 23 Sep 2025 10:50:40 -0600 Subject: [PATCH 36/37] refactor: address PR 141 feedback --- db/contact.py | 18 ++++++++++++------ db/field.py | 15 ++++++++++----- db/lexicon.py | 6 ++++-- db/observation.py | 8 ++++++-- db/publication.py | 16 +++++++++++----- db/sample.py | 35 ++++++++++++++++++----------------- db/sensor.py | 4 ++-- db/thing.py | 15 +++++++++++---- schemas/sample.py | 3 --- tests/conftest.py | 11 +++-------- tests/test_sample.py | 16 ++-------------- 11 files changed, 79 insertions(+), 68 deletions(-) diff --git a/db/contact.py b/db/contact.py index d2ac9466d..ce4346fdc 100644 --- a/db/contact.py +++ b/db/contact.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== from sqlalchemy import Integer, ForeignKey, String -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped, mapped_column from sqlalchemy_utils import TSVectorType from typing import List @@ -62,19 +62,22 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): cascade="all, delete-orphan", ) ) - authors = association_proxy("author_associations", "author") + authors: AssociationProxy[list["Author"]] = association_proxy( # noqa: F821 + "author_associations", "author" + ) thing_associations: Mapped[List["ThingContactAssociation"]] = relationship( "ThingContactAssociation", back_populates="contact", cascade="all, delete-orphan", passive_deletes=True, ) - things = association_proxy("thing_associations", "thing") + things: AssociationProxy[list["Thing"]] = association_proxy( # noqa: F821 + "thing_associations", "thing" + ) # Proxy to directly access the FieldEvent objects in which this Contact participated. # fmt: off - field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # noqa: F821 - # fmt: on + field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( # noqa: F821 relationship( "FieldEventContactAssociation", back_populates="contact", @@ -82,7 +85,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): passive_deletes=True, ) ) - field_events = association_proxy("field_event_contact_associations", "field_event") + # fmt: on + field_events: AssociationProxy[list["FieldEvent"]] = ( # noqa: F821 + association_proxy("field_event_contact_associations", "field_event") + ) class Phone(Base, AutoBaseMixin, ReleaseMixin): diff --git a/db/field.py b/db/field.py index d3b07c937..dde66e30e 100644 --- a/db/field.py +++ b/db/field.py @@ -1,7 +1,7 @@ from datetime import datetime from sqlalchemy import DateTime, ForeignKey from sqlalchemy.orm import mapped_column, relationship, Mapped -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term from db.contact import Contact @@ -89,7 +89,7 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): # --- Relationships --- thing: Mapped["Thing"] = relationship(back_populates="field_events") # noqa: F821 field_activities: Mapped[list["FieldActivity"]] = relationship( - back_populates="field_event" + "FieldActivity", back_populates="field_event" ) field_event_contact_associations: Mapped[list["FieldEventContactAssociation"]] = ( relationship( @@ -102,7 +102,7 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): # --- Association Proxies --- # Proxy to directly access the Contact objects participating in this event. - contacts: Mapped[list["Contact"]] = association_proxy( # noqa: F821 + contacts: AssociationProxy[list["Contact"]] = association_proxy( # noqa: F821 "field_event_contact_associations", "contact" ) @@ -140,7 +140,12 @@ class FieldActivity(Base, AutoBaseMixin, ReleaseMixin): ) # Relationships - field_event: Mapped["FieldEvent"] = relationship(back_populates="field_activities") + field_event: Mapped["FieldEvent"] = relationship( + "FieldEvent", back_populates="field_activities" + ) samples: Mapped[list["Sample"]] = relationship( # noqa: F821 - back_populates="field_activity" + "Sample", + back_populates="field_activity", + cascade="all, delete-orphan", + passive_deletes=True, ) diff --git a/db/lexicon.py b/db/lexicon.py index 4c8f68f63..ba03cec5c 100644 --- a/db/lexicon.py +++ b/db/lexicon.py @@ -15,7 +15,7 @@ # =============================================================================== from sqlalchemy import String, ForeignKey, Integer from sqlalchemy.orm import mapped_column, relationship -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from db.base import AutoBaseMixin, Base, lexicon_term @@ -35,7 +35,9 @@ class LexiconTerm(Base, AutoBaseMixin): back_populates="term", cascade="all, delete-orphan", ) - categories = association_proxy("category_associations", "category") + categories: AssociationProxy[list["LexiconCategory"]] = association_proxy( + "category_associations", "category" + ) def __repr__(self): return f"" diff --git a/db/observation.py b/db/observation.py index ef951c912..720fcda0f 100644 --- a/db/observation.py +++ b/db/observation.py @@ -63,8 +63,12 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): doc="Depth of the geothermal observation in feet", ) - sensor: Mapped["Sensor"] = relationship("Sensor") # noqa: F821 - sample: Mapped["Sample"] = relationship("Sample") # noqa: F821 + sensor: Mapped["Sensor"] = relationship( # noqa: F821 + "Sensor", back_populates="observations", passive_deletes=True + ) # noqa: F821 + sample: Mapped["Sample"] = relationship( # noqa: F821 + "Sample", back_populates="observations", passive_deletes=True + ) # noqa: F821 # ============= EOF ============================================= diff --git a/db/publication.py b/db/publication.py index 6b04691b3..2eec8213a 100644 --- a/db/publication.py +++ b/db/publication.py @@ -17,9 +17,9 @@ from db import lexicon_term from db.base import AutoBaseMixin, Base, AuditMixin -from sqlalchemy import Column, Integer, String, Text, Date, ForeignKey, Table, DateTime +from sqlalchemy import Column, Integer, String, Text, ForeignKey from sqlalchemy.orm import relationship -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy class Publication(Base, AutoBaseMixin): @@ -45,7 +45,9 @@ class Publication(Base, AutoBaseMixin): back_populates="publication", cascade="all, delete-orphan", ) - authors = association_proxy("author_associations", "author") + authors: AssociationProxy[list["Author"]] = association_proxy( + "author_associations", "author" + ) search_vector = Column(TSVectorType("title", "abstract", "doi", "publisher", "url")) @@ -65,14 +67,18 @@ class Author(Base, AutoBaseMixin): back_populates="author", cascade="all, delete-orphan", ) - publications = association_proxy("publication_associations", "publication") + publications: AssociationProxy[list["Publication"]] = association_proxy( + "publication_associations", "publication" + ) contact_associations = relationship( "AuthorContactAssociation", back_populates="author", cascade="all, delete-orphan", ) - contacts = association_proxy("author_associations", "contact") + contacts: AssociationProxy[list["Contact"]] = association_proxy( # noqa: F821 + "author_associations", "contact" + ) search_vector = Column(TSVectorType("name", "affiliation")) diff --git a/db/sample.py b/db/sample.py index 782e54a8b..0c3c8d3ca 100644 --- a/db/sample.py +++ b/db/sample.py @@ -15,13 +15,10 @@ # =============================================================================== from sqlalchemy import DateTime, ForeignKey from sqlalchemy.orm import mapped_column, relationship, Mapped -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy # import models from classes that are defined in separate files from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term -from db.sensor import Sensor - -from typing import Optional import datetime @@ -46,15 +43,8 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): ForeignKey("field_activity.id"), nullable=False ) - # nullable because sample can be collected by steel tape - sensor_id: Mapped[Optional[int]] = mapped_column( - ForeignKey("sensor.id"), - comment="Foreign key for the specific equipment used.", - nullable=True, - ) - - field_event_contact_id: Mapped[Optional[str]] = mapped_column( - ForeignKey("field_event_contact_association.id") + field_event_contact_id: Mapped[str] = mapped_column( + ForeignKey("field_event_contact_association.id"), nullable=True ) # --- Columns --- @@ -91,7 +81,6 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): field_activity: Mapped["FieldActivity"] = relationship( # noqa: F821 back_populates="samples" ) - sensor: Mapped["Sensor"] = relationship(back_populates="samples") # noqa: F821 # fmt: off field_event_contact: Mapped["FieldEventContactAssociation"] = relationship( # noqa: F821 back_populates="samples" @@ -99,9 +88,21 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): # fmt: on # association proxies to help keep code DRY - field_event = association_proxy("field_activity", "field_event") - thing = association_proxy("field_activity", "field_event.thing") - contact = association_proxy("field_event_contact", "contact") # noqa: F821 + field_event: AssociationProxy[list["FieldEvent"]] = association_proxy( # noqa: F821 + "field_activity", "field_event" + ) + thing: AssociationProxy[list["Thing"]] = association_proxy( # noqa: F821 + "field_activity", "field_event.thing" + ) + contact: AssociationProxy[list["Contact"]] = association_proxy( # noqa: F821 + "field_event_contact", "contact" + ) # noqa: F821 + observations: Mapped[list["Observation"]] = relationship( # noqa: F821 + "Observation", + back_populates="sample", + cascade="all, delete-orphan", + passive_deletes=True, + ) # ============= EOF ============================================= diff --git a/db/sensor.py b/db/sensor.py index ada548d0e..6ab04b7c9 100644 --- a/db/sensor.py +++ b/db/sensor.py @@ -40,8 +40,8 @@ class Sensor(Base, AutoBaseMixin, ReleaseMixin): recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) notes: Mapped[str] = mapped_column(String(50), nullable=True) - samples: Mapped[list["Sample"]] = relationship( # noqa: F821 - "Sample", + observations: Mapped[list["Observation"]] = relationship( # noqa: F821 + "Observation", back_populates="sensor", ) diff --git a/db/thing.py b/db/thing.py index a2ab09e89..f09fa4930 100644 --- a/db/thing.py +++ b/db/thing.py @@ -14,11 +14,12 @@ # limitations under the License. # =============================================================================== from sqlalchemy import Integer, ForeignKey, String, Column, Float -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType from db import lexicon_term +from db.asset import Asset from db.base import AutoBaseMixin, Base, ReleaseMixin @@ -35,7 +36,9 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): overlaps="things", cascade="all, delete-orphan", ) - assets = association_proxy("asset_associations", "asset") + assets: AssociationProxy[list["Asset"]] = association_proxy( + "asset_associations", "asset" + ) location_associations = relationship( "LocationThingAssociation", @@ -44,7 +47,9 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): cascade="all, delete-orphan", order_by="LocationThingAssociation.effective_start.desc()", ) - locations = association_proxy("location_associations", "location") + locations: AssociationProxy[list["Location"]] = association_proxy( # noqa: F821 + "location_associations", "location" + ) contact_associations = relationship( "ThingContactAssociation", @@ -52,7 +57,9 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): overlaps="contacts", cascade="all, delete-orphan", ) - contacts = association_proxy("contact_associations", "contact") + contacts: AssociationProxy[list["Contact"]] = association_proxy( # noqa: F821 + "contact_associations", "contact" + ) # Well fields well_depth = Column( diff --git a/schemas/sample.py b/schemas/sample.py index bd1f7468d..aec115d9a 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -84,7 +84,6 @@ def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime: # -------- CREATE ---------- class CreateSample(BaseCreateModel, ValidateSample): field_activity_id: int - sensor_id: int | None = None field_event_contact_id: int sample_date: Annotated[AwareDatetime, PastDatetime()] sample_name: str @@ -99,7 +98,6 @@ class CreateSample(BaseCreateModel, ValidateSample): # -------- UPDATE ---------- class UpdateSample(BaseUpdateModel, ValidateSample): field_activity_id: int | None = None # TODO: should this be editable? - sensor_id: int | None = None field_event_contact_id: int | None = None sample_date: Annotated[AwareDatetime, PastDatetime()] | None = None sample_name: str | None = None @@ -128,7 +126,6 @@ class SampleResponse(BaseResponseModel): contact: ContactResponse field_activity_id: int field_event_contact_id: int - sensor_id: int | None sample_date: Annotated[AwareDatetime, PastDatetime()] sample_name: str sample_matrix: str diff --git a/tests/conftest.py b/tests/conftest.py index 4674e3b9c..4db58422d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -515,14 +515,11 @@ def water_chemistry_field_activity(field_event): @pytest.fixture(scope="session") -def groundwater_level_sample( - groundwater_level_field_activity, sensor, field_event_contact -): +def groundwater_level_sample(groundwater_level_field_activity, field_event_contact): with session_ctx() as session: sample = Sample( field_activity_id=groundwater_level_field_activity.id, field_event_contact_id=field_event_contact.id, - sensor_id=sensor.id, sample_date="2025-01-01T12:00:00Z", sample_name="groundwater level sample name", sample_matrix="water", @@ -539,11 +536,10 @@ def groundwater_level_sample( @pytest.fixture(scope="session") -def water_chemistry_sample(water_chemistry_field_activity, sensor, field_event_contact): +def water_chemistry_sample(water_chemistry_field_activity, field_event_contact): with session_ctx() as session: sample = Sample( field_activity_id=water_chemistry_field_activity.id, - sensor_id=sensor.id, field_event_contact_id=field_event_contact.id, sample_date="2025-01-01T13:00:00Z", sample_name="water chemistry sample name", @@ -561,11 +557,10 @@ def water_chemistry_sample(water_chemistry_field_activity, sensor, field_event_c @pytest.fixture(scope="function") -def sample_to_delete(water_chemistry_field_activity, sensor, field_event_contact): +def sample_to_delete(water_chemistry_field_activity, field_event_contact): with session_ctx() as session: sample = Sample( field_activity_id=water_chemistry_field_activity.id, - sensor_id=sensor.id, field_event_contact_id=field_event_contact.id, sample_date="2025-01-01T13:00:00Z", sample_name="sample to delete", diff --git a/tests/test_sample.py b/tests/test_sample.py index a74ea44a0..b1b495afe 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -54,14 +54,13 @@ def test_validate_sample_top_and_bottom(): # ============= Post tests for samples ============================================= def test_add_sample( - groundwater_level_field_activity, water_well_thing, sensor, field_event_contact + groundwater_level_field_activity, water_well_thing, field_event_contact ): """ Test adding a sample. """ payload = { "field_activity_id": groundwater_level_field_activity.id, - "sensor_id": sensor.id, "field_event_contact_id": field_event_contact.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": "second groundwater level field activity name", @@ -88,7 +87,6 @@ def test_add_sample( assert data["field_activity_id"] == payload["field_activity_id"] assert data["contact"]["id"] == field_event_contact.contact_id assert data["field_event_contact_id"] == payload["field_event_contact_id"] - assert data["sensor_id"] == payload["sensor_id"] assert data["sample_date"] == payload["sample_date"] assert data["sample_name"] == payload["sample_name"] assert data["sample_matrix"] == payload["sample_matrix"] @@ -106,7 +104,6 @@ def test_add_sample( def test_409_add_sample_invalid_sample_name( groundwater_level_field_activity, groundwater_level_sample, - sensor, field_event_contact, ): """ @@ -115,7 +112,6 @@ def test_409_add_sample_invalid_sample_name( payload = { "field_activity_id": groundwater_level_field_activity.id, "field_event_contact_id": field_event_contact.id, - "sensor_id": sensor.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": groundwater_level_sample.sample_name, "sample_matrix": "water", @@ -145,7 +141,6 @@ def test_409_add_sample_invalid_sample_name( def test_409_add_sample_invalid_field_activity_id( groundwater_level_field_activity, groundwater_level_sample, - sensor, field_event_contact, ): """ @@ -153,7 +148,6 @@ def test_409_add_sample_invalid_field_activity_id( """ payload = { "field_activity_id": 999999, - "sensor_id": sensor.id, "field_event_contact_id": field_event_contact.id, "sample_date": "2025-01-01T14:00:00Z", "sample_name": "yet another sample name", @@ -183,15 +177,12 @@ def test_409_add_sample_invalid_field_activity_id( # ============= Patch tests for samples ============================================= -def test_patch_sample( - water_chemistry_sample, second_sensor, groundwater_level_field_activity -): +def test_patch_sample(water_chemistry_sample, groundwater_level_field_activity): """ Test updating a sample. """ payload = { "field_activity_id": groundwater_level_field_activity.id, - "sensor_id": second_sensor.id, # "field_event_contact_id": third_contact.id, "sample_date": "2025-01-02T00:00:00Z", "sample_name": "patched sample name", @@ -300,7 +291,6 @@ def test_get_samples(water_chemistry_sample, groundwater_level_sample): assert "field_activity_id" in item assert "contact" in item assert "field_event_contact_id" in item - assert "sensor_id" in item assert "sample_date" in item assert "sample_name" in item assert "sample_matrix" in item @@ -317,7 +307,6 @@ def test_get_sample_by_id( water_chemistry_field_activity, field_event, water_well_thing, - sensor, field_event_contact, ): """ @@ -335,7 +324,6 @@ def test_get_sample_by_id( assert data["field_activity"]["id"] == water_chemistry_field_activity.id assert data["field_activity_id"] == water_chemistry_field_activity.id assert data["field_event_contact_id"] == field_event_contact.id - assert data["sensor_id"] == sensor.id assert data["sample_date"] == water_chemistry_sample.sample_date assert data["sample_name"] == water_chemistry_sample.sample_name assert data["sample_matrix"] == water_chemistry_sample.sample_matrix From 955831e7acd77864645510dc73849ef1718c2be3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 23 Sep 2025 15:25:39 -0600 Subject: [PATCH 37/37] refactor: infer collecting organization from contact --- db/contact.py | 2 +- db/field.py | 5 ----- schemas/field.py | 1 - tests/conftest.py | 5 ++--- tests/test_contact.py | 4 ++-- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/db/contact.py b/db/contact.py index ce4346fdc..7a4f79340 100644 --- a/db/contact.py +++ b/db/contact.py @@ -36,7 +36,7 @@ class ThingContactAssociation(Base, AutoBaseMixin): class Contact(Base, AutoBaseMixin, ReleaseMixin): name: Mapped[str] = mapped_column(String(100), nullable=True) - organization: Mapped[str] = mapped_column(String(100), nullable=True) + organization: Mapped[str] = lexicon_term(nullable=True) role: Mapped[str] = lexicon_term(nullable=False) contact_type: Mapped[str] = lexicon_term(nullable=False) nma_pk_owners: Mapped[str] = mapped_column(String(100), nullable=True) diff --git a/db/field.py b/db/field.py index dde66e30e..ca7b99372 100644 --- a/db/field.py +++ b/db/field.py @@ -77,11 +77,6 @@ class FieldEvent(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="Date and time of the field event.", ) - collecting_organization: Mapped[str] = lexicon_term( - nullable=False, - default="NMBGMR", # TODO: put default in schema - comment="The organization that is collecting the samples from the field event", - ) notes: Mapped[str] = mapped_column( nullable=True, comment="Notes or comments about the field event.", diff --git a/schemas/field.py b/schemas/field.py index 43a042918..9f32ed72f 100644 --- a/schemas/field.py +++ b/schemas/field.py @@ -14,5 +14,4 @@ class FieldActivityResponse(BaseResponseModel): class FieldEventResponse(BaseResponseModel): thing_id: int event_date: AwareDatetime - collecting_organization: str | None notes: str | None diff --git a/tests/conftest.py b/tests/conftest.py index 4db58422d..250754c0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -227,7 +227,7 @@ def contact(water_well_thing): name="Test Contact", role="Owner", contact_type="Primary", - organization="Test Organization", + organization="NMBGMR", ) session.add(contact) session.commit() @@ -377,7 +377,7 @@ def third_contact(): name=None, role="Owner", contact_type="Primary", - organization="Third Organization", + organization="NMBGMR", ) session.add(contact) session.commit() @@ -464,7 +464,6 @@ def field_event(water_well_thing): field_event = FieldEvent( thing_id=water_well_thing.id, event_date="2025-01-01T00:00:00Z", - collecting_organization="NMBGMR", notes="field event fixture notes", release_status="draft", ) diff --git a/tests/test_contact.py b/tests/test_contact.py index b505ff4cb..22b5ad65a 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -72,7 +72,7 @@ def test_add_contact(spring_thing): "name": "Test Contact 2", "role": "Owner", "contact_type": "Primary", - "organization": "Well Owner LLC", + "organization": "NMBGMR", "thing_id": spring_thing.id, "emails": [ { @@ -160,7 +160,7 @@ def test_add_contact_409_bad_thing_id(): "name": "Test Contact 3", "role": "Owner", "contact_type": "Primary", - "organization": "Well Owner LLC", + "organization": "NMBGMR", "thing_id": bad_thing_id, "emails": [ {