BDMS 159: well transfers updates, water level transfer updates, & other revisions

api/search.py

@@ -81,8 +81,9 @@ def _get_thing_results(session: Session, q: str, limit: int) -> list[dict]:
         select(Thing).where(Thing.thing_type == "spring"), q, vector=vector, limit=limit
     )
 
-    wells = session.scalars(water_well_query).all()
-    springs = session.scalars(spring_well_query).all()
+    # unique needs to be called because of eager loads
+    wells = session.scalars(water_well_query).unique().all()
+    springs = session.scalars(spring_well_query).unique().all()
 
     def _make_response(group: str, thing: Thing, properties: dict) -> dict:
 

api/thing.py

@@ -62,7 +62,6 @@
     add_well_screen,
     get_db_things,
     get_thing_of_a_thing_type_by_id,
-    get_active_location,
 )
 from services.lexicon_helper import get_terms_by_category
 
@@ -328,7 +327,6 @@ async def get_thing_by_id(
     Retrieve a thing by ID from the database.
     """
     thing = simple_get_by_id(session, Thing, thing_id)
-    thing.active_location = get_active_location(session, thing)
     return thing
 
 

core/lexicon.json

@@ -9,7 +9,7 @@
     {"name": "coordinate_method", "description": null},
     {"name": "country", "description": null},
     {"name": "county", "description": null},
-    {"name": "current_use", "description": null},
+    {"name": "well_purpose", "description": null},
     {"name": "data_quality", "description": null},
     {"name": "data_source", "description": null},
     {"name": "depth_completion_source", "description": null},
@@ -20,9 +20,8 @@
     {"name": "participant_role", "description": null},
     {"name": "geochronology", "description": null},
     {"name": "horizontal_datum", "description": null},
-    {"name": "value_reason", "description": null},
-    {"name": "limit_type", "description": null},
     {"name": "groundwater_level_reason", "description": null},
+    {"name": "limit_type", "description": null},
     {"name": "measurement_method", "description": null},
     {"name": "monitoring_status", "description": null},
     {"name": "parameter_name", "description": null},
@@ -43,8 +42,8 @@
     {"name": "status", "description": null},
     {"name": "thing_type", "description": null},
     {"name": "unit", "description": null},
-    {"name": "vertical_datum", "description": null},
-    {"name": "well_purpose", "description": null}],
+    {"name": "vertical_datum", "description": null}
+  ],
   "terms": [
     {"categories": ["qc_type"], "term": "Normal", "definition": "The primary environmental sample collected from the well, spring, or soil boring."},
     {"categories": ["qc_type"], "term": "Duplicate", "definition": "A second, independent sample collected at the same location, at the same time, and in the same manner as the normal sample. This sample is sent to the primary laboratory."},
@@ -68,7 +67,7 @@
     {"categories": ["elevation_method"], "term": "Reported", "definition": "Reported"},
     {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"},
     {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"},
-    {"categories": ["elevation_method", "sample_method", "coordinate_method", "current_use", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"},
+    {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"},
     {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"},
     {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"},
     {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"},
@@ -84,19 +83,21 @@
     {"categories": ["coordinate_method"], "term": "Interpolated from DEM", "definition": "Interpolated from DEM"},
     {"categories": ["coordinate_method"], "term": "Reported", "definition": "Reported"},
     {"categories": ["coordinate_method"], "term": "Transit, theodolite, or other survey method", "definition": "Transit, theodolite, or other survey method"},
-    {"categories": ["current_use"], "term": "Open, unequipped well", "definition": "Open, unequipped well"},
-    {"categories": ["current_use"], "term": "Commercial", "definition": "Commercial"},
-    {"categories": ["current_use"], "term": "Domestic", "definition": "Domestic"},
-    {"categories": ["current_use"], "term": "Power generation", "definition": "Power generation"},
-    {"categories": ["current_use"], "term": "Irrigation", "definition": "Irrigation"},
-    {"categories": ["current_use"], "term": "Livestock", "definition": "Livestock"},
-    {"categories": ["current_use"], "term": "Mining", "definition": "Mining"},
-    {"categories": ["current_use"], "term": "Industrial", "definition": "Industrial"},
-    {"categories": ["current_use"], "term": "Observation", "definition": "Observation"},
-    {"categories": ["current_use"], "term": "Public supply", "definition": "Public supply"},
-    {"categories": ["current_use"], "term": "Shared domestic", "definition": "Shared domestic"},
-    {"categories": ["current_use"], "term": "Institutional", "definition": "Institutional"},
-    {"categories": ["current_use"], "term": "Unused", "definition": "Unused"},
+    {"categories": ["well_purpose"], "term": "Open, unequipped well", "definition": "Open, unequipped well"},
+    {"categories": ["well_purpose"], "term": "Commercial", "definition": "Commercial"},
+    {"categories": ["well_purpose"], "term": "Domestic", "definition": "Domestic"},
+    {"categories": ["well_purpose"], "term": "Power generation", "definition": "Power generation"},
+    {"categories": ["well_purpose"], "term": "Irrigation", "definition": "Irrigation"},
+    {"categories": ["well_purpose"], "term": "Livestock", "definition": "Livestock"},
+    {"categories": ["well_purpose"], "term": "Mining", "definition": "Mining"},
+    {"categories": ["well_purpose"], "term": "Industrial", "definition": "Industrial"},
+    {"categories": ["well_purpose"], "term": "Observation", "definition": "Observation"},
+    {"categories": ["well_purpose"], "term": "Public supply", "definition": "Public supply"},
+    {"categories": ["well_purpose"], "term": "Shared domestic", "definition": "Shared domestic"},
+    {"categories": ["well_purpose"], "term": "Institutional", "definition": "Institutional"},
+    {"categories": ["well_purpose"], "term": "Unused", "definition": "Unused"},
+    {"categories": ["well_purpose"], "term": "Exploration", "definition": "Exploration well"},
+    {"categories": ["well_purpose"], "term": "Injection", "definition": "Injection"},
     {"categories": ["data_quality"], "term": "Water level accurate to within two hundreths of a foot", "definition": "Good"},
     {"categories": ["data_quality"], "term": "Water level accurate to within one foot", "definition": "Fair"},
     {"categories": ["data_quality"], "term": "Water level accuracy not to nearest foot or water level not repeatable", "definition": "Poor"},
@@ -355,7 +356,7 @@
     {"categories": ["organization"], "term": "Bernalillo County", "definition": "Bernalillo County"},
     {"categories": ["organization"], "term": "BLM", "definition": "Bureau of Land Management"},
     {"categories": ["organization"], "term": "SFC", "definition": "Santa Fe County"},
-    {"categories": ["organization"], "term": "CSF", "definition": "City of Santa Fe"},
+    {"categories": ["organization"], "term": "City of Santa Fe", "definition": "City of Santa Fe"},
     {"categories": ["organization"], "term": "NESWCD", "definition": "Northeastern Soil & Water Conservation District"},
     {"categories": ["organization"], "term": "NMISC", "definition": "New Mexico Interstate Stream Commission"},
     {"categories": ["organization"], "term": "PVACD", "definition": "Pecos Valley Artesian Conservancy District"},
@@ -370,6 +371,8 @@
     {"categories": ["organization"], "term": "Taos SWCD", "definition": "Taos Soil and Water Conservation District"},
     {"categories": ["organization"], "term": "Otero SWCD", "definition": "Otero Soil and Water Conservation District"},
     {"categories": ["organization"], "term": "Northeastern SWCD", "definition": "Northeastern Soil and Water Conservation District"},
+    {"categories": ["organization"], "term": "CDWR", "definition": "Colorado Division of Water Resources"},
+    {"categories": ["organization"], "term": "Pendaries Village", "definition": "Pendaries Village"},
     {"categories": ["organization"], "term": "A&T Pump & Well Service, LLC", "definition": "A&T Pump & Well Service, LLC"},
     {"categories": ["organization"], "term": "A. G. Wassenaar, Inc", "definition": "A. G. Wassenaar, Inc"},
     {"categories": ["organization"], "term": "AMEC", "definition": "AMEC"},
@@ -401,6 +404,10 @@
     {"categories": ["organization"], "term": "Thompson Drilling, Inc", "definition": "Thompson Drilling, Inc"},
     {"categories": ["organization"], "term": "Witcher & Associates", "definition": "Witcher & Associates"},
     {"categories": ["organization"], "term": "Zeigler Geologic Consulting, LLC", "definition": "Zeigler Geologic Consulting, LLC"},
+    {"categories": ["organization"], "term": "Sandia Well Service, Inc", "definition": "Sandia Well Service, Inc"},
+    {"categories": ["organization"], "term": "San Marcos Association", "definition": "San Marcos Association"},
+    {"categories": ["organization"], "term": "URS", "definition": "URS"},
+    {"categories": ["organization"], "term": "Vista del Oro", "definition": "Vista del Oro"},
     {"categories": ["collection_method"], "term": "manual", "definition": "manual sampling"},
     {"categories": ["collection_method"], "term": "continuous", "definition": "continuous sampling"},
     {"categories": ["role"], "term": "Owner", "definition": "Owner"},
@@ -416,10 +423,12 @@
     {"categories": ["role"], "term": "Technician", "definition": "Technician"},
     {"categories": ["role"], "term": "Research Assistant", "definition": "Research Assistant"},
     {"categories": ["role"], "term": "Research Scientist", "definition": "Research Scientist"},
+    {"categories": ["role"], "term": "Graduate Student", "definition": "Graduate Student"},
     {"categories": ["role"], "term": "Operator", "definition": "Operator"},
     {"categories": ["role"], "term": "Biologist", "definition": "Biologist"},
     {"categories": ["role"], "term": "Lab Manager", "definition": "Lab Manager"},
     {"categories": ["role"], "term": "Publications Manager", "definition": "Publications Manager"},
+    {"categories": ["role"], "term": "Software Developer", "definition": "Software Developer"},
     {"categories": ["email_type", "phone_type", "address_type", "contact_type"], "term": "Primary", "definition": "primary"},
     {"categories": ["contact_type"], "term": "Secondary", "definition": "secondary"},
     {"categories": ["contact_type"], "term": "Field Event Participant", "definition": "A contact who has participated in a field event"},
@@ -434,10 +443,6 @@
     {"categories": ["spring_type"], "term": "Perennial", "definition": "perennial spring"},
     {"categories": ["spring_type"], "term": "Thermal", "definition": "thermal spring"},
     {"categories": ["spring_type"], "term": "Mineral", "definition": "mineral spring"},
-    {"categories": ["well_purpose"], "term": "Exploration", "definition": "Exploration well"},
-    {"categories": ["well_purpose"], "term": "Monitoring", "definition": "Monitoring"},
-    {"categories": ["well_purpose"], "term": "Production", "definition": "Production"},
-    {"categories": ["well_purpose"], "term": "Injection", "definition": "Injection"},
     {"categories": ["casing_material"], "term": "PVC", "definition": "Polyvinyl Chloride"},
     {"categories": ["casing_material"], "term": "Steel", "definition": "Steel"},
     {"categories": ["casing_material"], "term": "Concrete", "definition": "Concrete"},

db/location.py

@@ -99,7 +99,9 @@ class LocationThingAssociation(Base, AutoBaseMixin):
     )
 
     # --- Relationship Definitions ---
-    location: Mapped["Location"] = relationship(back_populates="thing_associations")
+    location: Mapped["Location"] = relationship(
+        back_populates="thing_associations", lazy="joined"
+    )
     thing: Mapped["Thing"] = relationship(back_populates="location_associations")
 
 

db/thing.py

@@ -48,6 +48,11 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix
     __versioned__ = {}
 
     # --- Columns ---
+    nma_pk_welldata: Mapped[str] = mapped_column(
+        nullable=True,
+        comment="To audit where the data came from in NM_Aquifer if it was transferred over",
+    )
+
     # TODO: should `name` be unique?
     name: Mapped[str] = mapped_column(
         nullable=False,
@@ -56,7 +61,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix
     # TODO: what is the purpose of the `description` field? Is it ever used?
     # description: Mapped[str] = mapped_column(String(500), nullable=True)
     thing_type: Mapped[str] = lexicon_term(
-        nullable=True,
+        nullable=False,
         comment="A controlled vocabulary field defining the type of infrastructure (e.g., 'Well', 'Spring', 'Stream Gauge').",
     )
     first_visit_date: Mapped[Date] = mapped_column(
@@ -119,13 +124,52 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix
 
     # One-To-Many: A Thing can be at many locations over time.
     # If the Thing is deleted, its location history will be deleted.
+    """
+    Developer's notes
+
+    If there are many location associations related to a thing, eagerly loading
+    the location associations may overburden the API and DB and introduce
+    performance issues. If this becomes an issue, active/current locations can 
+    be fetched in queries when retrieving things. Be thorough if following this
+    route as it will need to be included everywhere where a thing record is
+    needed, such as for GET /thing, GET /thing/{thing_id}, and GET /contact. See
+    below for an example of a way to retrieve the active/current location in a
+    query:
+
+    from sqlalchemy.orm import aliased
+    from sqlalchemy import select, func
+
+    LTA = aliased(LocationThingAssociation)
+    latest_assoc = (
+        select(
+            LTA.thing_id,
+            func.max(LTA.effective_start).label("max_start")
+        )
+        .where(LTA.effective_end == None)
+        .group_by(LTA.thing_id)
+        .subquery()
+    )
+
+    lta_alias = aliased(LocationThingAssociation)
+    query = (
+        select(Thing, Location)
+        .join(lta_alias, Thing.id == lta_alias.thing_id)
+        .join(Location, lta_alias.location_id == Location.id)
+        .join(
+            latest_assoc,
+            (latest_assoc.c.thing_id == lta_alias.thing_id) &
+            (latest_assoc.c.max_start == lta_alias.effective_start)
+        )
+    )
+    """
     location_associations = relationship(
         "LocationThingAssociation",
         back_populates="thing",
         overlaps="location",
         cascade="all, delete-orphan",
         passive_deletes=True,
         order_by="LocationThingAssociation.effective_start.desc()",
+        lazy="joined",
     )
 
     contact_associations = relationship(
@@ -199,6 +243,22 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix
         )
     )
 
+    @property
+    def current_location(self):
+        """
+        Returns the currently active Location by sorting the effective_start
+        field. Thing eagerly loads location_association, which eagerly loads
+        location, which will hopefully prevent N+1 query problems.
+        """
+        current_location = sorted(
+            self.location_associations, key=lambda x: x.effective_start, reverse=True
+        )
+        return (
+            current_location[0].location
+            if current_location and current_location[0].effective_end is None
+            else None
+        )
+
 
 class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin):
     """

schemas/contact.py

@@ -66,14 +66,17 @@ def validate_phone(cls, phone_number_str: str | None) -> str | None:
         if phone_number_str is not None:
             region = "US"
             try:
-                parsed_number = phonenumbers.parse(phone_number_str, region)
-                if phonenumbers.is_valid_number(parsed_number):
-                    formatted_number = phonenumbers.format_number(
-                        parsed_number, phonenumbers.PhoneNumberFormat.E164
-                    )
-                    return formatted_number
-                else:
-                    raise ValueError(f"Invalid phone number. {phone_number_str}")
+                # this is a major hack to deal with the phone numbers entered into
+                # NM_Aquifer without an area code
+                for p in (phone_number_str, f"505{phone_number_str}"):
+                    parsed_number = phonenumbers.parse(p, region)
+                    if phonenumbers.is_valid_number(parsed_number):
+                        formatted_number = phonenumbers.format_number(
+                            parsed_number, phonenumbers.PhoneNumberFormat.E164
+                        )
+                        return formatted_number
+                    else:
+                        raise ValueError(f"Invalid phone number. {phone_number_str}")
             except NumberParseException as e:
                 raise ValueError(f"Invalid phone number. {phone_number_str}")
 

schemas/location.py

@@ -20,16 +20,21 @@
 from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel
 from services.validation.geospatial import validate_wkt_geometry
 
-"""
-TODO
 
-Create common validator classes to be shared amongst create and update schemas.
-Since many fields are optional in the update schemas set check_fields=False in the field_validator.
-"""
+# -------- VALIDATE --------
+
+
+class ValidateLocation(BaseModel):
+    point: str
+
+    @classmethod
+    @field_validator("point", mode="before")
+    def validate_point_is_wkt(cls, wkt):
+        return validate_wkt_geometry(wkt)
 
 
 # -------- CREATE ----------
-class CreateLocation(BaseCreateModel):
+class CreateLocation(BaseCreateModel, ValidateLocation):
     """
     Schema for creating a sample location.
     """
@@ -44,11 +49,6 @@ class CreateLocation(BaseCreateModel):
     coordinate_accuracy: float | None = None
     coordinate_method: str | None = None
 
-    @classmethod
-    @field_validator("point")
-    def validate_point_is_wkt(cls, wkt):
-        return validate_wkt_geometry(wkt)
-
 
 class CreateGroupThing(BaseModel):
     """
@@ -102,7 +102,7 @@ class GroupLocationResponse(BaseResponseModel):
 
 
 # -------- UPDATE ----------
-class UpdateLocation(BaseUpdateModel):
+class UpdateLocation(BaseUpdateModel, ValidateLocation):
     """
     Schema for updating a location.
     """