diff --git a/api/search.py b/api/search.py index a3ff05b73..40b066cbf 100644 --- a/api/search.py +++ b/api/search.py @@ -81,8 +81,9 @@ def _get_thing_results(session: Session, q: str, limit: int) -> list[dict]: select(Thing).where(Thing.thing_type == "spring"), q, vector=vector, limit=limit ) - wells = session.scalars(water_well_query).all() - springs = session.scalars(spring_well_query).all() + # unique needs to be called because of eager loads + wells = session.scalars(water_well_query).unique().all() + springs = session.scalars(spring_well_query).unique().all() def _make_response(group: str, thing: Thing, properties: dict) -> dict: diff --git a/api/thing.py b/api/thing.py index 7e673de08..d0207f00b 100644 --- a/api/thing.py +++ b/api/thing.py @@ -62,7 +62,6 @@ add_well_screen, get_db_things, get_thing_of_a_thing_type_by_id, - get_active_location, ) from services.lexicon_helper import get_terms_by_category @@ -328,7 +327,6 @@ async def get_thing_by_id( Retrieve a thing by ID from the database. """ thing = simple_get_by_id(session, Thing, thing_id) - thing.active_location = get_active_location(session, thing) return thing diff --git a/core/lexicon.json b/core/lexicon.json index 3fc46d82f..daa4b0148 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -9,7 +9,7 @@ {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, {"name": "county", "description": null}, - {"name": "current_use", "description": null}, + {"name": "well_purpose", "description": null}, {"name": "data_quality", "description": null}, {"name": "data_source", "description": null}, {"name": "depth_completion_source", "description": null}, @@ -20,9 +20,8 @@ {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, {"name": "horizontal_datum", "description": null}, - {"name": "value_reason", "description": null}, - {"name": "limit_type", "description": null}, {"name": "groundwater_level_reason", "description": null}, + {"name": "limit_type", "description": null}, {"name": "measurement_method", "description": null}, {"name": "monitoring_status", "description": null}, {"name": "parameter_name", "description": null}, @@ -43,8 +42,8 @@ {"name": "status", "description": null}, {"name": "thing_type", "description": null}, {"name": "unit", "description": null}, - {"name": "vertical_datum", "description": null}, - {"name": "well_purpose", "description": null}], + {"name": "vertical_datum", "description": null} + ], "terms": [ {"categories": ["qc_type"], "term": "Normal", "definition": "The primary environmental sample collected from the well, spring, or soil boring."}, {"categories": ["qc_type"], "term": "Duplicate", "definition": "A second, independent sample collected at the same location, at the same time, and in the same manner as the normal sample. This sample is sent to the primary laboratory."}, @@ -68,7 +67,7 @@ {"categories": ["elevation_method"], "term": "Reported", "definition": "Reported"}, {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, - {"categories": ["elevation_method", "sample_method", "coordinate_method", "current_use", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, + {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, @@ -84,19 +83,21 @@ {"categories": ["coordinate_method"], "term": "Interpolated from DEM", "definition": "Interpolated from DEM"}, {"categories": ["coordinate_method"], "term": "Reported", "definition": "Reported"}, {"categories": ["coordinate_method"], "term": "Transit, theodolite, or other survey method", "definition": "Transit, theodolite, or other survey method"}, - {"categories": ["current_use"], "term": "Open, unequipped well", "definition": "Open, unequipped well"}, - {"categories": ["current_use"], "term": "Commercial", "definition": "Commercial"}, - {"categories": ["current_use"], "term": "Domestic", "definition": "Domestic"}, - {"categories": ["current_use"], "term": "Power generation", "definition": "Power generation"}, - {"categories": ["current_use"], "term": "Irrigation", "definition": "Irrigation"}, - {"categories": ["current_use"], "term": "Livestock", "definition": "Livestock"}, - {"categories": ["current_use"], "term": "Mining", "definition": "Mining"}, - {"categories": ["current_use"], "term": "Industrial", "definition": "Industrial"}, - {"categories": ["current_use"], "term": "Observation", "definition": "Observation"}, - {"categories": ["current_use"], "term": "Public supply", "definition": "Public supply"}, - {"categories": ["current_use"], "term": "Shared domestic", "definition": "Shared domestic"}, - {"categories": ["current_use"], "term": "Institutional", "definition": "Institutional"}, - {"categories": ["current_use"], "term": "Unused", "definition": "Unused"}, + {"categories": ["well_purpose"], "term": "Open, unequipped well", "definition": "Open, unequipped well"}, + {"categories": ["well_purpose"], "term": "Commercial", "definition": "Commercial"}, + {"categories": ["well_purpose"], "term": "Domestic", "definition": "Domestic"}, + {"categories": ["well_purpose"], "term": "Power generation", "definition": "Power generation"}, + {"categories": ["well_purpose"], "term": "Irrigation", "definition": "Irrigation"}, + {"categories": ["well_purpose"], "term": "Livestock", "definition": "Livestock"}, + {"categories": ["well_purpose"], "term": "Mining", "definition": "Mining"}, + {"categories": ["well_purpose"], "term": "Industrial", "definition": "Industrial"}, + {"categories": ["well_purpose"], "term": "Observation", "definition": "Observation"}, + {"categories": ["well_purpose"], "term": "Public supply", "definition": "Public supply"}, + {"categories": ["well_purpose"], "term": "Shared domestic", "definition": "Shared domestic"}, + {"categories": ["well_purpose"], "term": "Institutional", "definition": "Institutional"}, + {"categories": ["well_purpose"], "term": "Unused", "definition": "Unused"}, + {"categories": ["well_purpose"], "term": "Exploration", "definition": "Exploration well"}, + {"categories": ["well_purpose"], "term": "Injection", "definition": "Injection"}, {"categories": ["data_quality"], "term": "Water level accurate to within two hundreths of a foot", "definition": "Good"}, {"categories": ["data_quality"], "term": "Water level accurate to within one foot", "definition": "Fair"}, {"categories": ["data_quality"], "term": "Water level accuracy not to nearest foot or water level not repeatable", "definition": "Poor"}, @@ -355,7 +356,7 @@ {"categories": ["organization"], "term": "Bernalillo County", "definition": "Bernalillo County"}, {"categories": ["organization"], "term": "BLM", "definition": "Bureau of Land Management"}, {"categories": ["organization"], "term": "SFC", "definition": "Santa Fe County"}, - {"categories": ["organization"], "term": "CSF", "definition": "City of Santa Fe"}, + {"categories": ["organization"], "term": "City of Santa Fe", "definition": "City of Santa Fe"}, {"categories": ["organization"], "term": "NESWCD", "definition": "Northeastern Soil & Water Conservation District"}, {"categories": ["organization"], "term": "NMISC", "definition": "New Mexico Interstate Stream Commission"}, {"categories": ["organization"], "term": "PVACD", "definition": "Pecos Valley Artesian Conservancy District"}, @@ -370,6 +371,8 @@ {"categories": ["organization"], "term": "Taos SWCD", "definition": "Taos Soil and Water Conservation District"}, {"categories": ["organization"], "term": "Otero SWCD", "definition": "Otero Soil and Water Conservation District"}, {"categories": ["organization"], "term": "Northeastern SWCD", "definition": "Northeastern Soil and Water Conservation District"}, + {"categories": ["organization"], "term": "CDWR", "definition": "Colorado Division of Water Resources"}, + {"categories": ["organization"], "term": "Pendaries Village", "definition": "Pendaries Village"}, {"categories": ["organization"], "term": "A&T Pump & Well Service, LLC", "definition": "A&T Pump & Well Service, LLC"}, {"categories": ["organization"], "term": "A. G. Wassenaar, Inc", "definition": "A. G. Wassenaar, Inc"}, {"categories": ["organization"], "term": "AMEC", "definition": "AMEC"}, @@ -401,6 +404,10 @@ {"categories": ["organization"], "term": "Thompson Drilling, Inc", "definition": "Thompson Drilling, Inc"}, {"categories": ["organization"], "term": "Witcher & Associates", "definition": "Witcher & Associates"}, {"categories": ["organization"], "term": "Zeigler Geologic Consulting, LLC", "definition": "Zeigler Geologic Consulting, LLC"}, + {"categories": ["organization"], "term": "Sandia Well Service, Inc", "definition": "Sandia Well Service, Inc"}, + {"categories": ["organization"], "term": "San Marcos Association", "definition": "San Marcos Association"}, + {"categories": ["organization"], "term": "URS", "definition": "URS"}, + {"categories": ["organization"], "term": "Vista del Oro", "definition": "Vista del Oro"}, {"categories": ["collection_method"], "term": "manual", "definition": "manual sampling"}, {"categories": ["collection_method"], "term": "continuous", "definition": "continuous sampling"}, {"categories": ["role"], "term": "Owner", "definition": "Owner"}, @@ -416,10 +423,12 @@ {"categories": ["role"], "term": "Technician", "definition": "Technician"}, {"categories": ["role"], "term": "Research Assistant", "definition": "Research Assistant"}, {"categories": ["role"], "term": "Research Scientist", "definition": "Research Scientist"}, + {"categories": ["role"], "term": "Graduate Student", "definition": "Graduate Student"}, {"categories": ["role"], "term": "Operator", "definition": "Operator"}, {"categories": ["role"], "term": "Biologist", "definition": "Biologist"}, {"categories": ["role"], "term": "Lab Manager", "definition": "Lab Manager"}, {"categories": ["role"], "term": "Publications Manager", "definition": "Publications Manager"}, + {"categories": ["role"], "term": "Software Developer", "definition": "Software Developer"}, {"categories": ["email_type", "phone_type", "address_type", "contact_type"], "term": "Primary", "definition": "primary"}, {"categories": ["contact_type"], "term": "Secondary", "definition": "secondary"}, {"categories": ["contact_type"], "term": "Field Event Participant", "definition": "A contact who has participated in a field event"}, @@ -434,10 +443,6 @@ {"categories": ["spring_type"], "term": "Perennial", "definition": "perennial spring"}, {"categories": ["spring_type"], "term": "Thermal", "definition": "thermal spring"}, {"categories": ["spring_type"], "term": "Mineral", "definition": "mineral spring"}, - {"categories": ["well_purpose"], "term": "Exploration", "definition": "Exploration well"}, - {"categories": ["well_purpose"], "term": "Monitoring", "definition": "Monitoring"}, - {"categories": ["well_purpose"], "term": "Production", "definition": "Production"}, - {"categories": ["well_purpose"], "term": "Injection", "definition": "Injection"}, {"categories": ["casing_material"], "term": "PVC", "definition": "Polyvinyl Chloride"}, {"categories": ["casing_material"], "term": "Steel", "definition": "Steel"}, {"categories": ["casing_material"], "term": "Concrete", "definition": "Concrete"}, diff --git a/db/location.py b/db/location.py index 6ef7ce8f5..b1113eaad 100644 --- a/db/location.py +++ b/db/location.py @@ -99,7 +99,9 @@ class LocationThingAssociation(Base, AutoBaseMixin): ) # --- Relationship Definitions --- - location: Mapped["Location"] = relationship(back_populates="thing_associations") + location: Mapped["Location"] = relationship( + back_populates="thing_associations", lazy="joined" + ) thing: Mapped["Thing"] = relationship(back_populates="location_associations") diff --git a/db/thing.py b/db/thing.py index 2ec61dff6..13ce81bbb 100644 --- a/db/thing.py +++ b/db/thing.py @@ -48,6 +48,11 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix __versioned__ = {} # --- Columns --- + nma_pk_welldata: Mapped[str] = mapped_column( + nullable=True, + comment="To audit where the data came from in NM_Aquifer if it was transferred over", + ) + # TODO: should `name` be unique? name: Mapped[str] = mapped_column( nullable=False, @@ -56,7 +61,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix # TODO: what is the purpose of the `description` field? Is it ever used? # description: Mapped[str] = mapped_column(String(500), nullable=True) thing_type: Mapped[str] = lexicon_term( - nullable=True, + nullable=False, comment="A controlled vocabulary field defining the type of infrastructure (e.g., 'Well', 'Spring', 'Stream Gauge').", ) first_visit_date: Mapped[Date] = mapped_column( @@ -119,6 +124,44 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix # One-To-Many: A Thing can be at many locations over time. # If the Thing is deleted, its location history will be deleted. + """ + Developer's notes + + If there are many location associations related to a thing, eagerly loading + the location associations may overburden the API and DB and introduce + performance issues. If this becomes an issue, active/current locations can + be fetched in queries when retrieving things. Be thorough if following this + route as it will need to be included everywhere where a thing record is + needed, such as for GET /thing, GET /thing/{thing_id}, and GET /contact. See + below for an example of a way to retrieve the active/current location in a + query: + + from sqlalchemy.orm import aliased + from sqlalchemy import select, func + + LTA = aliased(LocationThingAssociation) + latest_assoc = ( + select( + LTA.thing_id, + func.max(LTA.effective_start).label("max_start") + ) + .where(LTA.effective_end == None) + .group_by(LTA.thing_id) + .subquery() + ) + + lta_alias = aliased(LocationThingAssociation) + query = ( + select(Thing, Location) + .join(lta_alias, Thing.id == lta_alias.thing_id) + .join(Location, lta_alias.location_id == Location.id) + .join( + latest_assoc, + (latest_assoc.c.thing_id == lta_alias.thing_id) & + (latest_assoc.c.max_start == lta_alias.effective_start) + ) + ) + """ location_associations = relationship( "LocationThingAssociation", back_populates="thing", @@ -126,6 +169,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix cascade="all, delete-orphan", passive_deletes=True, order_by="LocationThingAssociation.effective_start.desc()", + lazy="joined", ) contact_associations = relationship( @@ -199,6 +243,22 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMix ) ) + @property + def current_location(self): + """ + Returns the currently active Location by sorting the effective_start + field. Thing eagerly loads location_association, which eagerly loads + location, which will hopefully prevent N+1 query problems. + """ + current_location = sorted( + self.location_associations, key=lambda x: x.effective_start, reverse=True + ) + return ( + current_location[0].location + if current_location and current_location[0].effective_end is None + else None + ) + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/contact.py b/schemas/contact.py index 22d835240..12c388979 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -66,14 +66,17 @@ def validate_phone(cls, phone_number_str: str | None) -> str | None: if phone_number_str is not None: region = "US" try: - parsed_number = phonenumbers.parse(phone_number_str, region) - if phonenumbers.is_valid_number(parsed_number): - formatted_number = phonenumbers.format_number( - parsed_number, phonenumbers.PhoneNumberFormat.E164 - ) - return formatted_number - else: - raise ValueError(f"Invalid phone number. {phone_number_str}") + # this is a major hack to deal with the phone numbers entered into + # NM_Aquifer without an area code + for p in (phone_number_str, f"505{phone_number_str}"): + parsed_number = phonenumbers.parse(p, region) + if phonenumbers.is_valid_number(parsed_number): + formatted_number = phonenumbers.format_number( + parsed_number, phonenumbers.PhoneNumberFormat.E164 + ) + return formatted_number + else: + raise ValueError(f"Invalid phone number. {phone_number_str}") except NumberParseException as e: raise ValueError(f"Invalid phone number. {phone_number_str}") diff --git a/schemas/location.py b/schemas/location.py index 8f5304588..9340b8cac 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -20,16 +20,21 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from services.validation.geospatial import validate_wkt_geometry -""" -TODO -Create common validator classes to be shared amongst create and update schemas. -Since many fields are optional in the update schemas set check_fields=False in the field_validator. -""" +# -------- VALIDATE -------- + + +class ValidateLocation(BaseModel): + point: str + + @classmethod + @field_validator("point", mode="before") + def validate_point_is_wkt(cls, wkt): + return validate_wkt_geometry(wkt) # -------- CREATE ---------- -class CreateLocation(BaseCreateModel): +class CreateLocation(BaseCreateModel, ValidateLocation): """ Schema for creating a sample location. """ @@ -44,11 +49,6 @@ class CreateLocation(BaseCreateModel): coordinate_accuracy: float | None = None coordinate_method: str | None = None - @classmethod - @field_validator("point") - def validate_point_is_wkt(cls, wkt): - return validate_wkt_geometry(wkt) - class CreateGroupThing(BaseModel): """ @@ -102,7 +102,7 @@ class GroupLocationResponse(BaseResponseModel): # -------- UPDATE ---------- -class UpdateLocation(BaseUpdateModel): +class UpdateLocation(BaseUpdateModel, ValidateLocation): """ Schema for updating a location. """ diff --git a/schemas/thing.py b/schemas/thing.py index fc28b3558..6bf0befc1 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -15,11 +15,46 @@ # =============================================================================== from typing import List -from pydantic import BaseModel, model_validator, PastDate +from pydantic import BaseModel, model_validator, PastDate, Field from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse +# -------- VALIDATE ---------- + + +class ValidateWell(BaseModel): + well_depth: float | None = None # in feet + hole_depth: float | None = None # in feet + well_casing_depth: float | None = None # in feet + + @model_validator(mode="after") + def check_depths(self): + if ( + self.well_depth is not None + and self.hole_depth is not None + and self.well_depth > self.hole_depth + ): + raise ValueError("well depth must be less than than or equal to hole depth") + elif ( + self.well_depth is not None + and self.well_casing_depth is not None + and self.well_casing_depth > self.well_depth + ): + raise ValueError( + "well casing depth must be less than or equal to well depth" + ) + elif ( + self.hole_depth is not None + and self.well_casing_depth is not None + and self.well_casing_depth > self.hole_depth + ): + raise ValueError( + "well casing depth must be less than or equal to hole depth" + ) + + return self + # -------- CREATE ---------- class CreateThingIdLink(BaseModel): @@ -43,21 +78,32 @@ class CreateBaseThing(BaseCreateModel): e.g. POST /thing/water-well, POST /thing/spring determines the thing_type """ - location_id: int | None = None # Optional location ID for the thing + location_id: int | None group_id: int | None = None # Optional group ID for the thing name: str # Name of the thing - first_visit_date: PastDate # Date of NMBGMR's first visit + first_visit_date: PastDate | None = None # Date of NMBGMR's first visit -class CreateWell(CreateBaseThing): +class CreateWell(CreateBaseThing, ValidateWell): """ Schema for creating a well. """ well_purpose: str | None = None - well_depth: float | None = None # in feet - hole_depth: float | None = None # in feet + well_depth: float | None = Field( + default=None, gt=0, description="Well depth in feet" + ) + hole_depth: float | None = Field( + default=None, gt=0, description="Hole depth in feet" + ) well_construction_notes: str | None = None + well_casing_diameter: float | None = Field( + default=None, gt=0, description="Well casing diameter in inches" + ) + well_casing_depth: float | None = Field( + default=None, gt=0, description="Well casing depth in feet" + ) + well_casing_material: str | None = None class CreateSpring(CreateBaseThing): @@ -74,8 +120,8 @@ class CreateWellScreen(BaseCreateModel): """ thing_id: int - screen_depth_bottom: float - screen_depth_top: float + screen_depth_bottom: float = Field(gt=0, description="Screen depth bottom in feet") + screen_depth_top: float = Field(gt=0, description="Screen depth top in feet") screen_type: str | None = None screen_description: str | None = None @@ -93,8 +139,8 @@ def check_depths(self): class BaseThingResponse(BaseResponseModel): name: str thing_type: str - active_location: LocationResponse | None = None - first_visit_date: PastDate | None = None + current_location: LocationResponse | None + first_visit_date: PastDate | None class WellResponse(BaseThingResponse): @@ -102,15 +148,17 @@ class WellResponse(BaseThingResponse): Response schema for well details. """ - # api_id: str | None = None - # ose_pod_id: str | None = None - # usgs_id: str | None = None - well_purpose: str | None = None # e.g., "Production", "Observation", etc. - well_depth: float | None = None # in feet - hole_depth: float | None = None # in feet + well_depth: float | None = None + well_depth_unit: str = "ft" + hole_depth: float | None = None + hole_depth_unit: str = "ft" + well_casing_diameter: float | None = None # in inches + well_casing_diameter_unit: str = "in" + well_casing_depth: float | None = None + well_casing_depth_unit: str = "ft" + well_casing_material: str | None = None well_construction_notes: str | None = None - # Additional fields can be added as needed class SpringResponse(BaseThingResponse): @@ -149,7 +197,9 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse screen_depth_bottom: float + screen_depth_bottom_unit: str = "ft" screen_depth_top: float + screen_depth_top_unit: str = "ft" screen_type: str | None = None screen_description: str | None = None @@ -197,12 +247,15 @@ class UpdateThing(BaseUpdateModel): first_visit_date: PastDate | None = None # Date of NMBGMR's first visit -class UpdateWell(UpdateThing): +class UpdateWell(UpdateThing, ValidateWell): well_purpose: str | None = None well_depth: float | None = None # in feet hole_depth: float | None = None # in feet well_construction_notes: str | None = None + well_casing_diameter: float | None = None # in inches + well_casing_depth: float | None = None # in feet + well_casing_material: str | None = None class UpdateSpring(UpdateThing): diff --git a/services/geospatial_helper.py b/services/geospatial_helper.py index 0cebfe640..356c55634 100644 --- a/services/geospatial_helper.py +++ b/services/geospatial_helper.py @@ -25,6 +25,8 @@ from geoalchemy2.shape import to_shape from shapely.wkt import loads as wkt_loads from sqlalchemy import Select, select +from sqlalchemy.orm import aliased +from sqlalchemy import func def get_thing_features( @@ -42,10 +44,30 @@ def get_thing_features( # elif thing_type == "spring": # selection_args.append(SpringThing) + # Subquery: get the latest association for each thing (optionally only active) + lta_alias = aliased(LocationThingAssociation) + + latest_assoc = ( + select( + LocationThingAssociation.thing_id, + func.max(LocationThingAssociation.effective_start).label("max_start"), + ) + .where( + LocationThingAssociation.effective_end == None + ) # Only active, remove if you want most recent regardless of end + .group_by(LocationThingAssociation.thing_id) + .subquery() + ) + sql = ( select(Thing, ST_AsGeoJSON(Location.point).label("geojson"), Location.elevation) - .join(LocationThingAssociation, Thing.id == LocationThingAssociation.thing_id) - .join(Location, LocationThingAssociation.location_id == Location.id) + .join(lta_alias, Thing.id == lta_alias.thing_id) + .join(Location, lta_alias.location_id == Location.id) + .join( + latest_assoc, + (latest_assoc.c.thing_id == lta_alias.thing_id) + & (latest_assoc.c.max_start == lta_alias.effective_start), + ) ) if thing_type: @@ -65,7 +87,8 @@ def get_thing_features( else: sql = sql.where(Group.id == group) - return session.execute(sql).all() + # unique needs to be invoked to prevent duplicates from eager loading + return session.execute(sql).unique().all() def create_shapefile(things: list, filename: str = "things.shp") -> None: diff --git a/services/thing_helper.py b/services/thing_helper.py index 5d88cd564..bddabdbed 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -38,21 +38,6 @@ def wkb_to_geojson(wkb_element): return mapping(geom) -def get_active_location(session: Session, thing: Thing) -> Location | None: - """ - The following SQL query retrieves the active location associated with by - assuming that the latest effective_start is the active location. - """ - sql = ( - select(Location) - .join(LocationThingAssociation) - .where(LocationThingAssociation.thing_id == thing.id) - .order_by(LocationThingAssociation.effective_start.desc()) - ) - active_location = session.execute(sql).scalars().one_or_none() - return active_location - - def get_db_things( filter_, order, @@ -63,47 +48,39 @@ def get_db_things( within: str = None, ) -> list: - latest_assoc = ( - select( - LocationThingAssociation.thing_id, - func.max(LocationThingAssociation.effective_start).label("max_start"), - ) - .group_by(LocationThingAssociation.thing_id) - .subquery() - ) - if query: - sql = select(Thing, Location).where(make_query(Thing, query)) + sql = select(Thing).where(make_query(Thing, query)) else: - sql = select(Thing, Location) - - lta_alias = aliased(LocationThingAssociation) - sql = ( - sql.join(lta_alias, Thing.id == lta_alias.thing_id) - .join(Location, lta_alias.location_id == Location.id) - .join( - latest_assoc, - (latest_assoc.c.thing_id == lta_alias.thing_id) - & (latest_assoc.c.max_start == lta_alias.effective_start), - ) - ) + sql = select(Thing) if thing_type: sql = sql.where(Thing.thing_type == thing_type) if within: + latest_assoc = ( + select( + LocationThingAssociation.thing_id, + func.max(LocationThingAssociation.effective_start).label("max_start"), + ) + .group_by(LocationThingAssociation.thing_id) + .subquery() + ) + + lta_alias = aliased(LocationThingAssociation) + sql = ( + sql.join(lta_alias, Thing.id == lta_alias.thing_id) + .join(Location, lta_alias.location_id == Location.id) + .join( + latest_assoc, + (latest_assoc.c.thing_id == lta_alias.thing_id) + & (latest_assoc.c.max_start == lta_alias.effective_start), + ) + ) sql = make_within_wkt(sql, within) sql = order_sort_filter(sql, Thing, sort, order, filter_) - def transformer(records): - def make_new_record(thing, location): - thing.active_location = location - return thing - - return [make_new_record(*record) for record in records] - - return paginate(query=sql, conn=session, transformer=transformer) + return paginate(query=sql, conn=session) def get_thing_type_from_request(request: Request) -> str: @@ -141,7 +118,6 @@ def get_thing_of_a_thing_type_by_id(session: Session, request: Request, thing_id verify_thing_type_correspondence(thing, request) - thing.active_location = get_active_location(session, thing) return thing @@ -180,6 +156,7 @@ def add_thing( session.add(assoc) if location_id is not None: + # TODO: how do we want to handle effective_start? is it the date it gets entered? assoc = LocationThingAssociation() audit_add(user, assoc) assoc.location_id = location_id @@ -187,11 +164,11 @@ def add_thing( session.add(assoc) session.commit() + session.refresh(thing) except Exception as e: session.rollback() raise e - thing.active_location = get_active_location(session, thing) return thing @@ -237,7 +214,6 @@ def patch_thing( verify_thing_type_correspondence(thing, request) thing = model_patcher(session, Thing, thing_id, payload, user) - thing.active_location = get_active_location(session, thing) return thing diff --git a/services/util.py b/services/util.py index 31a260970..cb3d8826c 100644 --- a/services/util.py +++ b/services/util.py @@ -40,7 +40,12 @@ def get_tiger_data( "outFields": outfields, "returnGeometry": "false", } - resp = httpx.get(url, params=params, timeout=30) + try: + resp = httpx.get(url, params=params, timeout=30) + except Exception as e: + print(f"Error getting TIGER data for POINT ({lon} {lat}) {e}") + return None + data = resp.json() if not data.get("features"): return None @@ -76,11 +81,11 @@ def get_quad_name_from_point(lon: float, lat: float) -> str: "outFields": "CELL_NAME,CELL_MAPCODE", "returnGeometry": "false", } - - resp = httpx.get(url, params=params, timeout=30) try: + resp = httpx.get(url, params=params, timeout=30) data = resp.json() - except json.decoder.JSONDecodeError: + except Exception as e: + print(f"Error getting quad name for POINT ({lon} {lat}) {e}") return None if data["features"]: diff --git a/tests/conftest.py b/tests/conftest.py index 6373148f5..61fe086e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,10 +55,13 @@ def water_well_thing(location): first_visit_date="2023-03-03", thing_type="water well", release_status="draft", - well_purpose="Production", + well_purpose="Domestic", well_depth=10, hole_depth=10, well_construction_notes="Test well construction notes", + well_casing_diameter=5.0, + well_casing_depth=10.0, + well_casing_material="PVC", ) session.add(water_well) session.commit() @@ -70,6 +73,8 @@ def water_well_thing(location): assoc.effective_start = "2025-02-01T00:00:00Z" session.add(assoc) session.commit() + session.refresh(water_well) + session.refresh(assoc) yield water_well session.delete(water_well) session.delete(assoc) diff --git a/tests/test_location.py b/tests/test_location.py index 804db5632..628c1c352 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -138,9 +138,6 @@ def test_get_locations(location): response = client.get("/location") assert response.status_code == 200 data = response.json() - from pprint import pprint - - pprint(data, indent=2) assert data["total"] == 1 assert data["items"][0]["id"] == location.id assert data["items"][0]["created_at"] == location.created_at.isoformat().replace( diff --git a/tests/test_observation.py b/tests/test_observation.py index 22717744b..853fe9a05 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -62,8 +62,6 @@ def test_add_water_chemistry_observation(water_chemistry_sample, sensor): "sample_id": water_chemistry_sample.id, "sensor_id": sensor.id, "parameter_id": pH_parameter_id, - "value_reason": "Observed value not affected", - "observed_property": "pH", } response = client.post("/observation/water-chemistry", json=payload) data = response.json() diff --git a/tests/test_search.py b/tests/test_search.py index 6109bdf72..acf45c29e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -54,7 +54,6 @@ def test_search_api2(): response = client.get("/search", params={"q": "riochama"}) assert response.status_code == 200 data = response.json() - print(data) assert isinstance(data, list) assert len(data) == 1 assert data[0]["label"] == "riochama.png" diff --git a/tests/test_thing.py b/tests/test_thing.py index b5470458d..d2038befc 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -27,6 +27,7 @@ amp_viewer_function, ) from schemas.location import LocationResponse +from schemas.thing import ValidateWell @pytest.fixture(scope="module", autouse=True) @@ -51,6 +52,32 @@ def override_authentication_dependency_fixture(): app.dependency_overrides = {} +# VALIDATE tests =============================================================== + + +def test_validate_well_depth_hole_depth(): + with pytest.raises( + ValueError, match="well depth must be less than than or equal to hole depth" + ): + ValidateWell(well_depth=100.0, hole_depth=90.0) + + +def test_validate_well_depth_casing_depth(): + with pytest.raises( + ValueError, + match="well casing depth must be less than or equal to well depth", + ): + ValidateWell(well_depth=100.0, well_casing_depth=110.0) + + +def test_validate_hole_depth_casing_depth(): + with pytest.raises( + ValueError, + match="well casing depth must be less than or equal to hole depth", + ): + ValidateWell(hole_depth=100.0, well_casing_depth=110.0) + + # POST tests =================================================================== @@ -61,10 +88,13 @@ def test_add_water_well(location, group): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Monitoring", + "well_purpose": "Domestic", "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", + "well_casing_diameter": 5.0, + "well_casing_depth": 10.0, + "well_casing_material": "PVC", } response = client.post("/thing/water-well", json=payload) @@ -78,14 +108,21 @@ def test_add_water_well(location, group): assert data["thing_type"] == "water well" assert data["well_purpose"] == payload["well_purpose"] assert data["hole_depth"] == payload["hole_depth"] + assert data["hole_depth_unit"] == "ft" assert data["well_depth"] == payload["well_depth"] + assert data["well_depth_unit"] == "ft" assert data["well_construction_notes"] == payload["well_construction_notes"] + assert data["well_casing_diameter"] == payload["well_casing_diameter"] + assert data["well_casing_diameter_unit"] == "in" + assert data["well_casing_depth"] == payload["well_casing_depth"] + assert data["well_casing_depth_unit"] == "ft" + assert data["well_casing_material"] == payload["well_casing_material"] expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location cleanup_post_test(Thing, data["id"]) @@ -98,10 +135,13 @@ def test_add_water_well_409_bad_group_id(location): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Monitoring", + "well_purpose": "Domestic", "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", + "well_casing_diameter": 5.0, + "well_casing_depth": 10.0, + "well_casing_material": "PVC", } response = client.post("/thing/water-well", json=payload) @@ -121,7 +161,7 @@ def test_add_water_well_409_bad_location_id(group): "release_status": "draft", "name": "Test Well", "first_visit_date": "2023-01-01", - "well_purpose": "Monitoring", + "well_purpose": "Domestic", "well_depth": 100.0, "hole_depth": 110, "well_construction_notes": "this is a test of notes", @@ -159,7 +199,7 @@ def test_add_spring(location, group): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location cleanup_post_test(Thing, data["id"]) @@ -345,16 +385,30 @@ def test_get_water_wells(water_well_thing, location): assert data["items"][0]["release_status"] == water_well_thing.release_status assert data["items"][0]["well_purpose"] == water_well_thing.well_purpose assert data["items"][0]["well_depth"] == water_well_thing.well_depth + assert data["items"][0]["well_depth_unit"] == "ft" assert data["items"][0]["hole_depth"] == water_well_thing.hole_depth + assert data["items"][0]["hole_depth_unit"] == "ft" assert ( data["items"][0]["well_construction_notes"] == water_well_thing.well_construction_notes ) + assert ( + data["items"][0]["well_casing_diameter"] + == water_well_thing.well_casing_diameter + ) + assert data["items"][0]["well_casing_diameter_unit"] == "in" + assert data["items"][0]["well_casing_depth"] == water_well_thing.well_casing_depth + assert data["items"][0]["well_casing_depth_unit"] == "ft" + assert ( + data["items"][0]["well_casing_material"] + == water_well_thing.well_casing_material + ) + expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["items"][0]["active_location"] == expected_location + assert data["items"][0]["current_location"] == expected_location def test_get_water_well_by_id(water_well_thing, location): @@ -371,13 +425,21 @@ def test_get_water_well_by_id(water_well_thing, location): assert data["release_status"] == water_well_thing.release_status assert data["well_purpose"] == water_well_thing.well_purpose assert data["well_depth"] == water_well_thing.well_depth + assert data["well_depth_unit"] == "ft" assert data["hole_depth"] == water_well_thing.hole_depth + assert data["hole_depth_unit"] == "ft" assert data["well_construction_notes"] == water_well_thing.well_construction_notes + assert data["well_casing_diameter"] == water_well_thing.well_casing_diameter + assert data["well_casing_diameter_unit"] == "in" + assert data["well_casing_depth"] == water_well_thing.well_casing_depth + assert data["well_casing_depth_unit"] == "ft" + assert data["well_casing_material"] == water_well_thing.well_casing_material + expected_location = LocationResponse.model_validate(location).model_dump() expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location def test_get_water_well_by_id_404_not_found(water_well_thing): @@ -423,7 +485,7 @@ def test_get_springs(spring_thing, location): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["items"][0]["active_location"] == expected_location + assert data["items"][0]["current_location"] == expected_location def test_get_spring_by_id(spring_thing, location): @@ -443,7 +505,7 @@ def test_get_spring_by_id(spring_thing, location): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location def test_get_spring_by_id_404_not_found(spring_thing): @@ -623,45 +685,6 @@ def test_get_things(water_well_thing, spring_thing, location): data = response.json() assert data["total"] == 2 - assert data["items"][0]["id"] == water_well_thing.id - assert data["items"][0][ - "created_at" - ] == water_well_thing.created_at.isoformat().replace("+00:00", "Z") - assert data["items"][0]["name"] == water_well_thing.name - assert ( - data["items"][0]["first_visit_date"] - == water_well_thing.first_visit_date.isoformat() - ) - assert data["items"][0]["thing_type"] == water_well_thing.thing_type - assert data["items"][0]["release_status"] == water_well_thing.release_status - assert data["items"][0]["well_purpose"] == water_well_thing.well_purpose - assert data["items"][0]["well_depth"] == water_well_thing.well_depth - assert data["items"][0]["hole_depth"] == water_well_thing.hole_depth - assert ( - data["items"][0]["well_construction_notes"] - == water_well_thing.well_construction_notes - ) - assert data["items"][0]["spring_type"] is None - assert data["items"][0]["active_location"] == expected_location - - assert data["items"][1]["id"] == spring_thing.id - assert data["items"][1][ - "created_at" - ] == spring_thing.created_at.isoformat().replace("+00:00", "Z") - assert data["items"][1]["name"] == spring_thing.name - assert ( - data["items"][1]["first_visit_date"] - == spring_thing.first_visit_date.isoformat() - ) - assert data["items"][1]["thing_type"] == spring_thing.thing_type - assert data["items"][1]["release_status"] == spring_thing.release_status - assert data["items"][1]["spring_type"] == spring_thing.spring_type - assert data["items"][1]["well_purpose"] is None - assert data["items"][1]["well_depth"] is None - assert data["items"][1]["hole_depth"] is None - assert data["items"][1]["well_construction_notes"] is None - assert data["items"][1]["active_location"] == expected_location - def test_get_thing_by_id(water_well_thing, location): response = client.get(f"/thing/{water_well_thing.id}") @@ -687,7 +710,7 @@ def test_get_thing_by_id(water_well_thing, location): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location def test_get_thing_by_id_404_not_found(water_well_thing): @@ -775,7 +798,7 @@ def test_patch_water_well(water_well_thing, location): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location cleanup_patch_test(Thing, payload, water_well_thing) @@ -838,7 +861,7 @@ def test_patch_spring(spring_thing, location): expected_location["created_at"] = ( expected_location["created_at"].isoformat().replace("+00:00", "Z") ) - assert data["active_location"] == expected_location + assert data["current_location"] == expected_location cleanup_patch_test(Thing, payload, spring_thing) diff --git a/transfers/data/measured_by_mapper.json b/transfers/data/measured_by_mapper.json index 2c3e113c6..9d0d2b433 100644 --- a/transfers/data/measured_by_mapper.json +++ b/transfers/data/measured_by_mapper.json @@ -38,10 +38,11 @@ "Minton.": [null, "Minton Engineers", "Organization"], "MJ Darr.": [null, "MJDarrconsult, Inc", "Organization"], "MJ Darr consultants": [null, "MJDarrconsult, Inc", "Organization"], - "NESWCD": [null, "North East Soil and Water Conservation District", "Organization"], + "NESWCD": [null, "Northeastern Soil and Water Conservation District", "Organization"], "OSE, ST": [[null, "NMOSE", "Organization"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], "PVACD person": [null, "PVACD", "Organization"], - "SFC/Frost": ["Frost", "SFC", "Unknown"], + "Sandia Drillers": [null, "Sandia Well Service, Inc", "Organization"], + "SFC/Frost": ["Jack Frost", "NMOSE", "Hydrologist"], "SPCE HOA": [null, "San Pedro Creek Estates HOA", "Organization"], "Statewide Drilling": [null, "Statewide Drilling", "Organization"], "Tec Drilling": [null, "Tec Drilling Limited", "Organization"], @@ -58,6 +59,7 @@ "Taos SWCD": [null, "Taos SWCD", "Organization"], "TWDB": [null, "TWDB", "Organization"], "USFS": [null, "USFS", "Organization"], + "AG": [null, "USGS", "Organization"], "USGS": [null, "USGS", "Organization"], "USGS WRD": [null, "USGS WRD", "Organization"], @@ -66,17 +68,18 @@ "GGI-OSE": [null, "Glorieta Geoscience, Inc", "Organization"], "Glorieta Geoscienc": [null, "Glorieta Geoscience, Inc", "Organization"], "Glorieta Geoscience": [null, "Glorieta Geoscience, Inc", "Organization"], + "Lazarus": ["Jay Lazarus", "Glorieta Geoscience, Inc", "Hydrogeologist"], - "Driller": [null, "*MEASURING_AGENCY*", "Driller"], - "Tribble, Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Cruz", "Daniel B. Stephens & Associates", "Unknown"]], - "Tribble/Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Tribble, Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Tribble/Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Cruz-Tribble": [["Roy Cruz", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"]], "DBSA": [null, "Daniel B. Stephens & Associates, Inc", "Organization"], "DBStephens & Assoc": [null, "Daniel B. Stephens & Associates, Inc", "Organization"], - "City of Santa Fe": [null, "CSF", "Organization"], - "City of Santa Fe": [null, "CSF", "Organization"], - "City of Santa Fe": [null, "CSF", "Organization"], - "CityofSantaFe": [null, "CSF", "Organization"], + "City of Santa Fe": [null, "City of Santa Fe", "Organization"], + "City of Santa Fe": [null, "City of Santa Fe", "Organization"], + "City of Santa Fe": [null, "City of Santa Fe", "Organization"], + "CityofSantaFe": [null, "City of Santa Fe", "Organization"], "John Shomaker": [null, "John Shomaker & Associates, Inc", "Organization"], "John Shomaker & Associates, Inc": [null, "John Shomaker & Associates, Inc", "Organization"], @@ -90,7 +93,6 @@ "Fleming - Shomaker": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], "Fleming/Shomaker": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], "Shomaker - Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], - "Shomaker - Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], "Shomaker/Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], "Kuck": [null, "Kuckleman Pump Service", "Organization"], @@ -101,36 +103,102 @@ "OSE; Doug Rappuhn": ["Doug Rappuhn", "NMOSE", "Hydrologist"], "D.Rappuhn OSE": ["Doug Rappuhn", "NMOSE", "Hydrologist"], + "DR": ["Doug Rappuhn", "NMOSE", "Hydrologist"], + "DR, ST": [["Doug Rappuhn", "NMOSE", "Hydrologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], "Rodgers & Co": [null, "Rodgers & Company, Inc", "Organization"], "Rodgers & Co.": [null, "Rodgers & Company, Inc", "Organization"], + "Mike Rodgers": ["Mike Rodgers", "Rodgers & Company, Inc", "Driller"], "Sandia National labs": [null, "SNL", "Organization"], "SNL": [null, "SNL", "Organization"], "Santa Fe County": [null, "SFC", "Organization"], "SFCounty LF staff": [null, "SFC", "Organization"], + "SdC": [null, "SFC", "Organization"], + + "SM&Assoc": [null, "San Marcos Association", "Organization"], + "SMA": [null, "San Marcos Association", "Organization"], + + "URS": [null, "URS", "Organization"], + + "Duke Engring": ["Duke Engring", "Unknown", "Unknown"], + "Gamma log unit": ["Gamma log unit", "Unknown", "Unknown"], + "Hydrogeologic serv": ["Hydrogeologic serv", "Unknown", "Unknown"], + "MWB Consultants": ["MWB Consultants", "Unknown", "Unknown"], + "SPRI": ["SPRI", "Unknown", "Unknown"], + "UTM": ["UTM", "Unknown", "Unknown"], + "VeneKlasen": ["VeneKlasen", "Unknown", "Unknown"], + "?": ["?", "Unknown", "Unknown"], + "Conultant": ["Conultant", "Unknown", "Unknown"], + "Consulting Pro.": ["Consulting Pro.", "Unknown", "Unknown"], + "Pump company": ["Pump company", "Unknown", "Unknown"], + "PumpService": ["PumpService", "Unknown", "Unknown"], + "REPORTED": ["REPORTED", "Unknown", "Unknown"], + "Theis report": ["Theis report", "Unknown", "Unknown"], + "Unknown": ["Unknown", "Unknown", "Unknown"], + "Unknown; reported": ["Unknown; reported", "Unknown", "Unknown"], + "Water operator": ["Water operator", "Unknown", "Operator"], + "Well operator": ["Well operator", "Unknown", "Operator"], + "WWTP": ["WWTP", "Unknown", "Unknown"], + "WWTP personnel": ["WWTP personnel", "Unknown", "Unknown"], + "AL": ["Angela Lucero", "NMBGMR", "Hydrologist"], "AL, GR": [["Angela Lucero", "NMBGMR", "Hydrologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], "AL, SC": [["Angela Lucero", "NMBGMR", "Hydrologist"], ["Scott Christenson", "NMBGMR", "Technician"]], - "Amy Kronson": [["Amy Kronson", "NMBGMR", "Technician"]], - "Andrew Matejunas": [["Andrew Matejunas", "NMBGMR", "Research Assistant"]], - "Andy Manning": [["Andy Manning", "USGS", "Hydrogeologist"]], + "Amy Kronson": ["Amy Kronson", "NMBGMR", "Technician"], + "Andrew Matejunas": ["Andrew Matejunas", "NMSU", "Research Assistant"], + "Andy Manning": ["Andy Manning", "USGS", "Hydrogeologist"], + "Anders Lundahl": ["Anders Lundahl", "NMISC", "Specialist"], + "Anders Lundalh": ["Anders Lundahl", "NMISC", "Specialist"], + "Anthony Chavez": ["Anthony Chavez", "Bernalillo County", "Unknown"], + "BEI": ["BEI", "Unknown", "Unknown"], + "BG/RF": [["Brigitte Felix", "NMBGMR", "Publications Manager"], ["RG", "NMBGMR", "Unknown"]], + "Borchert": ["Claudia Borchert", "CSF", "Hydrogeologist"], + "Bob Borton": ["Bob Borton", "NMOSE", "Geologist"], + "Borton": ["Bob Borton", "NMOSE", "Geologist"], + "RL Borton": ["Bob Borton", "NMOSE", "Geologist"], + "Borton & Cooper": [["Bob Borton", "NMOSE", "Geologist"], ["Dennis Cooper", "NMOSE", "Engineer"]], + "Dennis Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], + "Dennis R. Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], "ce": ["Cathy Eisen", "NMBGMR", "Hydrogeologist"], "CE": ["Cathy Eisen", "NMBGMR", "Hydrogeologist"], "CE PJ": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], "CE, GR": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "CE TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "CE, TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "Chaves/Cruz": [["Chavez", "Unknown", "Unknown"], ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Chavez/Cruz": [["Chavez", "Unknown", "Unknown"], ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Cruz": ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"], + "Cruz/Frost": [["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"], ["Jack Frost", "NMOSE", "Hydrologist"]], "CM": ["Cris Morton", "NMBGMR", "Hydrogeologist"], + "CM, AK": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Ashish Kodam", "NMBGMR", "Software Developer"]], "CM, EM": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], "CM, LS, KP": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], - "CM, LS, KrPe": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Kirsten Pearthree", "NMBGMR", "Research Scientist"]], + "CM, LS, KrPe": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Kristen Pearthree", "NMBGMR", "Research Scientist"]], "CM, SC": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], - "CM, TK": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "CM, TK": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "Coons": ["Coons", "Unknown", "Unknown"], + "Cooper": ["Cooper", "Unknown", "Unknown"], + "Crocker": ["Crocker", "Unknown", "Unknown"], + "D.Bird": ["D.Bird", "Unknown", "Unknown"], + "D.D": ["D.D", "Unknown", "Unknown"], + "D.Duncan": ["D.Duncan", "Unknown", "Unknown"], + "Dames & Moore": [["Dames", "Unknown", "Unknown"], ["Moore", "Unknown", "Unknown"]], + "Dames/Moore": [["Dames", "Unknown", "Unknown"], ["Moore", "Unknown", "Unknown"]], "Dan McGregor": ["Dan McGregor", "Bernalillo County", "Hydrogeologist"], - "CE TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], - "CE, TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "Dave Snider": ["Dave Snider", "Pendaries Village", "Operator"], + "Jenkins": ["David N. Jenkins", "Unknown", "Unknown"], + "David N Jenkins": ["David N. Jenkins", "Unknown", "Unknown"], + "David N. Jenkins": ["David N. Jenkins", "Unknown", "Unknown"], + "DC": ["Dan Cadol", "NMT", "Hydrogeologist"], + "Decker": ["Decker", "Unknown", "Unknown"], + "DL, TK": [["Dan Lavery", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "Duncan": ["Duncan", "Unknown", "Unknown"], + "EA": ["EA", "NMBGMR", "Unknown"], + "EA/HB": [["EA", "NMBGMR", "Unknown"], ["HB", "NMBGMR", "Unknown"]], "EM": ["Ethan Mamer", "NMBGMR", "Hydrogeologist"], "EM, AL": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"],["Angela Lucero", "NMBGMR", "Hydrologist"]], "EM, CM": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], @@ -140,79 +208,98 @@ "EM, LS": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"]], "EM, MF": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Marissa Fichera", "NMBGMR", "Hydrogeologist"]], "EM, SMC": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]], - "EM, TK": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "EM, TK": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], "EM, TN": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], - "Gary Goss": ["Gary Goss", "*MEASURING_AGENCY*", "Hydrogeologist"], - "Anders Lundahl": ["Anders Lundahl", "*MEASURING_AGENCY*", "Specialist"], - "Anders Lundalh": ["Anders Lundahl", "*MEASURING_AGENCY*", "Specialist"], - "Bob Borton": ["Bob Borton", "NMBGMR", "Geologist"], - "Borton": ["Bob Borton", "NMBGMR", "Geologist"], - "Dennis Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], - "Dennis R. Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], + "Frost": ["Jack Frost", "NMOSE", "Hydrologist"], + "G. Boylan": ["G. Boylan", "Unknown", "Unknown"], + "Garcia": ["Garcia", "USGS", "Unknown"], + "Garcia/Johnson": [["Garcia", "USGS", "Unknown"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], + "Gary Goss": ["Gary Goss", null, "Hydrogeologist"], "GCR": ["Geoff Rawling", "NMBGMR", "Hydrogeologist"], + "GCR/PW": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["PW", "NMBGMR", "Unknown"]], + "GR/PW": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["PW", "NMBGMR", "Unknown"]], "GCR/ST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], "GCRST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], "GR/ST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], "Rawling/Wagner": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], - "GCR/ST/JM": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Joe Marcoline", "NMBGMR", "Unknown"]], + "GCR/ST/JM": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Joe Marcoline", "NMED", "Unknown"]], "GR": ["Geoff Rawling", "NMBGMR", "Hydrogeologist"], "GR, AL": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], "GR, CE": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], "GR, SC": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], - "GR, TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], - "GR/TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "GLR, SC": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "GLC, SK, SC": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Shari Kelley", "NMBGMR", "Geologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "GR, TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "GR/TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], "GR/LL": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], + "GR, MM": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Mark Mansell", "NMBGMR", "Technician"]], + "GR/RG": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["RG", "NMBGMR", "Unknown"]], + "HB": ["HB", "Unknown", "Unknown"], + "Heaton": ["Heaton", "Unknown", "Unknown"], + "Horner-Crocker": [["Horner", "Unknown", "Unknown"], ["Crocker", "Unknown", "Unknown"]], + "HR": ["HR", "Unknown", "Unknown"], + "J Evans": ["J Evans", "Unknown", "Unknown"], "JB": ["Joseph Beman", "NMBGMR", "Technician"], "JEB": ["Joseph Beman", "NMBGMR", "Technician"], + "Corbin": ["Jim Corbin", "Corbin Consulting, Inc", "Unknown"], "Jim Corbin": ["Jim Corbin", "Corbin Consulting, Inc", "Unknown"], "JM": ["Joe Marcoline", "NMED", "Unknown"], "Joe Marcoline": ["Joe Marcoline", "NMED", "Unknown"], "Johnson": ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], + "Johnson/Cruz": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Roy Cruz", "Daniel B. Stephens & Associates", "Unknown"]], "Johnson - Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], "Johnson-Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], "Johnson/Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], - "Johnson-Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], - "Johnson/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], - "Lyman": [["John Lyman", "Unknown", "Unknown"]], - "PJ/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], - "Jose Varela Lopez": ["Jose Varela Lopez", "Puerta del Canon Ranch", "Operator"], - "K. McLain": ["Katie McLain", "NMBGMR", "Hydrogeologist"], - "K. McLain, M. Hein": [["Katie McLain", "NMBGMR", "Hydrogeologist"], ["Marina Hein", "NMT", "Biologist"]], + "Johnson-Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Johnnie Lyman", "NMT", "Graduate Student"]], + "Johnson/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Johnnie Lyman", "NMT", "Graduate Student"]], + "Johnson/Robbins": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Robbins", "Unknown", "Unknown"]], + "Lyman": [["Johnnie Lyman", "NMT", "Graduate Student"]], + "PJ/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Johnnie Lyman", "NMT", "Graduate Student"]], + "Jose Varela Lopez": ["Jose Varela Lopez", "Puerta del Canon Ranch", "Owner"], + "K. McLain": ["Katie McLain", "NMT", "Graduate Student"], + "K. McLain, M. Hein": [["Katie McLain", "NMT", "Graduate Student"], ["Marina Hein", "NMT", "Graduate Student"]], "K.Summers": ["Kelly Summers", "NMBGMR", "Hydrologist"], "WK Summers": ["Kelly Summers", "NMBGMR", "Hydrologist"], "Kelsey McNamara": ["Kelsey McNamara", "NMBGMR", "Geologist"], "Kitty": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], "Kitty Pokorny": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], + "Kilmer/Jenkins": [["Kilmer", "Unknown", "Unknown"], ["David N. Jenkins", "Unknown", "Unknown"]], "KP": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], "KP, MF": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Marissa Fichera", "NMBGMR", "Hydrogeologist"]], + "KP, MR": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["MR", "NMBGMR", "Unknown"]], + "KP, MT": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["MT", "Unknown", "Unknown"]], "KP, ST": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], - "KP, TK": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "KP, TK": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], "KR": ["Kylian Robinson", "NMED", "Hydrogeologist"], "Kylian Robinson": ["Kylian Robinson", "NMED", "Hydrogeologist"], - "Leroy Romero": ["Leroy Romero", "Los Golondrinas", "Unknown"], + "Leroy Romero": ["Leroy Romero", "Los Golondrinas", "Owner"], "LL, TN": [["Lewis Land", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], "LS": ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], - "LS, TK": [["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], - "M. Hein": ["Marina Hein", "NMT", "Biologist"], - "MH, KM": [["Marina Hein", "NMT", "Biologist"], ["Katie McLain", "NMBGMR", "Hydrogeologist"]], + "LS, TK": [["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], + "M. Hein": ["Marina Hein", "NMT", "Graduate Student"], + "MH, KM": [["Marina Hein", "NMT", "Graduate Student"], ["Katie McLain", "NMT", "Graduate Student"]], + "Mourant": ["Mourant", "Unknown", "Unknown"], + "Myers report": ["Robert Myers", "USGS", "Hydrogeologist"], + "NT": ["Nathan Myers", "USGS", "Hydrogeologist"], "Patricia Rosacker": ["Patricia Rosacker", "CSF", "Lab Manager"], "PB, PJ": [["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], - "PB, PJ, TK": [["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "PB, PJ, TK": [["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], "Pepin": ["Jeff Pepin", "USGS", "Hydrologist"], "Pepin/Kelley": [["Jeff Pepin", "USGS", "Hydrologist"], ["Shari Kelley", "NMBGMR", "Geologist"]], "Mark Person": ["Mark Person", "NMT", "Hydrologist"], "PJ": ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], "PJ PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], "PJ, PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], - "PJ TK PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], - "PJ, TK, PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], - "RL Borton": [["R. L. Borton", "NMOSE", "Unknown"]], + "PJ TK PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"], ["Paul Bauer", "NMBGMR", "Geologist"]], + "PJ, TK, PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"], ["Paul Bauer", "NMBGMR", "Geologist"]], "RP": ["RP", "NMOSE", "Unknown"], + "Rankin": ["Rankin", "Unknown", "Unknown"], + "Robbins": ["Robbins", "Unknown", "Unknown"], "Sara Chudnoff": ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], "SMC": ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], "SMC, EM": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], "SMC, SC": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Hydrogeologist"]], - "SMC, TK": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "SMC, TK": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]], "SC": ["Scott Christenson", "NMBGMR", "Technician"], "SCC": ["Scott Christenson", "NMBGMR", "Technician"], "SD": ["Scott Christenson", "NMBGMR", "Technician"], @@ -222,13 +309,17 @@ "SC, EM": [["Scott Christenson", "NMBGMR", "Technician"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], "SC, GR": [["Scott Christenson", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], "SC, KP": [["Scott Christenson", "NMBGMR", "Technician"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], + "SC, MA": [["Scott Christenson", "NMBGMR", "Technician"], ["MA", "Unknown", "Unknown"]], + "SC, MR": [["Scott Christenson", "NMBGMR", "Technician"], ["MR", "NMBGMR", "Unknown"]], "SC, SMC": [["Scott Christenson", "NMBGMR", "Technician"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]], "SC, ST": [["Scott Christenson", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], - "SC, TK": [["Scott Christenson", "NMBGMR", "Technician"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "SC, TK": [["Scott Christenson", "NMBGMR", "Technician"], ["Trevor Kludt", "NMBGMR", "Technician"]], "SC, TN": [["Scott Christenson", "NMBGMR", "Technician"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], "SK": ["Shari Kelley", "NMBGMR", "Geologist"], "SK, SC, GR": [["Shari Kelley", "NMBGMR", "Geologist"], ["Scott Christenson", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], "SR": ["Stephanie Roussel", "USGS", "Hydrologist"], + "Spiegel": ["Zane Spiegel", "USGS", "Hydrogeologist"], + "Spiegel & Baldwin": [["Zane Spiegel", "USGS", "Hydrogeologist"], ["Brewster Baldwin", "USGS", "Hydrogeologist"]], "Stephanie Roussel": ["Stephanie Roussel", "USGS", "Hydrologist"], "SR, EM": [["Stephanie Roussel", "USGS", "Hydrologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], " Wagner": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], @@ -251,29 +342,33 @@ "Wagner/Rawling": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], "ST/JW": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Jim Witcher", "Witcher & Associates", "Hydrogeologist"]], "ST/LL": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], - "TK": ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], - "Trevor Kludt": ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], - "TK BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], - "TK, BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], - "TK/BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], - "tk cm": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], - "TK, CM": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], - "TK KR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], - "TK, KR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], - "TK, AL": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], - "TK, CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], - "TK,CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], - "TK, EM": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], - "TK, GR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], - "TK, GCR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], - "TK/GR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], - "TK/RG": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], - "TK, KrPe": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kirsten Pearthree", "NMBGMR", "Research Scientist"]], - "TK, PB, PJ": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], - "TK, SC": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], - "TK, ST, CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], - "TK, ST; CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], - "TK, TN": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "Steve": ["Steve", "Village of Magdalena", "Operator"], + "T.Decker": ["T.Decker", "Unknown", "Unknown"], + "TK": ["Trevor Kludt", "NMBGMR", "Technician"], + "Trevor Kludt": ["Trevor Kludt", "NMBGMR", "Technician"], + "TK BF": [["Trevor Kludt", "NMBGMR", "Technician"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "TK, BF": [["Trevor Kludt", "NMBGMR", "Technician"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "TK/BF": [["Trevor Kludt", "NMBGMR", "Technician"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "tk cm": [["Trevor Kludt", "NMBGMR", "Technician"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "TK, CM": [["Trevor Kludt", "NMBGMR", "Technician"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "TK KR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], + "TK, KR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], + "TK, AL": [["Trevor Kludt", "NMBGMR", "Technician"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], + "TK, CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK,CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, EM": [["Trevor Kludt", "NMBGMR", "Technician"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + "TK, GR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK, GCR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK/GR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK/RG": [["Trevor Kludt", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK, KrPe": [["Trevor Kludt", "NMBGMR", "Technician"], ["Kristen Pearthree", "NMBGMR", "Research Scientist"]], + "TK, PB, PJ": [["Trevor Kludt", "NMBGMR", "Technician"], ["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], + "TK, SC": [["Trevor Kludt", "NMBGMR", "Technician"], ["Scott Christenson", "NMBGMR", "Technician"]], + "TK, ST, CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, ST; CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, JAA": [["Trevor Kludt", "NMBGMR", "Technician"], ["JAA", "NMBGMR", "Unknown"]], + "TK, MR": [["Trevor Kludt", "NMBGMR", "Technician"], ["MR", "Unknown", "Unknown"]], + "TK, TN": [["Trevor Kludt", "NMBGMR", "Technician"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], "TN": ["Talon Newton", "NMBGMR", "Hydrogeologist"], "TN, LL": [["Talon Newton", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], "Wasiolek": ["Maryann Wasiolek", "Hydroscience Associates, Inc", "Hydrogeologist"], diff --git a/transfers/group_transfer.py b/transfers/group_transfer.py index 95e9c7949..8a414d680 100644 --- a/transfers/group_transfer.py +++ b/transfers/group_transfer.py @@ -16,7 +16,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from db import Thing, Group +from db import Thing, Group, GroupThingAssociation from db.engine import session_ctx from transfers.util import read_csv from transfers.logger import logger @@ -39,12 +39,15 @@ def transfer_groups( if prefix: # get all PointIDs that start with prefix sql = select(Thing).where(Thing.name.like(f"{prefix}%")) - records = session.scalars(sql).all() + records = session.scalars(sql).unique().all() if records: logger.info( f"Adding {len(records)} things to group {group.name}, prefix {prefix}" ) - group.things = records + for record in records: + gta = GroupThingAssociation(group=group, thing=record) + session.add(gta) + group.thing_associations.append(gta) session.add(group) session.commit() diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 2172575fd..76048b94a 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -60,7 +60,9 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - assoc.location = location assoc.thing = spring session.add(assoc) - session.commit() + session.commit() + session.expire(location) + session.refresh(location) def transfer_springs(session, limit=None): diff --git a/transfers/transfer.py b/transfers/transfer.py index 61b88ef9d..bdf29e450 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -19,6 +19,7 @@ load_dotenv() + from sqlalchemy.orm import Session from core.initializers import init_lexicon, init_parameter from db import Base @@ -29,8 +30,19 @@ from transfers.contact_transfer import transfer_contacts from transfers.sensor_transfer import init_sensor from transfers.waterlevels_transfer import transfer_water_levels -from transfers.well_transfer import transfer_wells, transfer_wellscreens - +from transfers.well_transfer import ( + transfer_wells, + transfer_wellscreens, + cleanup_locations, +) + +from transfers.asset_transfer import transfer_assets +from transfers.thing_transfer import ( + transfer_springs, + transfer_perennial_stream, + transfer_ephemeral_stream, + transfer_met, +) from transfers.util import timeit, timeit_direct from transfers.logger import logger, save_log_to_bucket @@ -71,6 +83,8 @@ def erase(session: Session): with session.bind.connect() as conn: conn.execute(text("DROP SCHEMA public CASCADE")) conn.execute(text("CREATE SCHEMA public")) + conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + conn.commit() Base.metadata.drop_all(session.bind) logger.info("Recreating tables") @@ -93,17 +107,23 @@ def transfer_all(sess, limit=100): timeit_direct(transfer_wells, sess, limit=limit) timeit_direct(transfer_wellscreens, sess) - # message("TRANSFERRING SPRINGS") - # timeit_direct(transfer_springs, sess, limit=limit) - # - # message("TRANSFERRING PERENNIAL STREAMS") - # timeit_direct(transfer_perennial_stream, sess, limit=limit) - # - # message("TRANSFERRING EPHEMERAL STREAMS") - # timeit_direct(transfer_ephemeral_stream, sess, limit=limit) - # - # message("TRANSFERRING METEOROLOGICAL") - # timeit_direct(transfer_met, sess, limit) + """ + Developer's note + this is a very time consuming operation and the results should + be saved to a file for later use. + """ + + message("TRANSFERRING SPRINGS") + timeit_direct(transfer_springs, sess, limit=limit) + + message("TRANSFERRING PERENNIAL STREAMS") + timeit_direct(transfer_perennial_stream, sess, limit=limit) + + message("TRANSFERRING EPHEMERAL STREAMS") + timeit_direct(transfer_ephemeral_stream, sess, limit=limit) + + message("TRANSFERRING METEOROLOGICAL") + timeit_direct(transfer_met, sess, limit) message("TRANSFERRING CONTACTS") timeit_direct(transfer_contacts, sess) @@ -114,7 +134,7 @@ def transfer_all(sess, limit=100): When transfering water chemistry data use the qc_type field to indicate normal/blanks/duplicates instead of what comes from LU_SampleType. Use those values, however, to map to the standard qc_type fields if applicable - (i.e. not applicable when sample type is "Soil or rock sample" or + (i.e. not applicable when sample type is "Soil or rock sample" or "Precipitation," but is applicable when sample type is "Equipment blank" or "Field duplicate") """ @@ -128,16 +148,16 @@ def transfer_all(sess, limit=100): message("TRANSFERRING WATER LEVELS") timeit_direct(transfer_water_levels, sess) - # message("TRANSFERRING ASSETS") - # timeit_direct(transfer_assets, sess) + message("TRANSFERRING ASSETS") + timeit_direct(transfer_assets, sess) - # if init or cleanup_wells_flag: - # cleanup_wells(sess) + message("CLEANING UP LOCATIONS") + timeit_direct(cleanup_locations, sess) def main(): message("START--------------------------------------") - limit = int(os.environ.get("TRANSFER_LIMIT", 100)) + limit = int(os.environ.get("TRANSFER_LIMIT", 200)) with session_ctx() as sess: transfer_all(sess, limit=limit) diff --git a/transfers/util.py b/transfers/util.py index 9bbc39484..c1836174e 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -16,6 +16,8 @@ import csv import os from datetime import datetime, timezone, timedelta +from pathlib import Path + import pytz import re import io @@ -84,11 +86,16 @@ def extract_organization(alternate_id: str) -> str: return "Unknown" -def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: - path = "/workspace/transfers/data/valid_welldata_datasources.csv" - if not os.path.exists(path): - path = "transfers/data/valid_welldata_datasources.csv" +def get_transfers_data_path(name): + root = Path("/workspace/transfers/data") + if not os.path.exists(root): + root = Path("./transfers/data/") + + return root / name + +def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: + path = get_transfers_data_path("valid_welldata_datasources.csv") with open(path, "r") as f: reader = csv.reader(f) _ = next(reader) @@ -100,9 +107,7 @@ def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: - path = "/workspace/transfers/data/valid_measuring_agency.csv" - if not os.path.exists(path): - path = "transfers/data/valid_measuring_agency.csv" + path = get_transfers_data_path("valid_measuring_agency.csv") with open(path, "r") as f: reader = csv.reader(f) @@ -175,20 +180,19 @@ def make_location(row: pd.Series) -> Location: if elevation_from_epqs: elevation_method = "USGS National Elevation Dataset (NED)" - elif not (pd.isna(row.AltitudeMethod)): + elif pd.isna(row.AltitudeMethod): + elevation_method = None + else: elevation_method = lexicon_mapper.map_value( f"LU_AltitudeMethod:{row.AltitudeMethod}" ) + if pd.isna(row.CoordinateMethod): + coordinate_method = None else: - elevation_method = None - - if not (pd.isna(row.CoordinateMethod)): coordinate_method = lexicon_mapper.map_value( f"LU_CoordinateMethod:{row.CoordinateMethod}" ) - else: - coordinate_method = None """ Developer's notes @@ -208,8 +212,9 @@ def make_location(row: pd.Series) -> Location: created_at = site_date elif row.DateCreated and not row.SiteDate: created_at = datetime.strptime(row.DateCreated, "%Y-%m-%d %H:%M:%S.%f") + elif not row.DateCreated and row.SiteDate: + created_at = datetime.strptime(row.SiteDate, "%Y-%m-%d %H:%M:%S.%f") else: - # TODO: should this be set to SiteDate if DateCreated is None and SiteDate is populated? created_at = None # convert created_at from MST/MDT to UTC @@ -301,7 +306,7 @@ def timeit_direct(func, *args, **kwargs): start = datetime.now() result = func(*args, **kwargs) end = datetime.now() - logger.info(f"{func.__name__} took {(end - start).total_seconds()} seconds") + logger.info(f"TIMING: {func.__name__} took {(end - start).total_seconds()} seconds") return result diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 8218c985c..e25d58903 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -26,8 +26,10 @@ Observation, FieldEvent, FieldActivity, - FieldEventContactAssociation, + FieldEventParticipant, Contact, + FieldEventParticipant, + Parameter, ) from transfers.util import ( filter_to_valid_point_ids, @@ -36,11 +38,13 @@ convert_mt_to_utc, filter_by_valid_measuring_agency, lexicon_mapper, + get_transfers_data_path, ) # constants SPACE_2 = " " * 2 SPACE_4 = " " * 4 +SPACE_6 = " " * 6 def get_dt_utc(row): @@ -70,10 +74,10 @@ def get_dt_utc(row): f"transfer_water_levels. Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to " f"invalid date/time: {e}" ) + return None def get_contacts_info(row, measured_by, measured_by_mapper): - measuring_agency = ( "Unknown" if pd.isna(row.MeasuringAgency) else row.MeasuringAgency ) @@ -91,128 +95,31 @@ def get_contacts_info(row, measured_by, measured_by_mapper): ns = [args[0]] os = [args[1]] rs = [args[2]] - elif measured_by is None: - ns = [None] - os = ["Unknown"] - rs = ["Unknown"] - elif measured_by in [ - "Anthony Chavez", - "BEI", - "BF/RG", - "Borchert", - "Borton & Cooper", - "CDWR", - "Chaves/Cruz", - "Chavez/Cruz", - "CM, AK", - "Coons", - "Cooper", - "Corbin", - "Crocker", - "Cruz", - "Cruz-Tribble", - "Cruz/Frost", - "D.Bird", - "D.D.", - "D.Duncan", - "Dames & Moore", - "Dames/Moore", - "Dave Snider", - "David N Jenkins", - "David N. Jenkins", - "DC", - "Decker", - "DL, TK", - "DR", - "DR, ST", - "Duke Engring", - "Duncan", - "EA", - "EA/HB", - "Frost", - "G.Boylan", - "GLR, SC", - "GLR, SK, SC", - "GR, MM", - "GR/PW", - "GR/RG", - "HB", - "Heaton", - "Horner-Crocker", - "HR", - "Hydrogeologic Serv", - "J.Evans", - "J.Frost", - "Jenkins", - "Johnson/Cruz", - "Johnson/Robbins", - "Kilmer/Jenkins", - "KP, MR", - "KP, MT", - "Lazarus", - "Mike Rodgers", - "Mourant", - "MWB Consultant", - "Myers report", - "Rankin", - "Sandia Drillers", - "SC, MR", - "SdC", - "SM&Assoc", - "SMA", - "Spiegel", - "Spiegel & Baldwin", - "SPRI", - "Steve", - "T.Decker", - "Topol", - "URS", - "UTM", - "VeneKlasen", - "Vista del Oro", - ]: - # set name to measured_by so that water level is logged to that - # person even if they are not known. this allows future updates + else: ns = [measured_by] os = ["Unknown"] rs = ["Unknown"] logger.warning( - f"{SPACE_4}The following record has not been mapped to a Contact: {row.MeasuredBy} // {row.MeasuringAgency} for PointID {row.PointID} (which comes from the WaterLevels table)" - ) - - elif measured_by in [ - "?", - "Consultant", - "Consulting Pro.", - "Gamma log unit", - "Pump company", - "PumpService", - "REPORTED", - "Theis report", - "Unknown", - "Unknown; reported", - "Water operator", - "WWTP", - "WWTP personnel", - ]: - # Unknowns - ns = [None] - os = [measuring_agency] - rs = ["Unknown"] - - else: - logger.critical( - f"Skipping the following record because it has no mappings: {row.MeasuredBy} // {row.MeasuringAgency} for PointID {row.PointID}" + f"{SPACE_6}The following record has not been mapped to a Contact: MeasuredBy {row.MeasuredBy} | MeasuringAgency {row.MeasuringAgency} for WaterLevels record with GLobalID {row.GlobalID}" ) - return return ns, os, rs def transfer_water_levels(session): + groundwater_parameter_id = ( + session.query(Parameter) + .filter(Parameter.parameter_name == "groundwater level") + .one() + .id + ) + # keep a dictionary of created Contacts to avoid repeated SQL queries + # keys are a tuple of (name, organization) since None is a common "name" created_contacts = {} - with open("transfers/data/measured_by_mapper.json", "r") as f: + path = get_transfers_data_path("measured_by_mapper.json") + + with open(path, "r") as f: measured_by_mapper = json.load(f) wd = read_csv("WaterLevels") @@ -243,14 +150,60 @@ def transfer_water_levels(session): if dt_utc is None: continue - if pd.isna(row.DepthToWater): - logger.warning( - f"{SPACE_4}No sample and observation have been made for WaterLevels record with GlobalID {row.GlobalID} because DepthToWater is NULL" + release_status = "public" if row.PublicRelease else "private" + + measured_by = None if pd.isna(row.MeasuredBy) else row.MeasuredBy + + """ + Developer's notes + + Use existing contact for the thing if measured by is the owner. + + If no contacts can be made or retrieved for the field event skip + it altogether and note in the log file. There must be at least one + contact associated with an event + """ + field_event_participants = [] + if measured_by not in ["Owner", "Owner report", "Well owner"]: + # --- Contact/FieldEventParticipant --- + contact_info = get_contacts_info(row, measured_by, measured_by_mapper) + + for name, organization, role in zip(*contact_info): + if (name, organization) in created_contacts: + contact = created_contacts[(name, organization)] + else: + try: + # create new contact if not already created + contact = Contact( + name=name, + role=role, + contact_type="Field Event Participant", + organization=organization, + nma_pk_waterlevels=row.GlobalID, + ) + session.add(contact) + session.flush() # to get the contact.id + + logger.info( + f"{SPACE_2}Created contact: ID {contact.id} | Name {contact.name} | Role {contact.role} | Organization {contact.organization} | nma_pk_waterlevels {contact.nma_pk_waterlevels}" + ) + + created_contacts[(name, organization)] = contact + except Exception as e: + logger.critical( + f"Contact cannot be created: Name {name} | Role {role} | Organization {organization} because of the following: {str(e)}" + ) + field_event_participants.append(contact) + else: + contact = thing.contacts[0] + field_event_participants.append(contact) + + if len(field_event_participants) == 0: + logger.critical( + f"No contacts can be associated with the WaterLevels record with GlobalID {row.GlobalID}, therefore no field event, field activity, sample, and observation can be made. Skipping." ) continue - release_status = "public" if row.PublicRelease else "private" - """ Developer's notes @@ -273,65 +226,6 @@ def transfer_water_levels(session): f"{SPACE_2}Created field event: ID {field_event.id} | Date {field_event.event_date} | Thing ID {field_event.thing.id} | Thing Name {field_event.thing.name}" ) - # --- FieldActivity --- - # TODO: use create schema to validate data - field_activity = FieldActivity( - field_event=field_event, - activity_type="groundwater level", - release_status=release_status, - ) - session.add(field_activity) - session.flush() - - logger.info( - f"{SPACE_4}Created field activity: ID {field_activity.id} | Type {field_activity.activity_type}" - ) - - measured_by = None if pd.isna(row.MeasuredBy) else row.MeasuredBy - - """ - Developer's notes - - Use existing contact for the thing if measured by is the owner - """ - field_event_contacts = [] - if measured_by not in ["Owner", "Owner report", "Well owner"]: - # --- Contact/FieldEventContactAssociation --- - contact_info = get_contacts_info(row, measured_by, measured_by_mapper) - if contact_info is None: - continue - contact_names, contact_organizations, roles = contact_info - - for i, c in enumerate(contact_names): - if c in created_contacts: - contact = created_contacts[c] - else: - # create new contact if not already created - name = contact_names[i] - organization = contact_organizations[i] - role = roles[i] - - contact = Contact( - name=name, - role=role, - contact_type="Field Event Participant", - organization=organization, - nma_pk_waterlevels=row.GlobalID, - ) - session.add(contact) - session.flush() # to get the contact.id - - logger.info( - f"{SPACE_4}Created contact: ID {contact.id} | Name {contact.name} | Role {contact.role} | Organization {contact.organization} | nma_pk_waterlevels {contact.nma_pk_waterlevels}" - ) - - created_contacts[c] = contact - - field_event_contacts.append(contact) - else: - contact = thing.contacts[0] - field_event_contacts.append(contact) - """ Developer's notes @@ -339,35 +233,72 @@ def transfer_water_levels(session): person who took the sample. The subsequent contact will be participants in the field event """ - for i, fec in enumerate(field_event_contacts): - field_event_contact = FieldEventContactAssociation( - field_event=field_event, contact=fec + for i, participant in enumerate(field_event_participants): + field_event_participant = FieldEventParticipant( + field_event=field_event, participant=participant ) if i == 0: - field_event_contact.field_contact_role = "Lead" - sampler = field_event_contact + field_event_participant.participant_role = "Lead" + sampler = field_event_participant else: - field_event_contact.field_contact_role = "Participant" + field_event_participant.participant_role = "Participant" - session.add(field_event_contact) + session.add(field_event_participant) session.flush() logger.info( - f"{SPACE_4}Created field event contact: ID {field_event_contact.id} | Contact Name {field_event_contact.contact.name} | Field Contact Role {field_event_contact.field_contact_role}" + f"{SPACE_4}Created field event contact: ID {field_event_participant.id} | Role {field_event_participant.participant_role} | Contact ID {field_event_participant.participant.id} | Contact Name {field_event_participant.participant.name} | Contact Org {field_event_participant.participant.organization}" ) + groundwater_level_reason = ( + lexicon_mapper.map_value(f"LU_LevelStatus:{row.LevelStatus}") + if not pd.isna(row.LevelStatus) + else None + ) + groundwater_level_reason = ( + "Water level not affected" + if groundwater_level_reason == "Water level not affected by status" + else groundwater_level_reason + ) + + if ( + groundwater_level_reason + == "Well was destroyed (no subsequent water levels should be recorded)" + ): + logger.warning( + "Well is destroyed - no field activity/sample/observation will be made" + ) + field_event.notes = groundwater_level_reason + session.refresh() + continue + + # --- FieldActivity --- + # TODO: use create schema to validate data + field_activity = FieldActivity( + field_event=field_event, + activity_type="groundwater level", + release_status=release_status, + ) + session.add(field_activity) + session.flush() + + logger.info( + f"{SPACE_4}Created field activity: ID {field_activity.id} | Type {field_activity.activity_type}" + ) + # --- Sample --- - if not pd.isna(row.MeasurementMethod): - sample_method = lexicon_mapper.map_value( + sample_method = ( + "null placeholder" + if pd.isna(row.MeasurementMethod) + else lexicon_mapper.map_value( f"LU_MeasurementMethod:{row.MeasurementMethod}" ) - else: - sample_method = "null placeholder" + ) # todo: use create schema to validate data sample = Sample( nma_pk_waterlevels=row.GlobalID, field_activity=field_activity, - field_event_contact=sampler, + field_event_participant=sampler, sample_date=dt_utc, sample_matrix="water", sample_name=str(uuid.uuid4()), @@ -382,14 +313,34 @@ def transfer_water_levels(session): f"{SPACE_4}Created sample: ID {sample.id} | Date {sample.sample_date} | Matrix {sample.sample_matrix} | Method {sample.sample_method}" ) - if not pd.isna(row.LevelStatus): - level_status = lexicon_mapper.map_value( - f"LU_LevelStatus:{row.LevelStatus}" - ) + # TODO: use create schema to validate data + + if pd.isna(row.MPHeight): + if not pd.isna(row.DepthToWater) and not pd.isna(row.DepthToWaterBGS): + logger.warning( + f"{SPACE_6}Calculating measuring_point_height as DepthToWater - DepthToWaterBGS because MPHeight is NULL" + ) + measuring_point_height = row.DepthToWater - row.DepthToWaterBGS + else: + logger.warning( + f"{SPACE_6}Setting measuring_point_height to None because MPHeight is NULL and DepthToWater or DepthToWaterBGS is NULL" + ) + measuring_point_height = None else: - level_status = None + # some mp heights are recorded as negative numbers, but they should be positive + measuring_point_height = abs(row.MPHeight) - # TODO: use create schema to validate data + if pd.isna(row.DepthToWater): + if not pd.isna(row.DepthToWaterBGS): + logger.warning( + f"{SPACE_6}Calculating observation value as DepthToWaterBGS + MPHeight (0 if MPHeight is NULL) because DepthToWater is NULL" + ) + value = row.DepthToWaterBGS + measuring_point_height + else: + # use None not NaN + value = None + else: + value = row.DepthToWater # TODO: after sensors have been added to the database update sensor_id (or sensor) for waterlevels that come from db sensors (like e probes?) observation = Observation( @@ -398,11 +349,11 @@ def transfer_water_levels(session): sensor_id=None, analysis_method_id=None, observation_datetime=dt_utc, - observed_property="groundwater level", - value=row.DepthToWater, + parameter_id=groundwater_parameter_id, + value=value, unit="ft", - measuring_point_height=row.MPHeight, - level_status=level_status, + measuring_point_height=measuring_point_height, + groundwater_level_reason=groundwater_level_reason, ) session.add(observation) session.flush() diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 33520814c..2bb513331 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import json import time from pydantic import ValidationError from sqlalchemy import select +from datetime import datetime from db import LocationThingAssociation, Thing, WellScreen, Location from schemas.thing import CreateWellScreen, CreateWell -from services.thing_helper import add_thing +from services.gcs_helper import get_storage_bucket from services.util import ( get_state_from_point, get_county_from_point, @@ -78,50 +80,69 @@ def transfer_wells(session, limit=0): logger.critical(f"Error making location for {row.PointID}: {e}") continue - session.add(location) - - # TODO: add guards for null values - # TODO: use schema to validate - - data = CreateWell( - # "nma_pk_welldata": row.WellID, - name=row.PointID, - hole_depth=row.HoleDepth, - well_depth=row.WellDepth, - well_construction_notes=row.ConstructionNotes, - # "driller_name": row.DrillerName, - # "construction_method": row.ConstructionMethod, - # "casing_diameter": row.CasingDiameter, - # "casing_depth": row.CasingDepth, - # "casing_description": row.CasingDescription, - release_status="public" if row.PublicRelease else "private", - # "data_reliability": row.DataReliability, - ) try: - well = add_thing( - session, - data, - thing_type="water well", + session.add(location) + + # well_purpose = None if pd.isna(row.CurrentUse) else lexicon_mapper.map_value(f"LU_CurrentUse:{row.CurrentUse}") + + # if pd.isna(row.CasingDescription): + # well_casing_material = None + # elif "pvc" in row.CasingDescription.lower(): + # well_casing_material = "PVC" + # elif "steel" in row.CasingDescription.lower(): + # well_casing_material = "Steel" + + if row.DateCreated and row.SiteDate: + + date_created = datetime.strptime( + row.DateCreated, "%Y-%m-%d %H:%M:%S.%f" + ).date() + site_date = datetime.strptime( + row.SiteDate, "%Y-%m-%d %H:%M:%S.%f" + ).date() + + if date_created < site_date: + first_visit_date = date_created + else: + first_visit_date = site_date + elif row.DateCreated and not row.SiteDate: + first_visit_date = datetime.strptime( + row.DateCreated, "%Y-%m-%d %H:%M:%S.%f" + ).date() + elif not row.DateCreated and row.SiteDate: + first_visit_date = datetime.strptime( + row.SiteDate, "%Y-%m-%d %H:%M:%S.%f" + ).date() + else: + first_visit_date = None + + # manually add the well rather than add_well from services/thing_helper.py + # so that effective_start can be set on the location assocation + data = CreateWell( + location_id=location.id, + nma_pk_welldata=row.WellID, + name=row.PointID, + first_visit_date=first_visit_date, + # well_purpose=well_purpose, + hole_depth=row.HoleDepth, + well_depth=row.WellDepth, + well_construction_notes=row.ConstructionNotes, + well_casing_diameter=row.CasingDiameter, + well_casing_depth=row.CasingDepth, + release_status="public" if row.PublicRelease else "private", ) + + CreateWell.model_validate(data) + well_data = data.model_dump(exclude=["location_id", "group_id"]) + well_data["thing_type"] = "water well" + well = Thing(**well_data) + session.add(well) except Exception as e: session.rollback() - logger.critical(f"Error creating well for {row.PointID}: {e}") + logger.critical(f"Error creating well for {row.PointID}: {e.errors()}") continue - # TODO: use current use LUT to get well type - - # wt = row.Meaning - # if wt not in ADDED: - # add_lexicon_term( - # session, - # wt, - # "Current use of the well, aka well purpose", - # [{"name": "current_use", "desciption": "Current use of the well"}], - # ) - # ADDED.append(wt) - # - # well.well_purpose = wt - - assoc = LocationThingAssociation() + + assoc = LocationThingAssociation(effective_start=location.created_at) assoc.location = location assoc.thing = well @@ -129,8 +150,10 @@ def transfer_wells(session, limit=0): try: session.commit() + session.expire(location) + session.refresh(location) except Exception as e: - logger.critical(f"Error committing well {row.PointID}: {e}") + logger.critical(f"Error committing well {row.PointID}: {e.errors()}") session.rollback() continue @@ -159,7 +182,7 @@ def transfer_wellscreens(session, limit=None): session.commit() sql = select(Thing).where(Thing.name == row.PointID) - thing = session.execute(sql).scalar_one_or_none() + thing = session.execute(sql).unique().scalar_one_or_none() if not thing: logger.warning( f"Thing with PointID {row.PointID} not found. Skipping well screen." @@ -189,21 +212,63 @@ def transfer_wellscreens(session, limit=None): session.commit() -def cleanup_wells(session): +def cleanup_locations(session): locations = session.query(Location).all() - for location in locations: + n = len(locations) + lut = {} - y, x = location.latlon - if not location.state: - location.state = get_state_from_point(x, y) + bucket = get_storage_bucket() + log_filename = "transfer_data/location_cleanup.json" + blob = bucket.blob(log_filename) + if blob.exists(): + lut = json.loads(blob.download_as_string()) - if not location.county: - location.county = get_county_from_point(x, y) + updates = [] + for i, location in enumerate(locations): + if i and not i % 100: + logger.info(f"Processing row {i} of {n}. dumping lut to {log_filename}") + blob.upload_from_string(json.dumps(lut)) + session.bulk_update_mappings(Location, updates) + session.commit() + updates = [] - if not location.quad_name: - location.quad_name = get_quad_name_from_point(x, y) + y, x = location.latlon + xykey = f"{y},{x}" + if xykey in lut: + state, county, quad_name = lut[xykey] + else: + state = location.state + county = location.county + quad_name = location.quad_name + if not state: + state = get_state_from_point(x, y) + + if not county: + county = get_county_from_point(x, y) + + if not quad_name: + quad_name = get_quad_name_from_point(x, y) + + lut[xykey] = [state, county, quad_name] + + updates.append( + { + "id": location.id, + "state": state, + "county": county, + "quad_name": quad_name, + } + ) - session.commit() + logger.info( + f"{i}/{n} lat: {y} lon: {x} state={state}, county={county}, quad" + f"={quad_name}" + ) + + blob.upload_from_string(json.dumps(lut)) + if updates: + session.bulk_update_mappings(Location, updates) + session.commit() # ============= EOF =============================================