Bdms 227 transfer updates

core/lexicon.json

@@ -85,7 +85,7 @@
     {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"},
     {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"},
     {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"},
-    {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"},
+    {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"},
     {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"},
     {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"},
     {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"},
@@ -578,6 +578,7 @@
     {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"},
     {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"},
     {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"},
+    {"categories": ["organization"], "term": "Quemado Municipal Water & SWA", "definition": "Quemado Municipal Water & SWA"},
     {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"},
     {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"},
     {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"},
@@ -698,10 +699,6 @@
     {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."},
     {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."},
     {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."},
-    {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"},
-    {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"},
-    {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"},
-    {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"},
     {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"},
     {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"},
     {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"},
@@ -1105,6 +1102,10 @@
     {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."},
     {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"},
     {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"},
+    {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"},
+    {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"},
+    {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"},
+    {"categories": ["well_pump_type"], "term": "Hand", "definition": "Hand Pump"},
     {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"},
     {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"},
     {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"}

schemas/thing.py

@@ -132,8 +132,15 @@ class CreateWell(CreateBaseThing, ValidateWell):
     measuring_point_height: float = Field(
         ge=0, description="Measuring point height in feet"
     )
-    measuring_point_description: str | None
+    measuring_point_description: str | None = None
     notes: list[CreateNote] | None = None
+    well_completion_date: PastOrTodayDate | None = None
+    well_completion_date_source: str | None = None
+    well_driller_name: str | None = None
+    well_construction_method: WellConstructionMethod | None = None
+    well_construction_method_source: str | None = None
+    well_pump_type: WellPumpType | None = None
+    is_suitable_for_datalogger: bool | None
 
 
 class CreateSpring(CreateBaseThing):
@@ -335,7 +342,7 @@ class WellScreenResponse(BaseResponseModel):
     aquifer_system: str | None = None
     aquifer_type: str | None = None
     geologic_formation_id: int | None = None
-    # geologic_formation: GeologicFormationResponse | None = None
+    geologic_formation: str | None = None
     screen_depth_bottom: float
     screen_depth_bottom_unit: str = "ft"
     screen_depth_top: float
@@ -355,7 +362,7 @@ def populate_aquifer_type_with_name(cls, aquifer_type):
             return aquifer_type.name
         return None
 
-    @field_validator("geologic_formation_id", mode="before")
+    @field_validator("geologic_formation", mode="before")
     def populate_geologic_formation_with_code(cls, geologic_formation):
         if geologic_formation is not None:
             return geologic_formation.formation_code

transfers/data/owners_organization_mapper.json

@@ -89,6 +89,7 @@
 	"Pecos Trail Inn": "Pecos Trail Inn",
 	"Pelican Spa": "Pelican Spa",
 	"Pistachio Tree Ranch": "Pistachio Tree Ranch",
+	"Quemado Mutual Water and Sewage Works Association": "Quemado Municipal Water & SWA",
 	"Rancho Encantado": "Rancho Encantado",
 	"Rancho San Lucas": "Rancho San Lucas",
 	"Rancho San Marcos": "Rancho San Marcos",

transfers/permissions_transfer.py

@@ -0,0 +1,87 @@
+from sqlalchemy.orm import Session
+from datetime import datetime
+
+from db import Thing, PermissionHistory
+from transfers.util import read_csv, logger, replace_nans
+
+"""
+Developer's notes
+
+According to Laila the column WellData.OpenWellLoggerOK only pertains to the
+physical properties of a well (that is, if a datalogger can be installed). It
+does not pertain to permissions.
+"""
+
+
+def transfer_permissions(session: Session):
+    """
+    The transferred wells and contacts need to be queried to know who gave
+    permission to which well since contact_id is required for PermissionHistory
+    """
+    wdf = read_csv("WellData", dtype={"OSEWelltagID": str})
+    wdf = replace_nans(wdf)
+
+    transferred_wells = (
+        session.query(Thing).filter(Thing.thing_type == "water well").all()
+    )
+
+    for well in transferred_wells:
+        if len(well.contacts) == 0:
+            logger.critical(
+                f"Well {well.name} has no associated contacts; skipping permission transfer."
+            )
+            continue
+        else:
+            # Assuming the first contact is the relevant one
+            contact_id = well.contacts[0].id
+
+        allow_water_level_samples = wdf.loc[
+            wdf["PointID"] == well.name, "MonitorOK"
+        ].values
+        if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None:
+            try:
+                permission_allowed = bool(allow_water_level_samples[0])
+                permission = PermissionHistory(
+                    contact_id=contact_id,
+                    permission_type="Water Level Sample",
+                    permission_allowed=permission_allowed,
+                    start_date=datetime.today().date(),
+                    target_id=well.id,
+                    target_table="thing",
+                )
+                session.add(permission)
+                logger.info(
+                    f"Transferred Water Level Sample permission for well {well.name}: {permission_allowed}."
+                )
+            except Exception as e:
+                logger.error(f"Error transferring permission for well {well.name}: {e}")
+                session.rollback()
+                pass
+
+        allow_water_chemistry_samples = wdf.loc[
+            wdf["PointID"] == well.name, "SampleOK"
+        ].values
+        if (
+            len(allow_water_chemistry_samples) > 0
+            and allow_water_chemistry_samples is not None
+        ):
+            try:
+                permission_allowed = bool(allow_water_chemistry_samples[0])
+                permission = PermissionHistory(
+                    contact_id=contact_id,
+                    permission_type="Water Chemistry Sample",
+                    permission_allowed=permission_allowed,
+                    start_date=datetime.today().date(),
+                    target_id=well.id,
+                    target_table="thing",
+                )
+                session.add(permission)
+                logger.info(
+                    f"Transferred Water Chemistry Sample permission for well {well.name}: {permission_allowed}."
+                )
+            except Exception as e:
+                logger.error(f"Error transferring permission for well {well.name}: {e}")
+                session.rollback()
+                pass
+
+    session.commit()

transfers/transfer.py

@@ -36,6 +36,7 @@
     transfer_wells,
     transfer_wellscreens,
 )
+from transfers.permissions_transfer import transfer_permissions
 
 from transfers.asset_transfer import transfer_assets
 from transfers.util import timeit, timeit_direct
@@ -124,6 +125,9 @@ def transfer_all(sess, limit=100):
     message("TRANSFERRING ASSETS")
     timeit_direct(transfer_assets, sess)
 
+    message("TRANSFERRING PERMISSIONS")
+    results = timeit_direct(transfer_permissions, sess)
+
 
 def transfer_debugging(sess, limit=100):
     message("STARTING TRANSFER DEBUG", new_line_at_top=False)

transfers/util.py

@@ -275,7 +275,7 @@ def make_location_data_provenance(
 ) -> list[DataProvenance]:
     provenance_records = []
 
-    if row.AltitudeAccuracy or row.CoordinateAccuracy:
+    if row.AltitudeAccuracy:
         provenance = DataProvenance(
             target_id=location.id,
             target_table="location",
@@ -369,7 +369,6 @@ def make_location_data_provenance(
             target_id=location.id,
             target_table="location",
             field_name="point",
-            origin_source=None,
             collection_method=coordinate_method,
             accuracy_value=accuracy_value,
             accuracy_unit=accuracy_unit,

transfers/well_transfer.py

@@ -36,6 +36,7 @@
     StatusHistory,
     MonitoringFrequencyHistory,
     MeasuringPointHistory,
+    DataProvenance,
 )
 from schemas.thing import CreateWell, CreateWellScreen
 from services.gcs_helper import get_storage_bucket
@@ -117,6 +118,21 @@ def _extract_casing_materials(row) -> list[str]:
     return materials
 
 
+def _extract_well_pump_type(row) -> str | None:
+    construction_notes = row.ConstructionNotes.lower()
+    if "pump" in construction_notes:
+        if "submersible" in construction_notes:
+            return "Submersible"
+        elif "jet" in construction_notes:
+            return "Jet"
+        elif "line shaft" in construction_notes or "lineshaft" in construction_notes:
+            return "Line Shaft"
+        elif "hand" in construction_notes:
+            return "Hand"
+        else:
+            return None
+
+
 def get_wells_to_transfer(
     sess: Session, flags: dict = None
 ) -> tuple[pd.DataFrame, pd.DataFrame]:
@@ -236,6 +252,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None
             well_casing_materials = (
                 [] if isna(row.CasingDescription) else _extract_casing_materials(row)
             )
+            well_pump_type = (
+                _extract_well_pump_type(row) if row.ConstructionNotes else None
+            )
 
             # manually add the well rather than add_well from services/thing_helper.py
             # so that effective_start can be set on the location assocation
@@ -257,6 +276,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None
                 notes=(
                     [{"content": row.Notes, "note_type": "Other"}] if row.Notes else []
                 ),
+                well_completion_date=row.CompletionDate,
+                well_driller_name=row.DrillerName,
+                well_construction_method=(
+                    lexicon_mapper.map_value(
+                        f"LU_ConstructionMethod:{row.ConstructionMethod}"
+                    )
+                    if not isna(row.ConstructionMethod)
+                    else None
+                ),
+                well_pump_type=well_pump_type,
+                is_suitable_for_datalogger=(
+                    bool(row.OpenWellLoggerOK)
+                    if not isna(row.OpenWellLoggerOK)
+                    else None
+                ),
             )
 
             CreateWell.model_validate(data)
@@ -277,6 +311,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None
                     "well_casing_materials",
                     "measuring_point_height",
                     "measuring_point_description",
+                    "well_completion_date_source",
+                    "well_construction_method_source",
                 ]
             )
             well_data["thing_type"] = "water well"
@@ -285,17 +321,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None
             well_data.pop("notes")
             well = Thing(**well_data)
             session.add(well)
-            # logger.info(f"Created well for {row.PointID}")
-
-            # flush well to access its ID for status_history
-            # session.flush()
-
-            # session.commit()
-            # session.refresh(well)
-            # if notes:
-            #     for ni in notes:
-            #         nn = well.add_note(ni['content'], ni['note_type'])
-            #         session.add(nn)
 
             if well_purposes:
                 for wp in well_purposes:
@@ -347,11 +372,42 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None
         for dp in data_provenances:
             session.add(dp)
 
+        if not isna(row.CompletionSource):
+            dp = DataProvenance(
+                target_id=well.id,
+                target_table="thing",
+                field_name="well_completion_date",
+                origin_type=lexicon_mapper.map_value(
+                    f"LU_Depth_CompletionSource:{row.CompletionSource}"
+                ),
+            )
+            session.add(dp)
+
+        if not isna(row.DataSource):
+            dp = DataProvenance(
+                target_id=well.id,
+                target_table="thing",
+                field_name="well_construction_method",
+                origin_source=row.DataSource,
+            )
+            session.add(dp)
+
+        if not isna(row.DepthSource):
+            dp = DataProvenance(
+                target_id=well.id,
+                target_table="thing",
+                field_name="well_depth",
+                origin_type=lexicon_mapper.map_value(
+                    f"LU_Depth_CompletionSource:{row.DepthSource}"
+                ),
+            )
+            session.add(dp)
+
         """
-            Developer's note
+        Developer's note
 
-            It's not clear when the measuring point from NM_Aquifer was 
-            determined, so I'm setting start_date to the day of the transfer
+        It's not clear when the measuring point from NM_Aquifer was 
+        determined, so I'm setting start_date to the day of the transfer
         """
         measuring_point_history = MeasuringPointHistory(
             thing_id=well.id,