diff --git a/core/lexicon.json b/core/lexicon.json index 44a89d2aa..8ef165e1a 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -85,7 +85,7 @@ {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["well_construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, + {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"}, {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, @@ -578,6 +578,7 @@ {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, + {"categories": ["organization"], "term": "Quemado Municipal Water & SWA", "definition": "Quemado Municipal Water & SWA"}, {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, @@ -698,10 +699,6 @@ {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, - {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, - {"categories": ["well_pump_type"], "term": "Jet Pump", "definition": "Jet Pump"}, - {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, - {"categories": ["well_pump_type"], "term": "Hand Pump", "definition": "Hand Pump"}, {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, {"categories": ["aquifer_type"], "term": "Confined, single", "definition": "Confined single aquifer"}, {"categories": ["aquifer_type"], "term": "Confined, multiple", "definition": "Confined multiple aquifers"}, @@ -1105,6 +1102,10 @@ {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, + {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, + {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, + {"categories": ["well_pump_type"], "term": "Hand", "definition": "Hand Pump"}, {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} diff --git a/schemas/thing.py b/schemas/thing.py index 2b8967553..28b056c82 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -132,8 +132,15 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_height: float = Field( ge=0, description="Measuring point height in feet" ) - measuring_point_description: str | None + measuring_point_description: str | None = None notes: list[CreateNote] | None = None + well_completion_date: PastOrTodayDate | None = None + well_completion_date_source: str | None = None + well_driller_name: str | None = None + well_construction_method: WellConstructionMethod | None = None + well_construction_method_source: str | None = None + well_pump_type: WellPumpType | None = None + is_suitable_for_datalogger: bool | None class CreateSpring(CreateBaseThing): @@ -335,7 +342,7 @@ class WellScreenResponse(BaseResponseModel): aquifer_system: str | None = None aquifer_type: str | None = None geologic_formation_id: int | None = None - # geologic_formation: GeologicFormationResponse | None = None + geologic_formation: str | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -355,7 +362,7 @@ def populate_aquifer_type_with_name(cls, aquifer_type): return aquifer_type.name return None - @field_validator("geologic_formation_id", mode="before") + @field_validator("geologic_formation", mode="before") def populate_geologic_formation_with_code(cls, geologic_formation): if geologic_formation is not None: return geologic_formation.formation_code diff --git a/transfers/data/owners_organization_mapper.json b/transfers/data/owners_organization_mapper.json index 5ce45a8bf..b4f29bd7b 100644 --- a/transfers/data/owners_organization_mapper.json +++ b/transfers/data/owners_organization_mapper.json @@ -89,6 +89,7 @@ "Pecos Trail Inn": "Pecos Trail Inn", "Pelican Spa": "Pelican Spa", "Pistachio Tree Ranch": "Pistachio Tree Ranch", + "Quemado Mutual Water and Sewage Works Association": "Quemado Municipal Water & SWA", "Rancho Encantado": "Rancho Encantado", "Rancho San Lucas": "Rancho San Lucas", "Rancho San Marcos": "Rancho San Marcos", diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py new file mode 100644 index 000000000..1dcca2b8f --- /dev/null +++ b/transfers/permissions_transfer.py @@ -0,0 +1,87 @@ +from sqlalchemy.orm import Session +from datetime import datetime + +from db import Thing, PermissionHistory +from transfers.util import read_csv, logger, replace_nans + +""" +Developer's notes + +According to Laila the column WellData.OpenWellLoggerOK only pertains to the +physical properties of a well (that is, if a datalogger can be installed). It +does not pertain to permissions. +""" + + +def transfer_permissions(session: Session): + """ + The transferred wells and contacts need to be queried to know who gave + permission to which well since contact_id is required for PermissionHistory + """ + wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) + wdf = replace_nans(wdf) + + transferred_wells = ( + session.query(Thing).filter(Thing.thing_type == "water well").all() + ) + + for well in transferred_wells: + if len(well.contacts) == 0: + logger.critical( + f"Well {well.name} has no associated contacts; skipping permission transfer." + ) + continue + else: + # Assuming the first contact is the relevant one + contact_id = well.contacts[0].id + + allow_water_level_samples = wdf.loc[ + wdf["PointID"] == well.name, "MonitorOK" + ].values + if len(allow_water_level_samples) > 0 and allow_water_level_samples is not None: + try: + permission_allowed = bool(allow_water_level_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Level Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Level Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + allow_water_chemistry_samples = wdf.loc[ + wdf["PointID"] == well.name, "SampleOK" + ].values + if ( + len(allow_water_chemistry_samples) > 0 + and allow_water_chemistry_samples is not None + ): + try: + permission_allowed = bool(allow_water_chemistry_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Chemistry Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Chemistry Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + session.commit() diff --git a/transfers/transfer.py b/transfers/transfer.py index 77275ed35..2b576a4b2 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -36,6 +36,7 @@ transfer_wells, transfer_wellscreens, ) +from transfers.permissions_transfer import transfer_permissions from transfers.asset_transfer import transfer_assets from transfers.util import timeit, timeit_direct @@ -124,6 +125,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING ASSETS") timeit_direct(transfer_assets, sess) + message("TRANSFERRING PERMISSIONS") + results = timeit_direct(transfer_permissions, sess) + def transfer_debugging(sess, limit=100): message("STARTING TRANSFER DEBUG", new_line_at_top=False) diff --git a/transfers/util.py b/transfers/util.py index cbf0f2b17..9b2afc84c 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -275,7 +275,7 @@ def make_location_data_provenance( ) -> list[DataProvenance]: provenance_records = [] - if row.AltitudeAccuracy or row.CoordinateAccuracy: + if row.AltitudeAccuracy: provenance = DataProvenance( target_id=location.id, target_table="location", @@ -369,7 +369,6 @@ def make_location_data_provenance( target_id=location.id, target_table="location", field_name="point", - origin_source=None, collection_method=coordinate_method, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ee54d0216..9e684cc8d 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -36,6 +36,7 @@ StatusHistory, MonitoringFrequencyHistory, MeasuringPointHistory, + DataProvenance, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -117,6 +118,21 @@ def _extract_casing_materials(row) -> list[str]: return materials +def _extract_well_pump_type(row) -> str | None: + construction_notes = row.ConstructionNotes.lower() + if "pump" in construction_notes: + if "submersible" in construction_notes: + return "Submersible" + elif "jet" in construction_notes: + return "Jet" + elif "line shaft" in construction_notes or "lineshaft" in construction_notes: + return "Line Shaft" + elif "hand" in construction_notes: + return "Hand" + else: + return None + + def get_wells_to_transfer( sess: Session, flags: dict = None ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -236,6 +252,9 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) + well_pump_type = ( + _extract_well_pump_type(row) if row.ConstructionNotes else None + ) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation @@ -257,6 +276,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None notes=( [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=( + lexicon_mapper.map_value( + f"LU_ConstructionMethod:{row.ConstructionMethod}" + ) + if not isna(row.ConstructionMethod) + else None + ), + well_pump_type=well_pump_type, + is_suitable_for_datalogger=( + bool(row.OpenWellLoggerOK) + if not isna(row.OpenWellLoggerOK) + else None + ), ) CreateWell.model_validate(data) @@ -277,6 +311,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "well_casing_materials", "measuring_point_height", "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" @@ -285,17 +321,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_data.pop("notes") well = Thing(**well_data) session.add(well) - # logger.info(f"Created well for {row.PointID}") - - # flush well to access its ID for status_history - # session.flush() - - # session.commit() - # session.refresh(well) - # if notes: - # for ni in notes: - # nn = well.add_note(ni['content'], ni['note_type']) - # session.add(nn) if well_purposes: for wp in well_purposes: @@ -347,11 +372,42 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None for dp in data_provenances: session.add(dp) + if not isna(row.CompletionSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_completion_date", + origin_type=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.CompletionSource}" + ), + ) + session.add(dp) + + if not isna(row.DataSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_construction_method", + origin_source=row.DataSource, + ) + session.add(dp) + + if not isna(row.DepthSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_depth", + origin_type=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.DepthSource}" + ), + ) + session.add(dp) + """ - Developer's note + Developer's note - It's not clear when the measuring point from NM_Aquifer was - determined, so I'm setting start_date to the day of the transfer + It's not clear when the measuring point from NM_Aquifer was + determined, so I'm setting start_date to the day of the transfer """ measuring_point_history = MeasuringPointHistory( thing_id=well.id,