diff --git a/.env.example b/.env.example index 227db2d9d..cbf54e954 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ DB_DRIVER=postgres POSTGRES_USER=admin POSTGRES_PASSWORD=password POSTGRES_DB= +POSTGRES_PORT=54321 # asset storage GCS_BUCKET_NAME= @@ -14,6 +15,9 @@ MODE=development # disable authentication (for development only) AUTHENTIK_DISABLE_AUTHENTICATION=1 +# erase and rebuild the database for step tests +REBUILD_DB=1 + # authentik AUTHENTIK_URL= AUTHENTIK_CLIENT_ID= diff --git a/.gitignore b/.gitignore index 4bf6245e0..9a894e920 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ requirements.txt # VS Code +# macOS +.DS_Store + # local development files development.db .env diff --git a/api/well_inventory.py b/api/well_inventory.py index 68b9cb323..5f4b072ab 100644 --- a/api/well_inventory.py +++ b/api/well_inventory.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import csv +from datetime import date import logging import re from collections import Counter @@ -38,17 +39,15 @@ from core.dependencies import session_dependency, amp_editor_dependency from db import ( Group, - ThingIdLink, - GroupThingAssociation, Location, - LocationThingAssociation, - MeasuringPointHistory, DataProvenance, FieldEvent, FieldEventParticipant, + FieldActivity, Contact, + PermissionHistory, + Thing, ) -from db.thing import Thing, WellPurpose, MonitoringFrequencyHistory from schemas.thing import CreateWell from schemas.well_inventory import WellInventoryRow from services.contact_helper import add_contact @@ -59,7 +58,7 @@ router = APIRouter(prefix="/well-inventory-csv") -def _add_location(model, well) -> Location: +def _make_location(model) -> Location: point = Point(model.utm_easting, model.utm_northing) # TODO: this needs to be more sophisticated in the future. Likely more than 13N and 12N will be used @@ -79,11 +78,8 @@ def _add_location(model, well) -> Location: point=transformed_point.wkt, elevation=elevation_m, ) - date_time = model.date_time - assoc = LocationThingAssociation(location=loc, thing=well) - assoc.effective_start = date_time - return loc, assoc + return loc def _make_contact(model: WellInventoryRow, well: Thing, idx) -> dict: @@ -133,6 +129,43 @@ def _make_contact(model: WellInventoryRow, well: Thing, idx) -> dict: } +def _make_well_permission( + well: Thing, + contact: Contact | None, + permission_type: str, + permission_allowed: bool, + start_date: date, +) -> PermissionHistory: + """ + Makes a PermissionHistory record for the given well and contact. + If the contact has not been provided, but a permission is to be created, + no PermissionHistory record is created and a 400 error is raised. + """ + if contact is None: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "At least one contact required for permission", + "type": "Contact required for permission", + "input": None, + } + ], + ) + + permission = PermissionHistory( + target_table="thing", + target_id=well.id, + contact=contact, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=None, + ) + return permission + + AUTOGEN_REGEX = re.compile(r"^[A-Za-z]{2}-$") @@ -414,32 +447,130 @@ def _add_field_staff( def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) -> str: name = model.well_name_point_id date_time = model.date_time - site_name = model.site_name + + # -------------------- + # Location and associated tables + # -------------------- + + # add Location + loc = _make_location(model) + session.add(loc) + session.flush() + + # add location notes + if model.directions_to_site: + directions_note = loc.add_note( + content=model.directions_to_site, note_type="Directions" + ) + session.add(directions_note) + + # add data provenance records + dp = DataProvenance( + target_id=loc.id, + target_table="location", + field_name="elevation", + collection_method=model.elevation_method, + ) + session.add(dp) + + # -------------------- + # Thing and associated tables + # -------------------- # add Thing + well_notes = [] + for note_content, note_type in ( + (model.specific_location_of_well, "Access"), + (model.special_requests, "General"), + (model.well_measuring_notes, "Measuring"), + ): + if note_content is not None: + well_notes.append({"content": note_content, "note_type": note_type}) + + alternate_ids = [] + for alternate_id, alternate_organization in ( + (model.site_name, "NMBGMR"), + (model.ose_well_record_id, "NMOSE"), + ): + if alternate_id is not None: + alternate_ids.append( + { + "alternate_id": alternate_id, + "alternate_organization": alternate_organization, + "relation": "same_as", + } + ) + + well_purposes = [] + if model.well_purpose: + well_purposes.append(model.well_purpose) + if model.well_purpose_2: + well_purposes.append(model.well_purpose_2) + + monitoring_frequencies = [] + if model.monitoring_frequency: + monitoring_frequencies.append( + { + "monitoring_frequency": model.monitoring_frequency, + "start_date": date_time.date(), + } + ) + data = CreateWell( + location_id=loc.id, + group_id=group.id, name=name, first_visit_date=date_time.date(), well_depth=model.total_well_depth_ft, + well_depth_source=model.depth_source, well_casing_diameter=model.casing_diameter_ft, measuring_point_height=model.measuring_point_height_ft, measuring_point_description=model.measuring_point_description, + well_completion_date=model.date_drilled, + well_completion_date_source=model.completion_source, + well_pump_type=model.well_pump_type, + well_pump_depth=model.well_pump_depth_ft, + is_suitable_for_datalogger=model.datalogger_possible, + notes=well_notes, + well_purposes=well_purposes, ) well_data = data.model_dump( exclude=[ - "location_id", - "group_id", "well_purposes", "well_casing_materials", - "measuring_point_height", - "measuring_point_description", ] ) + + """ + Developer's notes + + the add_thing function also handles: + - MeasuringPointHistory + - GroupThingAssociation + - LocationThingAssociation + - DataProvenance for well_completion_date + - DataProvenance for well_construction_method + - DataProvenance for well_depth + - Notes + - WellPurpose + - MonitoringFrequencyHistory + """ well = add_thing( session=session, data=well_data, user=user, thing_type="water well" ) session.refresh(well) + # ------------------ + # Field Events and related tables + # ------------------ + """ + Developer's notes + + These tables are not handled in add_thing because they are only relevant if + the well has been inventoried in the field, not if the well is added from + another source like a report, database, or map. + """ + # add field event fe = FieldEvent( event_date=date_time, @@ -459,64 +590,48 @@ def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) _add_field_staff(session, fsi, fe, role, user) - # add MonitoringFrequency - if model.monitoring_frequency: - mfh = MonitoringFrequencyHistory( - thing=well, - monitoring_frequency=model.monitoring_frequency, - start_date=date_time.date(), - ) - session.add(mfh) - - # add WellPurpose - for p in (model.well_purpose, model.well_purpose_2): - if not p: - continue - wp = WellPurpose(purpose=p, thing=well) - session.add(wp) - - # BDMS-221 adds MeasuringPointHistory model - measuring_point_height_ft = model.measuring_point_height_ft - if measuring_point_height_ft: - mph = MeasuringPointHistory( - thing=well, - measuring_point_height=measuring_point_height_ft, - measuring_point_description=model.measuring_point_description, - start_date=date_time.date(), - ) - session.add(mph) - - # add Location - loc, assoc = _add_location(model, well) - session.add(loc) - session.add(assoc) - session.flush() - - dp = DataProvenance( - target_id=loc.id, - target_table="location", - field_name="elevation", - collection_method=model.elevation_method, + # add field activity + fa = FieldActivity( + field_event=fe, + activity_type="well inventory", + notes="Well inventory conducted during field event.", ) - session.add(dp) - - gta = GroupThingAssociation(group=group, thing=well) - session.add(gta) - group.thing_associations.append(gta) + session.add(fa) - # add alternate ids - well.links.append( - ThingIdLink( - alternate_id=site_name, - alternate_organization="NMBGMR", - relation="same_as", - ) - ) + # ------------------ + # Contacts + # ------------------ + # add contacts + contact_for_permissions = None for idx in (1, 2): - contact = _make_contact(model, well, idx) - if contact: - add_contact(session, contact, user=user) + contact_dict = _make_contact(model, well, idx) + if contact_dict: + contact = add_contact(session, contact_dict, user=user) + + # Use the first created contact for permissions if available + if contact_for_permissions is None: + contact_for_permissions = contact + + # ------------------ + # Permissions + # ------------------ + + # add permissions + for permission_type, permission_allowed in ( + ("Water Level Sample", model.repeat_measurement_permission), + ("Water Chemistry Sample", model.sampling_permission), + ("Datalogger Installation", model.datalogger_installation_permission), + ): + if permission_allowed is not None: + permission = _make_well_permission( + well=well, + contact=contact_for_permissions, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=model.date_time.date(), + ) + session.add(permission) return model.well_name_point_id diff --git a/core/enums.py b/core/enums.py index 91b206cab..a2c73f521 100644 --- a/core/enums.py +++ b/core/enums.py @@ -50,7 +50,7 @@ MonitoringStatus: type[Enum] = build_enum_from_lexicon_category("monitoring_status") ParameterName: type[Enum] = build_enum_from_lexicon_category("parameter_name") Organization: type[Enum] = build_enum_from_lexicon_category("organization") -OriginSource: type[Enum] = build_enum_from_lexicon_category("origin_source") +OriginType: type[Enum] = build_enum_from_lexicon_category("origin_type") ParameterType: type[Enum] = build_enum_from_lexicon_category("parameter_type") PhoneType: type[Enum] = build_enum_from_lexicon_category("phone_type") PublicationType: type[Enum] = build_enum_from_lexicon_category("publication_type") diff --git a/core/lexicon.json b/core/lexicon.json index 0d14be5ac..85378e759 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -53,7 +53,7 @@ {"name": "well_purpose", "description": null}, {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, - {"name": "origin_source", "description": null}, + {"name": "origin_type", "description": null}, {"name": "well_pump_type", "description": null}, {"name": "permission_type", "description": null}, {"name": "formation_code", "description": null}, @@ -284,6 +284,7 @@ {"categories": ["relation"], "term": "OSEWellTagID", "definition": "NM OSE well tag ID"}, {"categories": ["relation"], "term": "OSEPOD", "definition": "NM OSE 'Point of Diversion' ID"}, {"categories": ["relation"], "term": "PLSS", "definition": "Public Land Survey System ID"}, + {"categories": ["activity_type"], "term": "well inventory", "definition": "well inventory"}, {"categories": ["activity_type"], "term": "groundwater level", "definition": "groundwater level"}, {"categories": ["activity_type"], "term": "water chemistry", "definition": "water chemistry"}, {"categories": ["participant_role"], "term": "Lead", "definition": "the leader of the field event"}, @@ -1151,18 +1152,19 @@ {"categories": ["lithology"],"term": "Ignesous, intrusive, undifferentiated","definition": "Ignesous, intrusive, undifferentiated"}, {"categories": ["lithology"],"term": "Limestone, sandstone and shale","definition": "Limestone, sandstone and shale"}, {"categories": ["lithology"],"term": "Sand, silt and clay","definition": "Sand, silt and clay"}, - {"categories": ["origin_source"], "term": "Reported by another agency", "definition": "Reported by another agency"}, - {"categories": ["origin_source"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, - {"categories": ["origin_source"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, - {"categories": ["origin_source"], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency"}, - {"categories": ["origin_source"], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller"}, - {"categories": ["origin_source"], "term": "Measured by source agency", "definition": "Measured by source agency"}, - {"categories": ["origin_source"], "term": "Reported by owner of well", "definition": "Reported by owner of well"}, - {"categories": ["origin_source"], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency"}, - {"categories": ["origin_source"], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff"}, - {"categories": ["origin_source"], "term": "Other", "definition": "Other"}, - {"categories": ["origin_source"], "term": "Data Portal", "definition": "Data Portal"}, + {"categories": ["origin_type"], "term": "Reported by another agency", "definition": "Reported by another agency"}, + {"categories": ["origin_type"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, + {"categories": ["origin_type"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, + {"categories": ["origin_type"], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency"}, + {"categories": ["origin_type"], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller"}, + {"categories": ["origin_type"], "term": "Measured by source agency", "definition": "Measured by source agency"}, + {"categories": ["origin_type"], "term": "Reported by owner of well", "definition": "Reported by owner of well"}, + {"categories": ["origin_type"], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency"}, + {"categories": ["origin_type"], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff"}, + {"categories": ["origin_type"], "term": "Other", "definition": "Other"}, + {"categories": ["origin_type"], "term": "Data Portal", "definition": "Data Portal"}, {"categories": ["note_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, + {"categories": ["note_type"], "term": "Directions", "definition": "Notes about directions to a location"}, {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, diff --git a/docker-compose.yml b/docker-compose.yml index 1c6dec4ef..30d22b9d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_HOST=db + - POSTGRES_PORT=5432 - MODE=${MODE} - AUTHENTIK_DISABLE_AUTHENTICATION=${AUTHENTIK_DISABLE_AUTHENTICATION} ports: diff --git a/schemas/thing.py b/schemas/thing.py index 0ccf80376..eae9191d3 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -27,6 +27,7 @@ WellConstructionMethod, WellPumpType, FormationCode, + OriginType, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate from schemas.group import GroupResponse @@ -43,6 +44,7 @@ class ValidateWell(BaseModel): hole_depth: float | None = None # in feet well_casing_depth: float | None = None # in feet measuring_point_height: float | None = None + well_pump_depth: float | None = None # in feet @model_validator(mode="after") def validate_values(self): @@ -59,6 +61,12 @@ def validate_values(self): "well casing depth must be less than or equal to hole depth" ) + if self.well_pump_depth is not None: + if self.well_depth is not None and self.well_pump_depth > self.well_depth: + raise ValueError("well pump depth must be less than well depth") + elif self.hole_depth is not None and self.well_pump_depth > self.hole_depth: + raise ValueError("well pump depth must be less than hole depth") + # if self.measuring_point_height is not None: # if ( # self.hole_depth is not None @@ -107,6 +115,21 @@ class CreateBaseThing(BaseCreateModel): group_id: int | None = None # Optional group ID for the thing name: str # Name of the thing first_visit_date: PastOrTodayDate | None = None # Date of NMBGMR's first visit + notes: list[CreateNote] | None = None + alternate_ids: list[CreateThingIdLink] | None = None + monitoring_frequencies: list[MonitoringFrequency] | None = None + + @field_validator("alternate_ids", mode="before") + def use_dummy_values(cls, v): + """ + When alternate IDs are provided they are assumed to be the same as + the thing being created. This gets handled in the function services/thing_helper.py::add_thing. + By using dummy values here we can avoid validation errors and then use the + thing's id when creating the actual links. + """ + for alternate_id in v: + alternate_id.thing_id = -1 # dummy value + return v class CreateWell(CreateBaseThing, ValidateWell): @@ -118,6 +141,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_depth: float | None = Field( default=None, gt=0, description="Well depth in feet" ) + well_depth_source: OriginType | None = None hole_depth: float | None = Field( default=None, gt=0, description="Hole depth in feet" ) @@ -128,16 +152,15 @@ class CreateWell(CreateBaseThing, ValidateWell): default=None, gt=0, description="Well casing depth in feet" ) well_casing_materials: list[CasingMaterial] | None = None - measuring_point_height: float = Field(description="Measuring point height in feet") measuring_point_description: str | None = None - notes: list[CreateNote] | None = None well_completion_date: PastOrTodayDate | None = None well_completion_date_source: str | None = None well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None + well_pump_depth: float | None = None is_suitable_for_datalogger: bool | None formation_completion_code: FormationCode | None = None diff --git a/schemas/well_inventory.py b/schemas/well_inventory.py index fb0d6c76f..0524baea6 100644 --- a/schemas/well_inventory.py +++ b/schemas/well_inventory.py @@ -216,17 +216,17 @@ class WellInventoryRow(BaseModel): repeat_measurement_permission: OptionalBool = None sampling_permission: OptionalBool = None datalogger_installation_permission: OptionalBool = None - public_availability_acknowledgement: OptionalBool = None + public_availability_acknowledgement: OptionalBool = None # TODO: needs a home special_requests: Optional[str] = None ose_well_record_id: Optional[str] = None date_drilled: OptionalDateTime = None completion_source: Optional[str] = None total_well_depth_ft: OptionalFloat = None - historic_depth_to_water_ft: OptionalFloat = None + historic_depth_to_water_ft: OptionalFloat = None # TODO: needs a home depth_source: Optional[str] = None well_pump_type: Optional[str] = None well_pump_depth_ft: OptionalFloat = None - is_open: OptionalBool = None + is_open: OptionalBool = None # TODO: needs a home datalogger_possible: OptionalBool = None casing_diameter_ft: OptionalFloat = None measuring_point_description: Optional[str] = None @@ -235,11 +235,11 @@ class WellInventoryRow(BaseModel): well_hole_status: Optional[str] = None monitoring_frequency: MonitoryFrequencyField = None - result_communication_preference: Optional[str] = None - contact_special_requests_notes: Optional[str] = None - sampling_scenario_notes: Optional[str] = None + result_communication_preference: Optional[str] = None # TODO: needs as home + contact_special_requests_notes: Optional[str] = None # TODO: needs a home + sampling_scenario_notes: Optional[str] = None # TODO: needs a home well_measuring_notes: Optional[str] = None - sample_possible: OptionalBool = None + sample_possible: OptionalBool = None # TODO: needs a home @model_validator(mode="after") def validate_model(self): diff --git a/services/thing_helper.py b/services/thing_helper.py index 100b49994..ec4e330d5 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -35,6 +35,9 @@ ThingAquiferAssociation, GroupThingAssociation, MeasuringPointHistory, + DataProvenance, + ThingIdLink, + MonitoringFrequencyHistory, ) from services.audit_helper import audit_add @@ -48,7 +51,7 @@ "well_casing_materials": (WellCasingMaterial, "material"), } -WELL_LOADER_OPTIONS = [ +WATER_WELL_LOADER_OPTIONS = [ selectinload(Thing.location_associations).selectinload( LocationThingAssociation.location ), @@ -62,7 +65,7 @@ ), ] -WELL_THING_TYPE = "water well" +WATER_WELL_THING_TYPE = "water well" def wkb_to_geojson(wkb_element): @@ -91,11 +94,11 @@ def get_db_things( if thing_type: sql = sql.where(Thing.thing_type == thing_type) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) else: # add all eager loads for generic thing query until/unless GET /thing is deprecated - sql = sql.options(*WELL_LOADER_OPTIONS) + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) if name: sql = sql.where(Thing.name == name) @@ -160,8 +163,8 @@ def get_thing_of_a_thing_type_by_id(session: Session, request: Request, thing_id thing_type = get_thing_type_from_request(request) sql = select(Thing).where(Thing.id == thing_id) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) thing = session.execute(sql).scalar_one_or_none() @@ -186,21 +189,44 @@ def add_thing( if request is not None: thing_type = get_thing_type_from_request(request) - if isinstance(data, BaseModel): - well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) - data = data.model_dump(exclude=well_descriptor_table_list) + # Extract data for related tables - notes = None - if "notes" in data: - notes = data.pop("notes") + # --------- + # BEGIN UNIVERSAL THING RELATED TABLES + # --------- + notes = data.pop("notes", None) + alternate_ids = data.pop("alternate_ids", None) location_id = data.pop("location_id", None) + effective_start = data.get("first_visit_date") group_id = data.pop("group_id", None) + monitoring_frequencies = data.pop("monitoring_frequencies", None) - # Extract measuring point data (stored in separate history table, not as Thing columns) + # ---------- + # END UNIVERSAL THING RELATED TABLES + # ---------- + + # ---------- + # BEGIN WATER WELL SPECIFIC RELATED TABLES + # ---------- + + # measuring point info measuring_point_height = data.pop("measuring_point_height", None) measuring_point_description = data.pop("measuring_point_description", None) + # data provenance info + well_completion_date_source = data.pop("well_completion_date_source", None) + well_construction_method_source = data.pop("well_construction_method_source", None) + well_depth_source = data.pop("well_depth_source", None) + + # descriptor tables + well_purposes = data.pop("well_purposes", None) + well_casing_materials = data.pop("well_casing_materials", None) + + # ---------- + # END WATER WELL SPECIFIC RELATED TABLES + # ---------- + try: thing = Thing(**data) thing.thing_type = thing_type @@ -211,17 +237,73 @@ def add_thing( session.flush() session.refresh(thing) - # Create MeasuringPointHistory record if measuring_point_height provided - if measuring_point_height is not None: - measuring_point_history = MeasuringPointHistory( - thing_id=thing.id, - measuring_point_height=measuring_point_height, - measuring_point_description=measuring_point_description, - start_date=datetime.now(tz=ZoneInfo("UTC")), - end_date=None, - ) - audit_add(user, measuring_point_history) - session.add(measuring_point_history) + # ---------- + # BEING WATER WELL SPECIFIC LOGIC + # ---------- + + if thing_type == WATER_WELL_THING_TYPE: + + # Create MeasuringPointHistory record if measuring_point_height provided + if measuring_point_height is not None: + measuring_point_history = MeasuringPointHistory( + thing_id=thing.id, + measuring_point_height=measuring_point_height, + measuring_point_description=measuring_point_description, + start_date=datetime.now(tz=ZoneInfo("UTC")), + end_date=None, + ) + audit_add(user, measuring_point_history) + session.add(measuring_point_history) + + if well_completion_date_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_completion_date", + origin_type=well_completion_date_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_depth_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_depth", + origin_type=well_depth_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_construction_method_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_construction_method", + origin_source=well_construction_method_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_purposes: + for purpose in well_purposes: + wp = WellPurpose(thing_id=thing.id, purpose=purpose) + audit_add(user, wp) + session.add(wp) + + if well_casing_materials: + for material in well_casing_materials: + wcm = WellCasingMaterial(thing_id=thing.id, material=material) + audit_add(user, wcm) + session.add(wcm) + + # ---------- + # END WATER WELL SPECIFIC LOGIC + # ---------- + + # ---------- + # BEGIN UNIVERSAL THING RELATED LOGIC + # ---------- # endpoint catches ProgrammingError if location_id or group_id do not exist if group_id: @@ -232,23 +314,47 @@ def add_thing( session.add(assoc) if location_id is not None: - # TODO: how do we want to handle effective_start? is it the date it gets entered? assoc = LocationThingAssociation() audit_add(user, assoc) assoc.location_id = location_id assoc.thing_id = thing.id + assoc.effective_start = effective_start session.add(assoc) - session.commit() - session.refresh(thing) - if notes: for n in notes: - nn = thing.add_note(n["content"], n["note_type"]) - session.add(nn) + thing_note = thing.add_note(n["content"], n["note_type"]) + session.add(thing_note) session.commit() session.refresh(thing) + if alternate_ids: + for aid in alternate_ids: + id_link = ThingIdLink( + thing_id=thing.id, + relation=aid["relation"], + alternate_id=aid["alternate_id"], + alternate_organization=aid["alternate_organization"], + ) + session.add(id_link) + + if monitoring_frequencies: + for mf in monitoring_frequencies: + mfh = MonitoringFrequencyHistory( + thing_id=thing.id, + monitoring_frquency=mf["monitoring_frequency"], + start_date=mf["start_date"], + end_date=mf.get("end_date", None), + ) + session.add(mfh) + + # ---------- + # END UNIVERSAL THING RELATED LOGIC + # ---------- + + session.commit() + session.refresh(thing) + except Exception as e: session.rollback() raise e diff --git a/tests/features/data/well-inventory-valid-comma-in-quotes.csv b/tests/features/data/well-inventory-valid-comma-in-quotes.csv index 7c1f2b28a..f347e0aef 100644 --- a/tests/features/data/well-inventory-valid-comma-in-quotes.csv +++ b/tests/features/data/well-inventory-valid-comma-in-quotes.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1D,"""Smith Farm, Domestic Well""",2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith T,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1G,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis E,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1D,"""Smith Farm, Domestic Well""",2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith T,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1G,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis E,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid-extra-columns.csv b/tests/features/data/well-inventory-valid-extra-columns.csv index 160ab9cc4..6b9eee613 100644 --- a/tests/features/data/well-inventory-valid-extra-columns.csv +++ b/tests/features/data/well-inventory-valid-extra-columns.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible,extra_column1,extract_column2 -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1v,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith B,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia V,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,, -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1f,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis B,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,, +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1v,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith B,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia V,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,, +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1f,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis B,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,, diff --git a/tests/features/data/well-inventory-valid-reordered.csv b/tests/features/data/well-inventory-valid-reordered.csv index 034c3c6a4..31427ab20 100644 --- a/tests/features/data/well-inventory-valid-reordered.csv +++ b/tests/features/data/well-inventory-valid-reordered.csv @@ -1,3 +1,3 @@ well_name_point_id,project,site_name,date_time,field_staff,utm_northing,utm_easting,utm_zone,elevation_method,elevation_ft,field_staff_2,measuring_point_height_ft,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -MRG-001_MP12,Middle Rio Grande Groundwater Monitoring,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,4000000,250000,13N,Survey-grade GPS,5250,B Chen,1.5,,John Smith A,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia A,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -MRG-003_MP12,Middle Rio Grande Groundwater Monitoring,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,4000000,250000,13N,Global positioning system (GPS),5320,,1.8,,Emily Davis A,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +MRG-001_MP12,Middle Rio Grande Groundwater Monitoring,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,4000000,250000,13N,Survey-grade GPS,5250,B Chen,1.5,,John Smith A,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia A,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +MRG-003_MP12,Middle Rio Grande Groundwater Monitoring,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,4000000,250000,13N,Global positioning system (GPS),5320,,1.8,,Emily Davis A,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid.csv b/tests/features/data/well-inventory-valid.csv index 7bcb39f71..18cdcddc6 100644 --- a/tests/features/data/well-inventory-valid.csv +++ b/tests/features/data/well-inventory-valid.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/well-inventory-csv.feature b/tests/features/well-inventory-csv.feature new file mode 100644 index 000000000..f7738960d --- /dev/null +++ b/tests/features/well-inventory-csv.feature @@ -0,0 +1,452 @@ +@backend +@BDMS-TBD +@production +Feature: Bulk upload well inventory from CSV + As a hydrogeologist or data specialist + I want to upload a CSV file containing well inventory data for multiple wells + So that well records can be created efficiently and accurately in the system + + Background: + Given a functioning api + And valid lexicon values exist for: + | lexicon category | + | contact_role | + | contact_type | + | phone_type | + | email_type | + | address_type | + | elevation_method | + | well_pump_type | + | well_purpose | + | well_hole_status | + | monitoring_frequency | + + @positive @happy_path @BDMS-TBD + Scenario: Uploading a valid well inventory CSV containing required and optional fields + Given a valid CSV file for bulk well inventory upload + And my CSV file is encoded in UTF-8 and uses commas as separators + And my CSV file contains multiple rows of well inventory data + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + And each "well_name_point_id" value is unique per row + And "date_time" values are valid ISO 8601 timestamps with timezone offsets (e.g. "2025-02-15T10:30:00-08:00") + And the CSV includes optional fields when available: + | optional field name | + | field_staff_2 | + | field_staff_3 | + | contact_1_name | + | contact_1_organization | + | contact_1_role | + | contact_1_type | + | contact_1_phone_1 | + | contact_1_phone_1_type | + | contact_1_phone_2 | + | contact_1_phone_2_type | + | contact_1_email_1 | + | contact_1_email_1_type | + | contact_1_email_2 | + | contact_1_email_2_type | + | contact_1_address_1_line_1 | + | contact_1_address_1_line_2 | + | contact_1_address_1_type | + | contact_1_address_1_state | + | contact_1_address_1_city | + | contact_1_address_1_postal_code | + | contact_1_address_2_line_1 | + | contact_1_address_2_line_2 | + | contact_1_address_2_type | + | contact_1_address_2_state | + | contact_1_address_2_city | + | contact_1_address_2_postal_code | + | contact_2_name | + | contact_2_organization | + | contact_2_role | + | contact_2_type | + | contact_2_phone_1 | + | contact_2_phone_1_type | + | contact_2_phone_2 | + | contact_2_phone_2_type | + | contact_2_email_1 | + | contact_2_email_1_type | + | contact_2_email_2 | + | contact_2_email_2_type | + | contact_2_address_1_line_1 | + | contact_2_address_1_line_2 | + | contact_2_address_1_type | + | contact_2_address_1_state | + | contact_2_address_1_city | + | contact_2_address_1_postal_code | + | contact_2_address_2_line_1 | + | contact_2_address_2_line_2 | + | contact_2_address_2_type | + | contact_2_address_2_state | + | contact_2_address_2_city | + | contact_2_address_2_postal_code | + | directions_to_site | + | specific_location_of_well | + | repeat_measurement_permission | + | sampling_permission | + | datalogger_installation_permission | + | public_availability_acknowledgement | + | result_communication_preference | + | contact_special_requests_notes | + | ose_well_record_id | + | date_drilled | + | completion_source | + | total_well_depth_ft | + | historic_depth_to_water_ft | + | depth_source | + | well_pump_type | + | well_pump_depth_ft | + | is_open | + | datalogger_possible | + | casing_diameter_ft | + | measuring_point_description | + | well_purpose | + | well_purpose_2 | + | well_hole_status | + | monitoring_frequency | + | sampling_scenario_notes | + | well_measuring_notes | + | sample_possible | +# And all optional lexicon fields contain valid lexicon values when provided +# And all optional numeric fields contain valid numeric values when provided +# And all optional date fields contain valid ISO 8601 timestamps when provided + + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format +# And null values in the response are represented as JSON null + And the response includes a summary containing: + | summary_field | value | + | total_rows_processed | 2 | + | total_rows_imported | 2 | + | validation_errors_or_warnings | 0 | + And the response includes an array of created well objects + + @positive @validation @column_order @BDMS-TBD + Scenario: Upload succeeds when required columns are present but in a different order + Given my CSV file contains all required headers but in a different column order + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @extra_columns @BDMS-TBD + Scenario: Upload succeeds when CSV contains extra, unknown columns + Given my CSV file contains extra columns but is otherwise valid + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @autogenerate_ids @BDMS-TBD + Scenario: Upload succeeds and system auto-generates well_name_point_id when prefixed with "XY- + Given my CSV file contains all valid columns but uses "XY-" prefix for well_name_point_id values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported with system-generated unique well_name_point_id values + + ########################################################################### + # NEGATIVE VALIDATION SCENARIOS + ########################################################################### + @negative @validation @transactional_import @BDMS-TBD + Scenario: No wells are imported when any row fails validation + Given my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the row missing "well_name_point_id" + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has an invalid postal code format + Given my CSV file contains a row that has an invalid postal code format in contact_1_address_1_postal_code + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid postal code format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid phone number format + Given my CSV file contains a row with a contact with a phone number that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid phone number format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid email format + Given my CSV file contains a row with a contact with an email that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid email format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a contact_role + Given my CSV file contains a row with a contact but is missing the required "contact_role" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_role" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a "contact_type" + Given my CSV file contains a row with a contact but is missing the required "contact_type" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an invalid "contact_type" + Given my CSV file contains a row with a contact_type value that is not in the valid lexicon for "contact_type" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an email without an email_type + Given my CSV file contains a row with a contact with an email but is missing the required "email_type" field for that email + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "email_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with a phone without a phone_type + Given my CSV file contains a row with a contact with a phone but is missing the required "phone_type" field for that phone + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "phone_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an address without an address_type + Given my CSV file contains a row with a contact with an address but is missing the required "address_type" field for that address + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "address_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has utm_easting utm_northing and utm_zone values that are not within New Mexico + Given my CSV file contains a row with utm_easting utm_northing and utm_zone values that are not within New Mexico + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid UTM coordinates + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when required fields are missing + Given my CSV file contains rows missing a required field "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes validation errors for all rows missing required fields + And the response identifies the row and field for each error + And no wells are imported + + @negative @validation @required_fields @BDMS-TBD + Scenario Outline: Upload fails when a required field is missing + Given my CSV file contains a row missing the required "" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the "" field + And no wells are imported + + Examples: + | required_field | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + + @negative @validation @boolean_fields @BDMS-TBD + Scenario: Upload fails due to invalid boolean field values + Given my CSV file contains a row with an invalid boolean value "maybe" in the "is_open" field +# And my CSV file contains other boolean fields such as "sample_possible" with valid boolean values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid boolean value for the "is_open" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when duplicate well_name_point_id values are present + Given my CSV file contains one or more duplicate "well_name_point_id" values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors indicating duplicated values + And each error identifies the row and field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid lexicon values + Given my CSV file contains invalid lexicon values for "contact_role" or other lexicon fields + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid date formats + Given my CSV file contains invalid ISO 8601 date values in the "date_time" or "date_drilled" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid numeric fields + Given my CSV file contains values that cannot be parsed as numeric in numeric-required fields such as "utm_easting" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + +# ########################################################################### +# # FILE FORMAT SCENARIOS +# ########################################################################### + + @negative @file_format @limits @BDMS-TBD + Scenario: Upload fails when the CSV exceeds the maximum allowed number of rows + Given my CSV file contains more rows than the configured maximum for bulk upload + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating the row limit was exceeded + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when file type is unsupported + Given I have a non-CSV file + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating unsupported file type + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when the CSV file is empty + Given my CSV file is empty + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating an empty file + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when CSV contains only headers + Given my CSV file contains column headers but no data rows + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error indicating that no data rows were found + And no wells are imported + + ########################################################################### + # HEADER & SCHEMA INTEGRITY SCENARIOS + ########################################################################### + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when a header row is repeated in the middle of the file + Given my CSV file contains a valid but duplicate header row + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating a repeated header row + And no wells are imported + + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when the header row contains duplicate column names + Given my CSV file header row contains the "contact_1_email_1" column name more than once + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating duplicate header names + And no wells are imported + + + ########################################################################### + # DELIMITER & QUOTING / EXCEL-RELATED SCENARIOS + ########################################################################### + + @negative @file_format @delimiter @BDMS-TBD + Scenario Outline: Upload fails when CSV uses an unsupported delimiter + Given my file is named with a .csv extension + And my file uses "" as the field delimiter instead of commas + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating an unsupported delimiter + And no wells are imported + + Examples: + | delimiter_description | + | semicolons | + | tab characters | + + @positive @file_format @quoting @BDMS-TBD + Scenario: Upload succeeds when fields contain commas inside properly quoted values + Given my CSV file header row contains all required columns + And my CSV file contains a data row where the "site_name" field value includes a comma and is enclosed in quotes +# And all other required fields are populated with valid values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported +# +# @negative @validation @numeric @excel @BDMS-TBD +# Scenario: Upload fails when numeric fields are provided in Excel scientific notation format +# Given my CSV file contains a numeric-required field such as "utm_easting" +# And Excel has exported the "utm_easting" value in scientific notation (for example "1.2345E+06") +# When I upload the file to the bulk upload endpoint +# Then the system returns a 422 Unprocessable Entity status code +# And the system should return a response in JSON format +# And the response includes a validation error indicating an invalid numeric format for "utm_easting" +# And no wells are imported \ No newline at end of file diff --git a/tests/test_transfer_legacy_dates.py b/tests/test_transfer_legacy_dates.py index f871acb3b..7119c839f 100644 --- a/tests/test_transfer_legacy_dates.py +++ b/tests/test_transfer_legacy_dates.py @@ -22,6 +22,7 @@ """ import datetime from unittest.mock import patch + import pandas as pd import pytest diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index aaa2eb0bd..91c388fb1 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -339,12 +339,18 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): "measuring_point_description", "well_completion_date_source", "well_construction_method_source", + "well_depth_source", + "alternate_ids", + "monitoring_frequencies", + "notes", + "well_depth_source", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID - well_data.pop("notes") well = Thing(**well_data) session.add(well)