diff --git a/core/enums.py b/core/enums.py index 568f3f96a..91b206cab 100644 --- a/core/enums.py +++ b/core/enums.py @@ -24,7 +24,9 @@ ) CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material") CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method") -ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method") +WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category( + "well_construction_method" +) ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type") CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method") WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose") @@ -68,8 +70,14 @@ Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum") ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type") SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type") +WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type") +PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type") GroupType: type[Enum] = build_enum_from_lexicon_category("group_type") MonitoringFrequency: type[Enum] = build_enum_from_lexicon_category( "monitoring_frequency" ) +AquiferType: type[Enum] = build_enum_from_lexicon_category("aquifer_type") +GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale") +Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") +FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code") # ============= EOF ============================================= diff --git a/core/formations.json b/core/formations.json new file mode 100644 index 000000000..e69de29bb diff --git a/core/lexicon.json b/core/lexicon.json index 9aa9b88ae..142f1745c 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2,9 +2,10 @@ {"name": "activity_type", "description": null}, {"name": "address_type", "description": null}, {"name": "analysis_method_type", "description": null}, + {"name": "aquifer_type", "description": null}, {"name": "casing_material", "description": null}, {"name": "collection_method", "description": null}, - {"name": "construction_method", "description": null}, + {"name": "well_construction_method", "description": null}, {"name": "contact_type", "description": null}, {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, @@ -18,6 +19,7 @@ {"name": "email_type", "description": null}, {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, + {"name": "geographic_scale", "description": null}, {"name": "groundwater_level_reason", "description": null}, {"name": "group_type", "description": null}, {"name": "horizontal_datum", "description": null}, @@ -51,7 +53,11 @@ {"name": "well_purpose", "description": null}, {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, - {"name": "origin_source", "description": null} + {"name": "origin_source", "description": null}, + {"name": "well_pump_type", "description": null}, + {"name": "permission_type", "description": null}, + {"name": "formation_code", "description": null}, + {"name": "lithology", "description": null} ], "terms": [ {"categories": ["review_status"], "term": "approved", "definition": "approved"}, @@ -78,15 +84,15 @@ {"categories": ["elevation_method"], "term": "Reported", "definition": "Reported"}, {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, - {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, - {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, - {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, - {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, - {"categories": ["construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, - {"categories": ["construction_method"], "term": "Air percussion", "definition": "Air percussion"}, - {"categories": ["construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, - {"categories": ["construction_method"], "term": "Driven", "definition": "Driven"}, - {"categories": ["construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, + {"categories": ["elevation_method", "sample_method", "coordinate_method", "well_purpose", "status", "organization", "role", "aquifer_type"], "term": "Unknown", "definition": "Unknown"}, + {"categories": ["well_construction_method"], "term": "Air-Rotary", "definition": "Air-Rotary"}, + {"categories": ["well_construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, + {"categories": ["well_construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, + {"categories": ["well_construction_method"], "term": "Hydraulic rotary (mud or water)", "definition": "Hydraulic rotary (mud or water)"}, + {"categories": ["well_construction_method"], "term": "Air percussion", "definition": "Air percussion"}, + {"categories": ["well_construction_method"], "term": "Reverse rotary", "definition": "Reverse rotary"}, + {"categories": ["well_construction_method"], "term": "Driven", "definition": "Driven"}, + {"categories": ["well_construction_method", "measurement_method"], "term": "Other (explain in notes)", "definition": "Other (explain in notes)"}, {"categories": ["coordinate_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["coordinate_method"], "term": "Survey-grade global positioning system (SGPS)", "definition": "Survey-grade global positioning system (SGPS)"}, {"categories": ["coordinate_method"], "term": "GPS, uncorrected", "definition": "GPS, uncorrected"}, @@ -572,6 +578,7 @@ {"categories": ["organization"], "term": "Yates Petroleum Corporation", "definition": "Yates Petroleum Corporation"}, {"categories": ["organization"], "term": "Zamora Accounting Services", "definition": "Zamora Accounting Services"}, {"categories": ["organization"], "term": "PLSS", "definition": "Public Land Survey System"}, + {"categories": ["organization"], "term": "Quemado Municipal Water & SWA", "definition": "Quemado Municipal Water & SWA"}, {"categories": ["collection_method"], "term": "Altimeter", "definition": "ALtimeter"}, {"categories": ["collection_method"], "term": "Differentially corrected GPS", "definition": "Differentially corrected GPS"}, {"categories": ["collection_method"], "term": "Survey-grade GPS", "definition": "Survey-grade GPS"}, @@ -692,6 +699,392 @@ {"categories": ["monitoring_frequency"], "term": "Annual", "definition": "Location is monitored once a year."}, {"categories": ["monitoring_frequency"], "term": "Decadal", "definition": "Location is monitored once every ten years."}, {"categories": ["monitoring_frequency"], "term": "Event-based", "definition": "Location is monitored based on specific events or triggers rather than a fixed schedule."}, + {"categories": ["aquifer_type"], "term": "Artesian", "definition": "Artesian"}, + {"categories": ["aquifer_type"], "term": "Confined single aquifer", "definition": "Confined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Unsaturated (dry)", "definition": "Unsaturated (dry)"}, + {"categories": ["aquifer_type"], "term": "Fractured", "definition": "Fractured"}, + {"categories": ["aquifer_type"], "term": "Confined multiple aquifers", "definition": "Confined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Unconfined multiple aquifers", "definition": "Unconfined multiple aquifers"}, + {"categories": ["aquifer_type"], "term": "Perched aquifer", "definition": "Perched aquifer"}, + {"categories": ["aquifer_type"], "term": "Confining layer or aquitard", "definition": "Confining layer or aquitard"}, + {"categories": ["aquifer_type"], "term": "Semi-confined", "definition": "Semi-confined"}, + {"categories": ["aquifer_type"], "term": "Unconfined single aquifer", "definition": "Unconfined single aquifer"}, + {"categories": ["aquifer_type"], "term": "Mixed (confined and unconfined multiple aquifers)", "definition": "Mixed (confined and unconfined multiple aquifers)"}, + {"categories": ["geographic_scale"], "term": "Major", "definition": "Major aquifers of national significance"}, + {"categories": ["geographic_scale"], "term": "Regional", "definition": "Important aquifers serving regions"}, + {"categories": ["geographic_scale"], "term": "Local", "definition": "Smaller, locally important aquifers"}, + {"categories": ["geographic_scale"], "term": "Minor", "definition": "Limited extent or yield"}, + {"categories": ["formation_code"],"term": "000EXRV","definition": "Extrusive Rocks"}, + {"categories": ["formation_code"],"term": "000IRSV","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "050QUAL","definition": "Quaternary Alluvium in Valleys"}, + {"categories": ["formation_code"],"term": "100QBAS","definition": "Quaternary basalt"}, + {"categories": ["formation_code"],"term": "110ALVM","definition": "Quaternary Alluvium"}, + {"categories": ["formation_code"],"term": "110AVMB","definition": "Alluvium, Bolson Deposits and Other Surface Deposits"}, + {"categories": ["formation_code"],"term": "110BLSN","definition": "Bolson Fill"}, + {"categories": ["formation_code"],"term": "110NTGU","definition": "Naha and Tsegi Alluvium Deposits, undifferentiated"}, + {"categories": ["formation_code"],"term": "110PTODC","definition": "Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche"}, + {"categories": ["formation_code"],"term": "111MCCR","definition": "McCathys Basalt Flow"}, + {"categories": ["formation_code"],"term": "112ANCH","definition": "Upper Santa Fe Group, Ancha Formation (QTa)"}, + {"categories": ["formation_code"],"term": "112CURB","definition": "Cuerbio Basalt"}, + {"categories": ["formation_code"],"term": "112LAMA","definition": "Lama Formation (QTl, QTbh) and other mountain front alluvial fans"}, + {"categories": ["formation_code"],"term": "112LAMAb","definition": "Lama Fm (QTl, QTbh) between Servilleta Basalts"}, + {"categories": ["formation_code"],"term": "112LGUN","definition": "Laguna Basalt Flow"}, + {"categories": ["formation_code"],"term": "112QTBF","definition": "Quaternary-Tertiary basin fill (not in valleys)"}, + {"categories": ["formation_code"],"term": "112QTBFlac","definition": "Quaternary-Tertiary basin fill, lacustrian-playa lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFpd","definition": "Quaternary-Tertiary basin fill, distal piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112QTBFppm","definition": "Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies"}, + {"categories": ["formation_code"],"term": "112SNTF","definition": "Santa Fe Group, undivided"}, + {"categories": ["formation_code"],"term": "112SNTFA","definition": "Upper Santa Fe Group, axial facies"}, + {"categories": ["formation_code"],"term": "112SNTFOB","definition": "Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin"}, + {"categories": ["formation_code"],"term": "112SNTFP","definition": "Upper Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "112TRTO","definition": "Tuerto Gravels (QTt)"}, + {"categories": ["formation_code"],"term": "120DTIL","definition": "Datil Formation"}, + {"categories": ["formation_code"],"term": "120ELRT","definition": "El Rito Formation"}, + {"categories": ["formation_code"],"term": "120IRSV","definition": "Tertiary Intrusives"}, + {"categories": ["formation_code"],"term": "120SBLC","definition": "Sierra Blanca Volcanics, undivided"}, + {"categories": ["formation_code"],"term": "120SRVB","definition": "Tertiary Servilletta Basalts (Tsb)"}, + {"categories": ["formation_code"],"term": "120SRVBf","definition": "Tertiary Servilletta Basalts, fractured (Tsbf)"}, + {"categories": ["formation_code"],"term": "120TSBV_Lower","definition": "Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "120TSBV_Upper","definition": "Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm)"}, + {"categories": ["formation_code"],"term": "121CHMT","definition": "Chamita Formation (Tc)"}, + {"categories": ["formation_code"],"term": "121CHMTv","definition": "Chamita Fm, Vallito member (Tcv)"}, + {"categories": ["formation_code"],"term": "121CHMTvs","definition": "Chamita Fm, sandy Vallito member (Tcvs)"}, + {"categories": ["formation_code"],"term": "121OGLL","definition": "Ogallala Formation"}, + {"categories": ["formation_code"],"term": "121PUYEF","definition": "Puye Conglomerate, Fanglomerate Member"}, + {"categories": ["formation_code"],"term": "121TSUQ","definition": "Tesuque Formation, undifferentiated unit"}, + {"categories": ["formation_code"],"term": "121TSUQa","definition": "Tesuque Fm lithosome A (Tta)"}, + {"categories": ["formation_code"],"term": "121TSUQacu","definition": "Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)"}, + {"categories": ["formation_code"],"term": "121TSUQacuf","definition": "Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)"}, + {"categories": ["formation_code"],"term": "121TSUQaml","definition": "Tesuque Fm lower-middle lithosome A (Ttaml)"}, + {"categories": ["formation_code"],"term": "121TSUQb","definition": "Tesuque Fm lithosome B (Ttb)"}, + {"categories": ["formation_code"],"term": "121TSUQbfl","definition": "Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)"}, + {"categories": ["formation_code"],"term": "121TSUQbfm","definition": "Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)"}, + {"categories": ["formation_code"],"term": "121TSUQbp","definition": "Tesuque Fm lithosome B, Pojoaque member (Ttbp)"}, + {"categories": ["formation_code"],"term": "121TSUQce","definition": "Tesuque Fm, Cejita member (Ttce)"}, + {"categories": ["formation_code"],"term": "121TSUQe","definition": "Tesuque Fm lithosome E (Tte)"}, + {"categories": ["formation_code"],"term": "121TSUQs","definition": "Tesuque Fm lithosome S (Tts)"}, + {"categories": ["formation_code"],"term": "121TSUQsa","definition": "Tesuque Fm lateral gradation lithosomes S and A (Ttsag)"}, + {"categories": ["formation_code"],"term": "121TSUQsc","definition": "Tesuque Fm coarse-grained lithosome S (Ttsc)"}, + {"categories": ["formation_code"],"term": "121TSUQsf","definition": "Tesuque Fm, fine-grained lithosome S (Ttsf)"}, + {"categories": ["formation_code"],"term": "122CHOC","definition": "Chamita and Ojo Caliente interlayered (Ttoc)"}, + {"categories": ["formation_code"],"term": "122CRTO","definition": "Chama El Rito Formation (Tesuque member, Ttc)"}, + {"categories": ["formation_code"],"term": "122OJOC","definition": "Ojo Caliente Formation (Tesuque member, Tto)"}, + {"categories": ["formation_code"],"term": "122PICR","definition": "Picuris Tuff"}, + {"categories": ["formation_code"],"term": "122PPTS","definition": "Popotosa Formation"}, + {"categories": ["formation_code"],"term": "122SNTFP","definition": "Lower Santa Fe Group, piedmont facies"}, + {"categories": ["formation_code"],"term": "123DTILSPRS","definition": "Datil Group ignimbrites and lavas and Spears Group, interbedded"}, + {"categories": ["formation_code"],"term": "123DTMGandbas","definition": "Datil and Mogollon Group andesite, basaltic andesite, and basalt flows"}, + {"categories": ["formation_code"],"term": "123DTMGign","definition": "Datil and Mogollon Group ignimbrites"}, + {"categories": ["formation_code"],"term": "123DTMGrhydac","definition": "Datil and Mogollon Group rhyolite and dacite flows"}, + {"categories": ["formation_code"],"term": "123ESPN","definition": "T Espinaso Formation (Te)"}, + {"categories": ["formation_code"],"term": "123GLST","definition": "T Galisteo Formation"}, + {"categories": ["formation_code"],"term": "123PICS","definition": "T Picuris Formation (Tp)"}, + {"categories": ["formation_code"],"term": "123PICSc","definition": "T Picuris Formation, basal conglomerate (Tpc)"}, + {"categories": ["formation_code"],"term": "123PICSl","definition": "T lower Picuris Formation (Tpl)"}, + {"categories": ["formation_code"],"term": "123SPRSDTMGlava","definition": "Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded"}, + {"categories": ["formation_code"],"term": "123SPRSlower","definition": "Spears Group, lower part; tuffaceous, gravelly debris and mud flows"}, + {"categories": ["formation_code"],"term": "123SPRSmid_uppe","definition": "Spears Group, middle-upper part; excludes Dog Spring Formation"}, + {"categories": ["formation_code"],"term": "124BACA","definition": "Baca Formation"}, + {"categories": ["formation_code"],"term": "124CBMN","definition": "Cub Mountain Formation"}, + {"categories": ["formation_code"],"term": "124LLVS","definition": "Llaves Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124PSCN","definition": "Poison Canyon Formation"}, + {"categories": ["formation_code"],"term": "124RGIN","definition": "Regina Member of San Jose Formation"}, + {"categories": ["formation_code"],"term": "124SNJS","definition": "San Jose Formation"}, + {"categories": ["formation_code"],"term": "124TPCS","definition": "TapicitosMember of San Jose Formation"}, + {"categories": ["formation_code"],"term": "125NCMN","definition": "Nacimiento Formation"}, + {"categories": ["formation_code"],"term": "125NCMNS","definition": "Nacimiento Formation, Sandy Shale Facies"}, + {"categories": ["formation_code"],"term": "125RTON","definition": "Raton Formation"}, + {"categories": ["formation_code"],"term": "130CALDFLOOR","definition": "Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo."}, + {"categories": ["formation_code"],"term": "180TKSCC_Upper","definition": "Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "180TKTR","definition": "Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T"}, + {"categories": ["formation_code"],"term": "210CRCS","definition": "Cretaceous System, undivided"}, + {"categories": ["formation_code"],"term": "210GLUPC_Lower","definition": "K Gallup Sandstone and lower Crevasse Canyon Fm"}, + {"categories": ["formation_code"],"term": "210HOSTD","definition": "K Hosta Dalton"}, + {"categories": ["formation_code"],"term": "210MCDK","definition": "K Mancos/Dakota undivided"}, + {"categories": ["formation_code"],"term": "210MNCS","definition": "Mancos Shale, undivided"}, + {"categories": ["formation_code"],"term": "210MNCSL","definition": "K Lower Mancos"}, + {"categories": ["formation_code"],"term": "210MNCSU","definition": "K Upper Mancos"}, + {"categories": ["formation_code"],"term": "211CLFHV","definition": "Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co."}, + {"categories": ["formation_code"],"term": "211CRLL","definition": "Carlile Shale"}, + {"categories": ["formation_code"],"term": "211CRVC","definition": "Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DKOT","definition": "Dakota Sandstone or Formation"}, + {"categories": ["formation_code"],"term": "211DLCO","definition": "Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211DLTN","definition": "Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211FRHS","definition": "Fort Hays Limestone Member of Niobrara Formation"}, + {"categories": ["formation_code"],"term": "211FRLD","definition": "Fruitland Formation"}, + {"categories": ["formation_code"],"term": "211FRMG","definition": "Farmington Sandstone Member of Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211GBSNC","definition": "Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211GLLG","definition": "Gallego Sandstone Member of Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GLLP","definition": "Gallup Sandstone"}, + {"categories": ["formation_code"],"term": "211GRRG","definition": "Greenhorn and Graneros Formations"}, + {"categories": ["formation_code"],"term": "211GRRS","definition": "Graneros Shale"}, + {"categories": ["formation_code"],"term": "211HOST","definition": "Hosta Tongue of Point Lookout Sandstone of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211KRLD","definition": "Kirtland Shale"}, + {"categories": ["formation_code"],"term": "211LWIS","definition": "Lewis Shale"}, + {"categories": ["formation_code"],"term": "211MENF","definition": "Menefee Formation"}, + {"categories": ["formation_code"],"term": "211MENFU","definition": "K Upper Menefee (above Harmon Sandstone)"}, + {"categories": ["formation_code"],"term": "211MVRD","definition": "Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211OJAM","definition": "Ojo Alamo Sandstone"}, + {"categories": ["formation_code"],"term": "211PCCF","definition": "Pictured Cliffs Sandstone"}, + {"categories": ["formation_code"],"term": "211PIRR","definition": "Pierre Shale"}, + {"categories": ["formation_code"],"term": "211PNLK","definition": "Point Lookout Sandstone"}, + {"categories": ["formation_code"],"term": "211SMKH","definition": "Smoky Hill Marl Member"}, + {"categories": ["formation_code"],"term": "211TLLS","definition": "Twowells Sandstone Lentil of Pike of Dakota Sandstone"}, + {"categories": ["formation_code"],"term": "212KTRP","definition": "K Dakota Sandstone, Moenkopi Fm, Artesia Group"}, + {"categories": ["formation_code"],"term": "217PRGR","definition": "Purgatoire Formation"}, + {"categories": ["formation_code"],"term": "220ENRD","definition": "Entrada Sandstone"}, + {"categories": ["formation_code"],"term": "220JURC","definition": "Jurassic undivided"}, + {"categories": ["formation_code"],"term": "220NAVJ","definition": "Navajo Sandstone"}, + {"categories": ["formation_code"],"term": "221BLFF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221CSPG","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ERADU","definition": "Entrada Sandstone of San Rafael Group, Upper"}, + {"categories": ["formation_code"],"term": "221MRSN","definition": "Morrison Formation"}, + {"categories": ["formation_code"],"term": "221MRSN/BBSN","definition": "Brushy Basin Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/JCKP","definition": "Jackpile Sandstone Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/RCAP","definition": "Recapture Shale Member of Morrison"}, + {"categories": ["formation_code"],"term": "221MRSN/WWCN","definition": "Westwater Canyon Member of Morrison"}, + {"categories": ["formation_code"],"term": "221SLWS","definition": "Salt Wash Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221SMVL","definition": "Summerville Formation of San Rafael Group"}, + {"categories": ["formation_code"],"term": "221TDLT","definition": "J Todilto"}, + {"categories": ["formation_code"],"term": "221WSRC","definition": "Westwater Canyon Sandstone Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221ZUNIS","definition": "Zuni Sandstone"}, + {"categories": ["formation_code"],"term": "231AGZC","definition": "Tr Agua Zarca"}, + {"categories": ["formation_code"],"term": "231AGZCU","definition": "Tr Upper Agua Zarca"}, + {"categories": ["formation_code"],"term": "231CHNL","definition": "Chinle Formation"}, + {"categories": ["formation_code"],"term": "231CORR","definition": "Correo Sandstone Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231DCKM","definition": "Dockum Group"}, + {"categories": ["formation_code"],"term": "231PFDF","definition": "Tr Petrified Forest"}, + {"categories": ["formation_code"],"term": "231PFDFL","definition": "Tr Lower Petrified Forest (below middle sandstone)"}, + {"categories": ["formation_code"],"term": "231PFDFM","definition": "Tr Middle Petrified Forest sandstone"}, + {"categories": ["formation_code"],"term": "231PFDFU","definition": "Tr Upper Petrified Forest (above middle sandstone)"}, + {"categories": ["formation_code"],"term": "231RCKP","definition": "Rock Point Member of Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "231SNRS","definition": "Santa Rosa Sandstone"}, + {"categories": ["formation_code"],"term": "231SNSL","definition": "Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231SRMP","definition": "Shinarump Member of Chinle Formation"}, + {"categories": ["formation_code"],"term": "231WNGT","definition": "Wingate Sandstone"}, + {"categories": ["formation_code"],"term": "260SNAN","definition": "P San Andres"}, + {"categories": ["formation_code"],"term": "260SNAN_lower","definition": "Lower San Andres Formation"}, + {"categories": ["formation_code"],"term": "261SNGL","definition": "P San Andres - Glorieta Sandstone in Rio Bonito member"}, + {"categories": ["formation_code"],"term": "300YESO","definition": "P Yeso"}, + {"categories": ["formation_code"],"term": "300YESO_lower","definition": "Lower Yeso Formation"}, + {"categories": ["formation_code"],"term": "300YESO_upper","definition": "Upper Yeso Formation"}, + {"categories": ["formation_code"],"term": "310ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "310DCLL","definition": "De Chelly Sandstone Member of Cutler Formation"}, + {"categories": ["formation_code"],"term": "310GLOR","definition": "Glorieta Sandstone Member of San Andres Formation (of Manzano Group)"}, + {"categories": ["formation_code"],"term": "310MBLC","definition": "Meseta Blanca Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310TRRS","definition": "Torres Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "310YESOG","definition": "Yeso Formation, Manzono Group"}, + {"categories": ["formation_code"],"term": "312CSTL","definition": "Castile Formation"}, + {"categories": ["formation_code"],"term": "312RSLR","definition": "Rustler Formation"}, + {"categories": ["formation_code"],"term": "313ARTS","definition": "Artesia Group"}, + {"categories": ["formation_code"],"term": "313BLCN","definition": "Bell Canyon Formation"}, + {"categories": ["formation_code"],"term": "313BRUC","definition": "Brushy Canyon Formation of Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "313CKBF","definition": "Chalk Bluff Formation"}, + {"categories": ["formation_code"],"term": "313CLBD","definition": "Carlsbad Limestone"}, + {"categories": ["formation_code"],"term": "313CPTN","definition": "Capitan Limestone"}, + {"categories": ["formation_code"],"term": "313GDLP","definition": "Guadalupian Series"}, + {"categories": ["formation_code"],"term": "313GOSP","definition": "Goat Seep Dolomite"}, + {"categories": ["formation_code"],"term": "313SADG","definition": "San Andres Limestone and Glorieta Sandstone"}, + {"categories": ["formation_code"],"term": "313SADR","definition": "San Andres Limestone, undivided"}, + {"categories": ["formation_code"],"term": "313TNSL","definition": "Tansill Formation"}, + {"categories": ["formation_code"],"term": "313YATS","definition": "Yates Formation, Guadalupe Group"}, + {"categories": ["formation_code"],"term": "315LABR","definition": "P Laborcita (Bursum)"}, + {"categories": ["formation_code"],"term": "315YESOABO","definition": "Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations"}, + {"categories": ["formation_code"],"term": "318ABO","definition": "P Abo"}, + {"categories": ["formation_code"],"term": "318BSPG","definition": "Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "318JOYT","definition": "Joyita Sandstone Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "318YESO","definition": "Yeso Formation"}, + {"categories": ["formation_code"],"term": "319BRSM","definition": "Bursum Formation and Equivalent Rocks"}, + {"categories": ["formation_code"],"term": "320HLDR","definition": "Penn Holder"}, + {"categories": ["formation_code"],"term": "320PENN","definition": "Pennsylvanian undivided"}, + {"categories": ["formation_code"],"term": "320SNDI","definition": "Sandia Formation"}, + {"categories": ["formation_code"],"term": "321SGDC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "322BEMN","definition": "Penn Beeman"}, + {"categories": ["formation_code"],"term": "325GBLR","definition": "Penn Gobbler"}, + {"categories": ["formation_code"],"term": "325MDER","definition": "Madera Limestone, undivided"}, + {"categories": ["formation_code"],"term": "325MDERL","definition": "Penn Lower Madera"}, + {"categories": ["formation_code"],"term": "325MDERU","definition": "Penn Upper Madera"}, + {"categories": ["formation_code"],"term": "325SAND","definition": "Penn Sandia"}, + {"categories": ["formation_code"],"term": "326MGDL","definition": "Magdalena Group"}, + {"categories": ["formation_code"],"term": "340EPRS","definition": "Espiritu Santo Formation"}, + {"categories": ["formation_code"],"term": "350PZBA","definition": "Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm"}, + {"categories": ["formation_code"],"term": "350PZBB","definition": "Tul Basin area - Paleozoic strata below Bursum Fm"}, + {"categories": ["formation_code"],"term": "400EMBD","definition": "Embudo Granite (undifferentiated PreCambrian near Santa Fe)"}, + {"categories": ["formation_code"],"term": "400PCMB","definition": "Precambrian Erathem"}, + {"categories": ["formation_code"],"term": "400PREC","definition": "undifferentiated PreCambrian crystalline rocks (X)"}, + {"categories": ["formation_code"],"term": "400PRECintr","definition": "PreCambrian crystalline rocks and local Tertiary intrusives"}, + {"categories": ["formation_code"],"term": "400PRST","definition": "Priest Granite"}, + {"categories": ["formation_code"],"term": "400TUSS","definition": "Tusas Granite"}, + {"categories": ["formation_code"],"term": "410PRCG","definition": "PreCambrian granite (Xg)"}, + {"categories": ["formation_code"],"term": "410PRCGf","definition": "PreCambrian granite, fractured (Xgf)"}, + {"categories": ["formation_code"],"term": "410PRCQ","definition": "PreCambrian quartzite (Xq)"}, + {"categories": ["formation_code"],"term": "410PRCQf","definition": "PreCambrian quartzite, fractured (Xqf)"}, + {"categories": ["formation_code"],"term": "121GILA","definition": "Gila Conglomerate (group)"}, + {"categories": ["formation_code"],"term": "312DYLK","definition": "Dewey Lake Redbeds"}, + {"categories": ["formation_code"],"term": "120WMVL","definition": "Wimsattville Formation"}, + {"categories": ["formation_code"],"term": "313GRBG","definition": "Grayburg Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318ABOL","definition": "Abo Sandstone (Lower Tongue)"}, + {"categories": ["formation_code"],"term": "318ABOU","definition": "Abo Sandstone (Upper Tongue)"}, + {"categories": ["formation_code"],"term": "112SNTFU","definition": "Santa Fe Group, Upper Part"}, + {"categories": ["formation_code"],"term": "310FRNR","definition": "Forty-Niner Member of Rustler Formation"}, + {"categories": ["formation_code"],"term": "312OCHO","definition": "Ochoan Series"}, + {"categories": ["formation_code"],"term": "313AZOT","definition": "Azotea Tongue of Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313QUEN","definition": "Queen Formation"}, + {"categories": ["formation_code"],"term": "319HUCO","definition": "Hueco Limestone"}, + {"categories": ["formation_code"],"term": "313SVRV","definition": "Seven Rivers Formation"}, + {"categories": ["formation_code"],"term": "313CABD","definition": "Carlsbad Group"}, + {"categories": ["formation_code"],"term": "320GRMS","definition": "Gray Mesa Member of Madera Formation"}, + {"categories": ["formation_code"],"term": "211CLRDH","definition": "Colorado Shale"}, + {"categories": ["formation_code"],"term": "120BRLM","definition": "Bearwallow Mountain Andesite"}, + {"categories": ["formation_code"],"term": "122RUBO","definition": "Rubio Peak Formation"}, + {"categories": ["formation_code"],"term": "313SADRL","definition": "San Andres Limestone, Lower Cherty Member"}, + {"categories": ["formation_code"],"term": "313SADRU","definition": "San Andres Limestone, Upper Clastic Member"}, + {"categories": ["formation_code"],"term": "313BRNL","definition": "Bernal Formation of Artesia Group"}, + {"categories": ["formation_code"],"term": "318CPDR","definition": "Chupadera Formation"}, + {"categories": ["formation_code"],"term": "121BDHC","definition": "Bidahochi Formation"}, + {"categories": ["formation_code"],"term": "313SADY","definition": "San Andres Limestone and Yeso Formation, undivided"}, + {"categories": ["formation_code"],"term": "221SRFLL","definition": "San Rafael Group, Lower Part"}, + {"categories": ["formation_code"],"term": "221BLUF","definition": "Bluff Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "221COSP","definition": "Cow Springs Sandstone of Morrison Formation"}, + {"categories": ["formation_code"],"term": "317ABYS","definition": "Abo and Yeso, undifferentiated"}, + {"categories": ["formation_code"],"term": "221BRSB","definition": "Brushy Basin Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "310SYDR","definition": "San Ysidro Member of Yeso Formation"}, + {"categories": ["formation_code"],"term": "400SDVL","definition": "Sandoval Granite"}, + {"categories": ["formation_code"],"term": "221SRFL","definition": "San Rafael Group"}, + {"categories": ["formation_code"],"term": "310SGRC","definition": "Sangre de Cristo Formation"}, + {"categories": ["formation_code"],"term": "231TCVS","definition": "Tecovas Formation of Dockum Group"}, + {"categories": ["formation_code"],"term": "211DCRS","definition": "D-Cross Tongue of Mancos Shale of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211ALSN","definition": "Allison Member of Menefee Formation of Mesaverde Group"}, + {"categories": ["formation_code"],"term": "211LVNN","definition": "La Ventana Tongue of Cliff House Sandstone"}, + {"categories": ["formation_code"],"term": "211MORD","definition": "Madrid Formation"}, + {"categories": ["formation_code"],"term": "210PRMD","definition": "Pyramid Shale"}, + {"categories": ["formation_code"],"term": "124ANMS","definition": "Animas Formation"}, + {"categories": ["formation_code"],"term": "211NBRR","definition": "Niobrara Formation"}, + {"categories": ["formation_code"],"term": "111ALVM","definition": "Holocene Alluvium"}, + {"categories": ["formation_code"],"term": "122SNTFL","definition": "Santa Fe Group, Lower Part"}, + {"categories": ["formation_code"],"term": "111CPLN","definition": "Capulin Basalts"}, + {"categories": ["formation_code"],"term": "120CRSN","definition": "Carson Conflomerate"}, + {"categories": ["formation_code"],"term": "111CRMS","definition": "Covered/Reclaimed Mine Spoil"}, + {"categories": ["formation_code"],"term": "111CRMSA","definition": "Covered/Reclaimed Mine Spoil and Ash"}, + {"categories": ["formation_code"],"term": "111SPOL","definition": "Spoil"}, + {"categories": ["formation_code"],"term": "110TURT","definition": "Tuerto Gravel of Santa Fe Group"}, + {"categories": ["formation_code"],"term": "221RCPR","definition": "Recapture Shale Member of Morrison Formation"}, + {"categories": ["formation_code"],"term": "320BLNG","definition": "Bullington Member of Magdalena Formation"}, + {"categories": ["formation_code"],"term": "112ANCHsr","definition": "Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits"}, + {"categories": ["formation_code"],"term": "121TSUQae","definition": "Tesuque Fm Lithosomes A and E"}, + {"categories": ["formation_code"],"term": "230TRSC","definition": "Triassic undifferentiated"}, + {"categories": ["formation_code"],"term": "122TSUQdx","definition": "Tesuque Fm, Dixon member (Ttd)"}, + {"categories": ["formation_code"],"term": "123PICSu","definition": "T upper Picuris Formation (Tpu)"}, + {"categories": ["formation_code"],"term": "123PICSm","definition": "T middle Picuris Formation (Tpm)"}, + {"categories": ["formation_code"],"term": "123PICSmc","definition": "T middle conglomerate Picuris Formation (Tpmc)"}, + {"categories": ["formation_code"],"term": "120VBVC","definition": "Tertiary volcanic breccia/volcaniclastic conglomerate"}, + {"categories": ["formation_code"],"term": "120VCSS","definition": "Tertiary volcaniclastic sandstone"}, + {"categories": ["formation_code"],"term": "124DMDT","definition": "Diamond Tail Formation"}, + {"categories": ["formation_code"],"term": "325ALMT","definition": "Penn Alamitos Formation"}, + {"categories": ["formation_code"],"term": "400SAND","definition": "Sandia Granite"}, + {"categories": ["formation_code"],"term": "318VCPK","definition": "Victorio Peak Limestone"}, + {"categories": ["formation_code"],"term": "318BSVP","definition": "Bone Spring and Victorio Peak Limestones"}, + {"categories": ["formation_code"],"term": "100ALVM","definition": "Alluvium"}, + {"categories": ["formation_code"],"term": "310PRMN","definition": "Permian System"}, + {"categories": ["formation_code"],"term": "110AVPS","definition": "Alluvium and Permian System"}, + {"categories": ["formation_code"],"term": "313CRCX","definition": "Capitan Reef Complex and Associated Limestones"}, + {"categories": ["formation_code"],"term": "112SLBL","definition": "Salt Bolson"}, + {"categories": ["formation_code"],"term": "112SBCRC","definition": "Salt Bolson and Capitan Reef Complex"}, + {"categories": ["formation_code"],"term": "313CRDM","definition": "Capitan Reef Complex - Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "112SBDM","definition": "Salt Bolson and Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "120BLSN","definition": "Bolson Deposits"}, + {"categories": ["formation_code"],"term": "112SBCR","definition": "Salt Bolson and Cretaceous Rocks"}, + {"categories": ["formation_code"],"term": "112HCBL","definition": "Hueco Bolson"}, + {"categories": ["formation_code"],"term": "120IVIG","definition": "Intrusive Rocks"}, + {"categories": ["formation_code"],"term": "112RLBL","definition": "Red Light Draw Bolson"}, + {"categories": ["formation_code"],"term": "112EFBL","definition": "Eagle Flat Bolson"}, + {"categories": ["formation_code"],"term": "112GRBL","definition": "Green River Bolson"}, + {"categories": ["formation_code"],"term": "123SAND","definition": "Sanders Canyon Formation"}, + {"categories": ["formation_code"],"term": "210MRNH","definition": "Moreno Hill Formation"}, + {"categories": ["formation_code"],"term": "320ALMT","definition": "Alamito Shale"}, + {"categories": ["formation_code"],"term": "313DLRM","definition": "Delaware Mountain Group"}, + {"categories": ["formation_code"],"term": "300PLZC","definition": "Paleozoic Erathem"}, + {"categories": ["formation_code"],"term": "122SPRS","definition": "Spears Member of Datil Formation"}, + {"categories": ["formation_code"],"term": "110AVTV","definition": "Alluvium and Tertiary Volcanics"}, + {"categories": ["formation_code"],"term": "313DMBS","definition": "Delaware Mountain Group - Bone Spring Limestone"}, + {"categories": ["formation_code"],"term": "120ERSV","definition": "Tertiary extrusives"}, + {"categories": ["lithology"],"term": "Alluvium","definition": "Alluvium"}, + {"categories": ["lithology"],"term": "Anhydrite","definition": "Anhydrite"}, + {"categories": ["lithology"],"term": "Arkose","definition": "Arkose"}, + {"categories": ["lithology"],"term": "Boulders","definition": "Boulders"}, + {"categories": ["lithology"],"term": "Boulders, silt and clay","definition": "Boulders, silt and clay"}, + {"categories": ["lithology"],"term": "Boulders and sand","definition": "Boulders and sand"}, + {"categories": ["lithology"],"term": "Bentonite","definition": "Bentonite"}, + {"categories": ["lithology"],"term": "Breccia","definition": "Breccia"}, + {"categories": ["lithology"],"term": "Basalt","definition": "Basalt"}, + {"categories": ["lithology"],"term": "Conglomerate","definition": "Conglomerate"}, + {"categories": ["lithology"],"term": "Chalk","definition": "Chalk"}, + {"categories": ["lithology"],"term": "Chert","definition": "Chert"}, + {"categories": ["lithology"],"term": "Clay","definition": "Clay"}, + {"categories": ["lithology"],"term": "Caliche","definition": "Caliche"}, + {"categories": ["lithology"],"term": "Calcite","definition": "Calcite"}, + {"categories": ["lithology"],"term": "Clay, some sand","definition": "Clay, some sand"}, + {"categories": ["lithology"],"term": "Claystone","definition": "Claystone"}, + {"categories": ["lithology"],"term": "Coal","definition": "Coal"}, + {"categories": ["lithology"],"term": "Cobbles","definition": "Cobbles"}, + {"categories": ["lithology"],"term": "Cobbles, silt and clay","definition": "Cobbles, silt and clay"}, + {"categories": ["lithology"],"term": "Cobbles and sand","definition": "Cobbles and sand"}, + {"categories": ["lithology"],"term": "Dolomite","definition": "Dolomite"}, + {"categories": ["lithology"],"term": "Dolomite and shale","definition": "Dolomite and shale"}, + {"categories": ["lithology"],"term": "Evaporite","definition": "Evaporite"}, + {"categories": ["lithology"],"term": "Gneiss","definition": "Gneiss"}, + {"categories": ["lithology"],"term": "Gypsum","definition": "Gypsum"}, + {"categories": ["lithology"],"term": "Graywacke","definition": "Graywacke"}, + {"categories": ["lithology"],"term": "Gravel and clay","definition": "Gravel and clay"}, + {"categories": ["lithology"],"term": "Gravel, cemented","definition": "Gravel, cemented"}, + {"categories": ["lithology"],"term": "Gravel, sand and silt","definition": "Gravel, sand and silt"}, + {"categories": ["lithology"],"term": "Granite, gneiss","definition": "Granite, gneiss"}, + {"categories": ["lithology"],"term": "Granite","definition": "Granite"}, + {"categories": ["lithology"],"term": "Gravel, silt and clay","definition": "Gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Gravel","definition": "Gravel"}, + {"categories": ["lithology"],"term": "Igneous undifferentiated","definition": "Igneous undifferentiated"}, + {"categories": ["lithology"],"term": "Lignite","definition": "Lignite"}, + {"categories": ["lithology"],"term": "Limestone and dolomite","definition": "Limestone and dolomite"}, + {"categories": ["lithology"],"term": "Limestone and shale","definition": "Limestone and shale"}, + {"categories": ["lithology"],"term": "Limestone","definition": "Limestone"}, + {"categories": ["lithology"],"term": "Marl","definition": "Marl"}, + {"categories": ["lithology"],"term": "Mudstone","definition": "Mudstone"}, + {"categories": ["lithology"],"term": "Metamorphic undifferentiated","definition": "Metamorphic undifferentiated"}, + {"categories": ["lithology"],"term": "Marlstone","definition": "Marlstone"}, + {"categories": ["lithology"],"term": "No Recovery","definition": "No Recovery"}, + {"categories": ["lithology"],"term": "Peat","definition": "Peat"}, + {"categories": ["lithology"],"term": "Quartzite","definition": "Quartzite"}, + {"categories": ["lithology"],"term": "Rhyolite","definition": "Rhyolite"}, + {"categories": ["lithology"],"term": "Sand","definition": "Sand"}, + {"categories": ["lithology"],"term": "Schist","definition": "Schist"}, + {"categories": ["lithology"],"term": "Sand and clay","definition": "Sand and clay"}, + {"categories": ["lithology"],"term": "Sand and gravel","definition": "Sand and gravel"}, + {"categories": ["lithology"],"term": "Sandstone and shale","definition": "Sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand and silt","definition": "Sand and silt"}, + {"categories": ["lithology"],"term": "Sand, gravel and clay","definition": "Sand, gravel and clay"}, + {"categories": ["lithology"],"term": "Shale","definition": "Shale"}, + {"categories": ["lithology"],"term": "Silt","definition": "Silt"}, + {"categories": ["lithology"],"term": "Siltstone and shale","definition": "Siltstone and shale"}, + {"categories": ["lithology"],"term": "Siltstone","definition": "Siltstone"}, + {"categories": ["lithology"],"term": "Slate","definition": "Slate"}, + {"categories": ["lithology"],"term": "Sand, some clay","definition": "Sand, some clay"}, + {"categories": ["lithology"],"term": "Sandstone","definition": "Sandstone"}, + {"categories": ["lithology"],"term": "Silt and clay","definition": "Silt and clay"}, + {"categories": ["lithology"],"term": "Travertine","definition": "Travertine"}, + {"categories": ["lithology"],"term": "Tuff","definition": "Tuff"}, + {"categories": ["lithology"],"term": "Volcanic undifferentiated","definition": "Volcanic undifferentiated"}, + {"categories": ["lithology"],"term": "Clay, yellow","definition": "Clay, yellow"}, + {"categories": ["lithology"],"term": "Clay, red","definition": "Clay, red"}, + {"categories": ["lithology"],"term": "Surficial sediment","definition": "Surficial sediment"}, + {"categories": ["lithology"],"term": "Limestone and sandstone, interbedded","definition": "Limestone and sandstone, interbedded"}, + {"categories": ["lithology"],"term": "Gravel and boulders","definition": "Gravel and boulders"}, + {"categories": ["lithology"],"term": "Sand, silt and gravel","definition": "Sand, silt and gravel"}, + {"categories": ["lithology"],"term": "Sand, gravel, silt and clay","definition": "Sand, gravel, silt and clay"}, + {"categories": ["lithology"],"term": "Andesite","definition": "Andesite"}, + {"categories": ["lithology"],"term": "Ignesous, intrusive, undifferentiated","definition": "Ignesous, intrusive, undifferentiated"}, + {"categories": ["lithology"],"term": "Limestone, sandstone and shale","definition": "Limestone, sandstone and shale"}, + {"categories": ["lithology"],"term": "Sand, silt and clay","definition": "Sand, silt and clay"}, {"categories": ["origin_source"], "term": "Reported by another agency", "definition": "Reported by another agency"}, {"categories": ["origin_source"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, {"categories": ["origin_source"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, @@ -709,6 +1102,13 @@ {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, - {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"} + {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"}, + {"categories": ["well_pump_type"], "term": "Submersible", "definition": "Submersible"}, + {"categories": ["well_pump_type"], "term": "Jet", "definition": "Jet Pump"}, + {"categories": ["well_pump_type"], "term": "Line Shaft", "definition": "Line Shaft"}, + {"categories": ["well_pump_type"], "term": "Hand", "definition": "Hand Pump"}, + {"categories": ["permission_type"], "term": "Water Level Sample", "definition": "Permissions for taking water level samples"}, + {"categories": ["permission_type"], "term": "Water Chemistry Sample", "definition": "Permissions for water taking chemistry samples"}, + {"categories": ["permission_type"], "term": "Datalogger Installation", "definition": "Permissions for installing dataloggers"} ] } \ No newline at end of file diff --git a/db/__init__.py b/db/__init__.py index 5a58441f8..4a0fc8e70 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -33,7 +33,7 @@ from db.notes import * from db.observation import * from db.parameter import * -from db.permission import * +from db.permission_history import * from db.publication import * from db.regulatory_limit import * from db.sample import * @@ -43,6 +43,11 @@ from db.transducer import * from db.measuring_point_history import * from db.data_provenance import * +from db.aquifer_system import * +from db.geologic_formation import * +from db.thing_aquifer_association import * +from db.thing_geologic_formation_association import * +from db.aquifer_type import * from sqlalchemy import ( func, diff --git a/db/aquifer_system.py b/db/aquifer_system.py new file mode 100644 index 000000000..c202d77c9 --- /dev/null +++ b/db/aquifer_system.py @@ -0,0 +1,84 @@ +""" +SQLAlchemy model for the AquiferSystem table. + +This is a master reference table for aquifer systems and hydrogeologic units. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +from constants import SRID_WGS84 + +if TYPE_CHECKING: + from db.thing import WellScreen, ThingAquiferAssociation, Thing + from db.aquifer_type import AquiferType + + +class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + name: Mapped[str] = mapped_column( + nullable=False, + unique=True, + comment="The full, human-readable name of the aquifer system (e.g., 'Ogallala Aquifer').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the aquifer system, its characteristics, and its significance.", + ) + # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table. + primary_aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to classify the aquifer system as a whole (e.g., 'Unconfined', 'Confined', 'Perched').", + ) + geographic_scale: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), + nullable=True, + comment="A spatial representation of the aquifer system's boundary.", + ) + # Hierarchical relationship fields (may be implemented in future iterations) + # Example: High Plains Aquifer (parent) contains Ogallala Aquifer (child) + # parent_id = Column(Integer, ForeignKey('aquifer_system.id')) + # parent = relationship('AquiferSystem', remote_side=[id], backref='subsystems') + + # --- Relationships --- + # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table. + thing_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: An AquiferSystem can be the target for many individual WellScreens. + well_screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", + back_populates="aquifer_system", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # --- Association Proxies --- + # Proxy to directly access Things (wells) associated with this AquiferSystem. + things: AssociationProxy[List["Thing"]] = association_proxy( + "thing_associations", "thing" + ) + # Proxy to directly access all AquiferTypes associated with this AquiferSystem. + aquifer_types: AssociationProxy[List["AquiferType"]] = association_proxy( + "thing_associations", "aquifer_types" + ) + + # --- Table Arguments --- + __table_args__ = (Index("ix_aquifersystem_name", "name"),) diff --git a/db/aquifer_type.py b/db/aquifer_type.py new file mode 100644 index 000000000..32900d801 --- /dev/null +++ b/db/aquifer_type.py @@ -0,0 +1,58 @@ +""" +SQLAlchemy model for the AquiferType table. + +This table stores the specific aquifer characteristics/types associated with +a Thing-AquiferSystem relationship. It allows capturing that a single aquifer +can have multiple characteristics simultaneously. + +Example: + A well in the "Ogallala" aquifer might tap portions that are both + "Fractured" AND "Confined". This would create: + - One AquiferSystem: "Ogallala" + - One ThingAquiferAssociation: linking well to Ogallala + - Two AquiferType records: "Fractured" and "Confined" +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.thing_aquifer_association import ThingAquiferAssociation + + +class AquiferType(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the specific aquifer types/characteristics for a + Thing-AquiferSystem association. + + This allows modeling the fact that: + - A single aquifer can have multiple characteristics + - Different wells may tap different characteristics of the same aquifer + - Characteristics are attributes of the relationship, not the aquifer itself + + Fields from WellData CSV: + - AquiferType: May contain multiple codes (e.g., "FC" = Fractured + Confined) + - Each code becomes a separate AquiferType record + """ + + # --- Columns --- + thing_aquifer_association_id: Mapped[int] = mapped_column( + ForeignKey("thing_aquifer_association.id", ondelete="CASCADE"), + nullable=False, + comment="Links to the Thing-Aquifer association this type describes.", + ) + aquifer_type: Mapped[str] = lexicon_term( + nullable=False, + comment="Controlled vocabulary for aquifer hydrologic properties. " + "Examples: 'Unconfined', 'Confined', 'Perched', 'Fractured', 'Unconsolidated'.", + ) + + # --- Relationships --- + # Many-to-One: Multiple aquifer types can belong to one association + thing_aquifer_association: Mapped["ThingAquiferAssociation"] = relationship( + "ThingAquiferAssociation", back_populates="aquifer_types" + ) diff --git a/db/base.py b/db/base.py index e9b0d7f2b..765a341bc 100644 --- a/db/base.py +++ b/db/base.py @@ -53,7 +53,6 @@ declared_attr, Mapped, mapped_column, - relationship, ) from sqlalchemy_continuum import make_versioned from sqlalchemy_searchable import make_searchable @@ -179,25 +178,6 @@ def properties(self): # ============= Polymorphic Helper Mixins ============================================= -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/contact.py b/db/contact.py index 7855814fb..558724df9 100644 --- a/db/contact.py +++ b/db/contact.py @@ -26,7 +26,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission import Permission + from db.permission_history import PermissionHistory class ThingContactAssociation(Base, AutoBaseMixin): @@ -74,8 +74,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["Permission"]] = relationship( - "Permission", back_populates="contact", cascade="all, delete, delete-orphan" + permissions: Mapped[List["PermissionHistory"]] = relationship( + "PermissionHistory", + back_populates="contact", + cascade="all, delete, delete-orphan", ) # One-To-Many: A Contact can be associated with many Authors (in Publications). author_associations: Mapped[List["AuthorContactAssociation"]] = relationship( diff --git a/db/data_provenance.py b/db/data_provenance.py index 06c468c8d..20505d94c 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -19,7 +19,7 @@ from sqlalchemy import Integer, Index, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin from db import lexicon_term @@ -53,9 +53,13 @@ class DataProvenance(AutoBaseMixin, ReleaseMixin, Base): ) # Values from the following NMAquifer tables are included as `origin_source` terms in the lexicon: # 'LU_DataSource', 'LU_Depth_CompletionSource'. - origin_source: Mapped[str] = lexicon_term( + origin_type: Mapped[str] = lexicon_term( nullable=True, - comment="Indicates the origin source of the data (e.g'Driller's Log', 'Well Report'.", + comment="Indicates the type of origin the data (e.g'Driller's Log', 'Well Report'.", + ) + origin_source: Mapped[str] = mapped_column( + nullable=True, + comment="The specific source of the data (e.g., 'J. Brown Thesis, \"I like APIs\", Pomona College, 1994').", ) # Values from the following NMAquifer tables are included as `collection_method` terms in the lexicon: # 'LU_AltitudeMethod','LU_CoordinateMethod'. @@ -116,7 +120,7 @@ def data_provenance(cls): "DataProvenance", primaryjoin=and_( cls.id == foreign(DataProvenance.target_id), - DataProvenance.target_table == pascal_to_snake(cls.__name__), + DataProvenance.target_table == cls.__tablename__, ), lazy="selectin", viewonly=True, diff --git a/db/geologic_formation.py b/db/geologic_formation.py new file mode 100644 index 000000000..2379f50f4 --- /dev/null +++ b/db/geologic_formation.py @@ -0,0 +1,82 @@ +""" +SQLAlchemy model for the GeologicFormation table. + +This table is a master reference table for geologic formations. Its purpose is to store definitions and descriptions +of various geologic formations that can be referenced by other tables in the database. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy import Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy +from geoalchemy2 import Geometry + +from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.lexicon import lexicon_term + +from constants import SRID_WGS84 + +if TYPE_CHECKING: + from db.thing import Thing, WellScreen + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) + + +class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin): + __versioned__ = {} + + # TODO: Let the API map formation codes to names using a formations.json file that can be periodically updated + # from the authoritative source (.e.g USGS). A placeholder `formations.json` file had been added to the `core` + # directory. + # name: Mapped[str] = mapped_column( + # nullable=False, + # unique=True, + # comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').", + # ) + formation_code: Mapped[str] = lexicon_term( + nullable=True, + unique=True, + comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').", + ) + description: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A detailed description of the geologic formation, its characteristics, and its significance.", + ) + # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table. + # This should be implemented after AMMP reviews and cleans up their formation terms and codes. + lithology: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary for the primary, dominant rock type" + "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').", + ) + boundary: Mapped[Geometry] = mapped_column( + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True), + nullable=True, + comment="A spatial representation of the geologic formation's extent.", + ) + + # --- Relationships --- + # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the + # ThingGeologicFormationAssociation join table. + thing_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="geologic_formation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + ) + # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it. + well_screens: Mapped[List["WellScreen"]] = relationship( + "WellScreen", back_populates="geologic_formation", passive_deletes=True + ) + + # --- Association Proxies --- + # Provides direct access to Things (wells) that penetrate this formation. + things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing") + + # --- Table Arguments --- + __table_args__ = (Index("ix_geologicformation_formation_code", "formation_code"),) diff --git a/db/notes.py b/db/notes.py index ab8384064..0e2e8ab8b 100644 --- a/db/notes.py +++ b/db/notes.py @@ -97,7 +97,7 @@ def notes(cls): "Notes", primaryjoin=and_( cls.id == foreign(Notes.target_id), - Notes.target_table == cls.__name__, + Notes.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", @@ -120,7 +120,7 @@ def add_note( content=content, note_type=note_type, target_id=self.id, - target_table=self.__class__.__name__, + target_table=self.__class__.__tablename__, release_status=release_status, ) diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index 340e587f7..000000000 --- a/db/permission.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/permission_history.py b/db/permission_history.py new file mode 100644 index 000000000..591046bba --- /dev/null +++ b/db/permission_history.py @@ -0,0 +1,96 @@ +""" +models/permission.py + +This model defines the `Permission` table, a polymorphic table that tracks +all legal and administrative agreements related to site access and activity. +Its purpose is to track who granted permission, what activities they authorized, +which entity the permission applies to, and for what period of time. +""" + +from typing import TYPE_CHECKING +from datetime import date +from sqlalchemy import Integer, ForeignKey, String, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + + +if TYPE_CHECKING: + from db.contact import Contact + from db.thing import Thing + from db.location import Location + + +class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a specific grant of permission from a Contact for a + specific entity (e.g., a Thing or Location). + """ + + # --- Foreign Keys --- + contact_id: Mapped[int] = mapped_column( + Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + ) + + # --- Columns --- + permission_type: Mapped[str] = lexicon_term(nullable=False) + permission_allowed: Mapped[bool] = mapped_column(nullable=False, default=False) + start_date: Mapped[date] = mapped_column(nullable=False) + end_date: Mapped[date] = mapped_column(nullable=True) + notes: Mapped[str] = mapped_column(nullable=True) + + # --- Polymorphic Columns --- + target_id: Mapped[int] = mapped_column(nullable=False) + target_table: Mapped[str] = mapped_column(String(50), nullable=False) + + # --- Relationships --- + # Many-To-One: A Permission is granted by one Contact. + contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to find the specific parent record for a given child. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " + "PermissionHistory.target_table == 'thing')", + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " + "PermissionHistory.target_table == 'location')", + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.target_table}_target") + + +class PermissionHistoryMixin: + """ + Mixin for models that can have permissions (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + Permission table. + """ + + @declared_attr + def permission_history(cls): + # One-to-Many polymorphic relationship + return relationship( + "PermissionHistory", + primaryjoin=( + and_( + cls.id == foreign(PermissionHistory.target_id), + PermissionHistory.target_table == cls.__tablename__, + ) + ), + lazy="selectin", + viewonly=True, + ) diff --git a/db/status_history.py b/db/status_history.py index 8b3ee2321..15b5aec2f 100644 --- a/db/status_history.py +++ b/db/status_history.py @@ -19,7 +19,7 @@ ) from sqlalchemy.orm import Mapped, mapped_column, declared_attr, relationship, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term, pascal_to_snake +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term class StatusHistory(Base, AutoBaseMixin, ReleaseMixin): @@ -47,7 +47,7 @@ def status_history(cls): "StatusHistory", primaryjoin=and_( cls.id == foreign(StatusHistory.target_id), - StatusHistory.target_table == pascal_to_snake(cls.__name__), + StatusHistory.target_table == cls.__tablename__, ), cascade="all, delete-orphan", lazy="selectin", diff --git a/db/thing.py b/db/thing.py index 9f30d08e2..cec618259 100644 --- a/db/thing.py +++ b/db/thing.py @@ -26,8 +26,9 @@ AutoBaseMixin, Base, ReleaseMixin, - PermissionMixin, ) +from db.permission_history import PermissionHistoryMixin +from services.util import retrieve_latest_polymorphic_history_table_record from db.status_history import StatusHistoryMixin from db.measuring_point_history import MeasuringPointHistory from db.data_provenance import DataProvenanceMixin @@ -40,6 +41,12 @@ from db.sensor import Sensor from db.contact import Contact from db.group import Group, GroupThingAssociation + from db.aquifer_system import AquiferSystem + from db.thing_aquifer_association import ThingAquiferAssociation + from db.geologic_formation import GeologicFormation + from db.thing_geologic_formation_association import ( + ThingGeologicFormationAssociation, + ) class Thing( @@ -47,7 +54,7 @@ class Thing( AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, - PermissionMixin, + PermissionHistoryMixin, DataProvenanceMixin, NotesMixin, ): @@ -64,10 +71,6 @@ class Thing( comment="To audit where the data came from in NM_Aquifer if it was transferred over", ) - # notes = mapped_column(Text, nullable=True) - # measuring_notes = mapped_column(Text, nullable=True) - # water_notes = mapped_column(Text, nullable=True) - # TODO: should `name` be unique? name: Mapped[str] = mapped_column( nullable=False, @@ -116,6 +119,32 @@ class Thing( well_construction_notes: Mapped[str] = mapped_column(Text, nullable=True) + well_completion_date: Mapped[date] = mapped_column( + nullable=True, comment="the date the well was completed if known" + ) + well_driller_name: Mapped[str] = mapped_column( + String(200), nullable=True, comment="Name of the well driller." + ) + well_construction_method = lexicon_term(nullable=True) + well_pump_type: Mapped[str] = lexicon_term(nullable=True) + well_pump_depth: Mapped[float] = mapped_column( + Float, + nullable=True, + info={"unit": "feet below ground surface"}, + comment="Depth of the well pump from ground surface to the pump intake (in feet).", + ) + formation_completion_code: Mapped[str] = lexicon_term( + nullable=True, + comment="The geologic formation in which the well was completed (from WellData.FormationZone). " + "This indicates the target formation for the well, not the full stratigraphic column. " + "For detailed depth-interval stratigraphy, see formation_associations.", + ) + # TODO: should this be required for every well in the database? AMMP review + is_suitable_for_datalogger: Mapped[bool] = mapped_column( + nullable=True, + comment="Indicates if the well is suitable for datalogger installation.", + ) + # Spring-related columns spring_type: Mapped[str] = lexicon_term( nullable=True, @@ -263,6 +292,26 @@ class Thing( lazy="joined", ) + # One-To-Many: A Thing can be associated with many AquiferSystems via the ThingAquiferAssociation join table. + aquifer_associations: Mapped[List["ThingAquiferAssociation"]] = relationship( + "ThingAquiferAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) + + # Many-To-Many: A Thing can penetrate many GeologicFormations. + formation_associations: Mapped[List["ThingGeologicFormationAssociation"]] = ( + relationship( + "ThingGeologicFormationAssociation", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" @@ -288,6 +337,16 @@ class Thing( "group_associations", "group" ) + # Proxy to directly access AquiferSystems associated with this Thing + aquifer_systems: AssociationProxy[List["AquiferSystem"]] = association_proxy( + "aquifer_associations", "aquifer_system" + ) + + # Proxy to directly access the GeologicFormations penetrated by this Thing. + geologic_formations: AssociationProxy[List["GeologicFormation"]] = ( + association_proxy("formation_associations", "geologic_formation") + ) + # Full-text search vector search_vector = Column(TSVectorType("name", "well_construction_notes")) @@ -379,7 +438,48 @@ def measuring_point_description(self) -> str | None: @property def well_depth_source(self) -> str | None: - return self._get_data_provenance_attribute("well_depth", "origin_source") + return self._get_data_provenance_attribute("well_depth", "origin_type") + + @property + def well_completion_date_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_completion_date", "origin_type" + ) + + @property + def well_construction_method_source(self) -> str | None: + return self._get_data_provenance_attribute( + "well_construction_method", "origin_source" + ) + + @property + def aquifers(self) -> List[dict]: + """ + Returns a list of aquifer systems and their associated types for this Thing. + Each aquifer system is represented as a dictionary with its name and a list of types. + """ + aquifer_list = [] + for association in self.aquifer_associations: + aquifer_info = { + "aquifer_system": association.aquifer_system.name, + "aquifer_types": [ + atype.aquifer_type for atype in association.aquifer_types + ], + } + aquifer_list.append(aquifer_info) + return aquifer_list + + @property + def permissions(self) -> list: + """ + Returns the associated permissions or an empty list. If there are no + associated permissions, an empty list is returned instead of None to + allow the API to serialize correctly (see schemas/thing.py). + """ + if self.permission_history: + return self.permission_history + else: + return [] class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): @@ -406,6 +506,12 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): thing_id: Mapped[int] = mapped_column( ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="SET NULL"), nullable=True + ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), nullable=True + ) screen_depth_top: Mapped[float] = mapped_column( info={"unit": "feet below ground surface"}, nullable=True ) @@ -423,6 +529,14 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): # Many-To-One: A WellScreen belongs to one Thing. thing: Mapped["Thing"] = relationship("Thing", back_populates="screens") + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="well_screens", passive_deletes=True + ) + + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="well_screens", passive_deletes=True + ) + class WellPurpose(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py new file mode 100644 index 000000000..cca5758a9 --- /dev/null +++ b/db/thing_aquifer_association.py @@ -0,0 +1,51 @@ +""" +SQLAlchemy model for the ThingAquiferAssociation table. + +This table is a join table (or "association object") whose purpose is to manage +the many-to-many relationship between a Thing and an AquiferSystem. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey + +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.aquifer_system import AquiferSystem + from db.aquifer_type import AquiferType + + +class ThingAquiferAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the association of a Thing to an AquiferSystem. This is an Association Object. + """ + + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + aquifer_system_id: Mapped[int] = mapped_column( + ForeignKey("aquifer_system.id", ondelete="CASCADE"), nullable=False + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="aquifer_associations", lazy="joined" + ) + + # Many-To-One: This association links to one AquiferSystem. + aquifer_system: Mapped["AquiferSystem"] = relationship( + "AquiferSystem", back_populates="thing_associations", lazy="joined" + ) + # One-To-Many: An association can have multiple aquifer types. + aquifer_types: Mapped[list["AquiferType"]] = relationship( + "AquiferType", + back_populates="thing_aquifer_association", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="joined", + ) diff --git a/db/thing_geologic_formation_association.py b/db/thing_geologic_formation_association.py new file mode 100644 index 000000000..0707df269 --- /dev/null +++ b/db/thing_geologic_formation_association.py @@ -0,0 +1,60 @@ +""" +SQLAlchemy model for the ThingGeologicFormationAssociation table. + +This table is an association object that creates a many-to-many relationship between a Thing (well) and a +GeologicFormation. It stores the lithology for a well, detailing the depth intervals for each formation it penetrates. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.thing import Thing + from db.geologic_formation import GeologicFormation + + +class ThingGeologicFormationAssociation(Base, AutoBaseMixin, ReleaseMixin): + """ + This is a= join table (Association Object). It represents the association of a Thing to a + GeologicFormation at a specific depth interval. + """ + + # --- Foreign Keys --- + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), + nullable=False, + comment="The foreign key linking this record to the `Thing` table." + "Deleting a `Thing` will cascade and delete its formation log.", + ) + geologic_formation_id: Mapped[int] = mapped_column( + ForeignKey("geologic_formation.id", ondelete="SET NULL"), + nullable=True, + comment="The foreign key linking this record to the `GeologicFormation` table." + "This is set to `SET NULL` on delete, as deleting a formation definition (a rare admin action)" + "should not delete the historical fact that a well had a pick at this depth.", + ) + + # Depth interval fields + top_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the top of the geologic formation, as measured from ground surface.", + ) + bottom_depth: Mapped[float] = mapped_column( + nullable=False, + comment="The depth (in feet) to the bottom of the geologic formation, as measured from ground surface.", + ) + + # --- Relationship Definitions --- + # Many-To-One: This association links to one Thing. + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="formation_associations", lazy="joined" + ) + + # Many-To-One: This association links to one GeologicFormation. + geologic_formation: Mapped["GeologicFormation"] = relationship( + "GeologicFormation", back_populates="thing_associations", lazy="joined" + ) diff --git a/run_bdd.sh b/run_bdd.sh index 3674ae7c8..cd05769e4 100755 --- a/run_bdd.sh +++ b/run_bdd.sh @@ -66,7 +66,7 @@ export BASE_URL=${BASE_URL:-http://localhost:8000} # tests/features/thing-query-parameters.feature #uv run behave tests/features/well-inventory-csv.feature -# uv run behave tests/features/well-core-information.feature --capture +# uv run behave tests/features/well-additional-information.feature --capture uv run behave tests/features --tags="@backend and @production" --capture echo "✅ BDD test run complete." diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py new file mode 100644 index 000000000..1e1961873 --- /dev/null +++ b/schemas/aquifer_system.py @@ -0,0 +1,51 @@ +from typing import List + +from pydantic import BaseModel +from schemas import BaseResponseModel +from schemas.validators import GeometryMixin +from core.enums import AquiferType, GeographicScale # Import specific Enums + + +# ------ CREATE ---------- +class CreateAquiferSystem(GeometryMixin): + """ + Schema for creating an aquifer system. + Used during data transfer and API creation. + """ + + name: str + description: str | None = None + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None = None + # boundary field inherited from GeometryMixin + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] + + +class GeoJSONProperties(BaseResponseModel): + """ + Response schema for aquifer system details. + """ + + name: str + description: str | None = None + primary_aquifer_type: AquiferType + geographic_scale: GeographicScale | None + + +class AquiferSystemGeoJSONResponse(BaseModel): + """ + Response schema for aquifer system details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py new file mode 100644 index 000000000..67a3cb24a --- /dev/null +++ b/schemas/geologic_formation.py @@ -0,0 +1,88 @@ +from typing import List + +from pydantic import BaseModel, field_validator, Field + +from schemas import BaseResponseModel +from schemas.validators import DepthIntervalMixin, GeometryMixin +from core.enums import FormationCode, Lithology + + +# ------ CREATE ---------- +class CreateGeologicFormation(GeometryMixin): + """ + Schema for creating a geologic formation. + Used during data transfer and API creation. + """ + + # formation_code has its own custom uppercase validator + formation_code: FormationCode | None = None + description: str | None = None + lithology: Lithology | None = None + # boundary: inherited from GeometryMixin + + @field_validator("formation_code", mode="before") + @classmethod + def upper_case_code(cls, v: str | None) -> str | None: + """ + Automatically uppercase the formation code. + """ + if isinstance(v, str): + return v.upper() + return v + + +class CreateThingGeologicFormationAssociation(DepthIntervalMixin): + """ + Schema for linking a Thing (Well) to a GeologicFormation. + Uses DepthIntervalMixin to enforce bottom_depth > top_depth. + """ + + thing_id: int + geologic_formation_id: int + top_depth: float = Field(ge=0) + bottom_depth: float = Field(ge=0) + + +# ------ RESPONSE ---------- +class GeoJSONGeometry(BaseModel): + """ + Geometry schema for GeoJSON response. + """ + + type: str = "MULTIPOLYGON" + coordinates: List[List[List[float]]] + + +class GeoJSONProperties(BaseResponseModel): + """ + Response schema for geologic formation details. + """ + + formation_code: str | None = None + description: str | None = None + lithology: str | None = None + + +class GeologicFormationGeoJSONResponse(BaseModel): + """ + Response schema for geologic formation details. + """ + + type: str = "Feature" + geometry: GeoJSONGeometry + properties: GeoJSONProperties + + +class ThingGeologicFormationAssociationResponse(BaseResponseModel): + """ + Response schema for the association between a Thing and a GeologicFormation. + Includes depth interval information. + """ + + thing_id: int + geologic_formation_id: int | None = None + geologic_formation: GeologicFormationGeoJSONResponse | None = None + top_depth: float + top_depth_unit: str = "ft" + bottom_depth: float + bottom_depth_unit: str = "ft" diff --git a/schemas/location.py b/schemas/location.py index e911e3359..218790496 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -97,7 +97,7 @@ class GeoJSONUTMCoordinates(BaseModel): ) -class GeoJSONProperties(BaseModel): +class GeoJSONProperties(BaseResponseModel): elevation: float elevation_unit: str = "ft" vertical_datum: str = "NAVD88" @@ -147,6 +147,9 @@ def populate_fields(cls, data: Any) -> Any: data_dict["geometry"]["coordinates"] = coordinates # populate properties + data_dict["properties"]["id"] = data_dict.get("id") + data_dict["properties"]["created_at"] = data_dict.get("created_at") + data_dict["properties"]["release_status"] = data_dict.get("release_status") data_dict["properties"]["notes"] = data_dict.get("notes") data_dict["properties"]["elevation"] = convert_m_to_ft(elevation_m) data_dict["properties"]["elevation_method"] = data_dict.get("elevation_method") diff --git a/schemas/permission_history.py b/schemas/permission_history.py new file mode 100644 index 000000000..e0619d90e --- /dev/null +++ b/schemas/permission_history.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel +from schemas import PastOrTodayDate + +from core.enums import PermissionType + + +# ------ RESPONSE ---------- +class PermissionHistoryResponse(BaseModel): + """ + Even though permission_allowed and start_date are not-nullable in the + database, they are nullable here to accommodate cases where no permission + record exists for a given permission type. + """ + + permission_type: PermissionType + permission_allowed: bool | None + start_date: PastOrTodayDate | None + end_date: PastOrTodayDate | None diff --git a/schemas/thing.py b/schemas/thing.py index cf8c3ef2b..7a7982494 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -24,12 +24,17 @@ ScreenType, Organization, MonitoringFrequency, + Organization, + MonitoringFrequency, + WellConstructionMethod, + WellPumpType, + FormationCode, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote - +from schemas.permission_history import PermissionHistoryResponse # -------- VALIDATE ---------- @@ -128,8 +133,16 @@ class CreateWell(CreateBaseThing, ValidateWell): measuring_point_height: float = Field( ge=0, description="Measuring point height in feet" ) - measuring_point_description: str | None + measuring_point_description: str | None = None notes: list[CreateNote] | None = None + well_completion_date: PastOrTodayDate | None = None + well_completion_date_source: str | None = None + well_driller_name: str | None = None + well_construction_method: WellConstructionMethod | None = None + well_construction_method_source: str | None = None + well_pump_type: WellPumpType | None = None + is_suitable_for_datalogger: bool | None + formation_completion_code: FormationCode | None = None class CreateSpring(CreateBaseThing): @@ -146,6 +159,8 @@ class CreateWellScreen(BaseCreateModel): """ thing_id: int + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float = Field(gt=0, description="Screen depth bottom in feet") screen_depth_top: float = Field(gt=0, description="Screen depth top in feet") screen_type: ScreenType | None = None @@ -220,14 +235,25 @@ class WellResponse(BaseThingResponse): well_casing_depth_unit: str = "ft" well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None + well_completion_date: PastOrTodayDate | None + well_completion_date_source: str | None + well_driller_name: str | None + well_construction_method: WellConstructionMethod | None + well_construction_method_source: str | None + well_pump_type: WellPumpType | None + well_pump_depth: float | None + well_pump_depth_unit: str = "ft" + is_suitable_for_datalogger: bool | None well_status: str | None measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None - + aquifers: list[dict] = [] water_notes: list[NoteResponse] | None = None measuring_notes: list[NoteResponse] | None = None general_notes: list[NoteResponse] | None = None + permissions: list[PermissionHistoryResponse] + formation_completion_code: FormationCode | None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): @@ -248,6 +274,43 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): materials = [] return materials + @field_validator("permissions", mode="before") + def populate_permission_history_with_latest_records(cls, permissions): + """ + Populate the permission history with the latest records for each + type of permission. If multiple records exist for the same permission type + only the most recent one is included. If there are no records + the permission_allowed will be None + """ + permissions_to_return = [] + for permission_type in [ + "Water Level Sample", + "Water Chemistry Sample", + "Datalogger Installation", + ]: + # Filter records for the current permission type + filtered_records = [ + record + for record in permissions + if record.permission_type == permission_type and record.end_date is None + ] + if filtered_records: + # Get the most recent record based on start_date + latest_record = max( + filtered_records, key=lambda record: record.start_date + ) + permissions_to_return.append(latest_record) + else: + permissions_to_return.append( + PermissionHistoryResponse( + permission_type=permission_type, + permission_allowed=None, + start_date=None, + end_date=None, + ) + ) + return permissions_to_return + class SpringResponse(BaseThingResponse): """ @@ -269,6 +332,11 @@ class WellScreenResponse(BaseResponseModel): thing_id: int thing: WellResponse + aquifer_system_id: int | None = None + aquifer_system: str | None = None + aquifer_type: str | None = None + geologic_formation_id: int | None = None + geologic_formation: str | None = None screen_depth_bottom: float screen_depth_bottom_unit: str = "ft" screen_depth_top: float @@ -276,6 +344,24 @@ class WellScreenResponse(BaseResponseModel): screen_type: str | None = None screen_description: str | None = None + @field_validator("aquifer_system", mode="before") + def populate_aquifer_system_with_name(cls, aquifer_system): + if aquifer_system is not None: + return aquifer_system.name + return None + + @field_validator("aquifer_type", mode="before") + def populate_aquifer_type_with_name(cls, aquifer_type): + if aquifer_type is not None: + return aquifer_type.name + return None + + @field_validator("geologic_formation", mode="before") + def populate_geologic_formation_with_code(cls, geologic_formation): + if geologic_formation is not None: + return geologic_formation.formation_code + return None + class GeoJSONGeometry(BaseModel): """ @@ -342,6 +428,8 @@ class UpdateThingIdLink(BaseUpdateModel): class UpdateWellScreen(BaseUpdateModel): + aquifer_system_id: int | None = None + geologic_formation_id: int | None = None screen_depth_bottom: float | None = None screen_depth_top: float | None = None screen_description: str | None = None diff --git a/schemas/validators.py b/schemas/validators.py new file mode 100644 index 000000000..963047bc2 --- /dev/null +++ b/schemas/validators.py @@ -0,0 +1,43 @@ +""" +schemas/validators.py +Reusable Pydantic validators and mixins for aquifer and geology related schemas. +May consider expansion for other domain models in the future. +""" + +from pydantic import model_validator, field_validator, BaseModel, Field +from services.validation.geospatial import validate_wkt_geometry + + +class DepthIntervalMixin(BaseModel): + """ + Mixin to enforce: + 1. Depths are non-negative (via Field constraints). + 2. Bottom depth > top depth (via model_validator). + Assumes the model has 'top_depth' and 'bottom_depth' fields. + """ + + top_depth: float = Field(ge=0) + bottom_depth: float = Field(ge=0) + + @model_validator(mode="after") + def check_depth_logical_order(self) -> "DepthIntervalMixin": + if self.bottom_depth <= self.top_depth: + raise ValueError( + f"Bottom depth ({self.bottom_depth}) must be greater " + f"than top depth ({self.top_depth})" + ) + return self + + +class GeometryMixin(BaseModel): + """ + Mixin to validate WKT strings for boundary fields. + Delegates logic to the validate_wkt_geometry service function. + """ + + boundary: str | None = None + + @field_validator("boundary") + @classmethod + def validate_wkt(cls, v: str | None) -> str | None: + return validate_wkt_geometry(v) diff --git a/services/util.py b/services/util.py index 77cd5d5cd..6fbdd0269 100644 --- a/services/util.py +++ b/services/util.py @@ -4,12 +4,14 @@ import pyproj import httpx from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import DeclarativeBase from constants import SRID_WGS84 TRANSFORMERS = {} METERS_TO_FEET = 3.28084 +METERS_TO_FEET = 3.28084 def transform_srid(geometry, source_srid, target_srid): @@ -43,6 +45,20 @@ def convert_ft_to_m(feet: float | None) -> float | None: return round(feet / METERS_TO_FEET, 6) +def convert_m_to_ft(meters: float | None) -> float | None: + """Convert a length from meters to feet.""" + if meters is None: + return None + return round(meters * METERS_TO_FEET, 6) + + +def convert_ft_to_m(feet: float | None) -> float | None: + """Convert a length from feet to meters.""" + if feet is None: + return None + return round(feet / METERS_TO_FEET, 6) + + def get_tiger_data( lon: float, lat: float, layer: int, outfields: str = "*" ) -> dict | None: @@ -181,11 +197,10 @@ def retrieve_latest_polymorphic_history_table_record( DeclarativeBase | None The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permissions": + if polymorphic_relationship == "permission_history": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" - polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ r diff --git a/tests/features/environment.py b/tests/features/environment.py index 9b801e9d7..a89eacd00 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -28,12 +28,20 @@ Parameter, Deployment, TransducerObservationBlock, + WellCasingMaterial, + PermissionHistory, + Contact, StatusHistory, ThingIdLink, WellPurpose, MeasuringPointHistory, MonitoringFrequencyHistory, DataProvenance, + AquiferSystem, + AquiferType, + ThingAquiferAssociation, + GeologicFormation, + ThingGeologicFormationAssociation, ) from db.engine import session_ctx @@ -87,9 +95,13 @@ def add_well(context, session, location, name_num): well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, - # notes="These are some test well notes", - # measuring_notes="These are some measuring notes", - # water_notes="This are some water notes", + well_completion_date="2013-05-15", + well_driller_name="Jonsi", + well_construction_method="Driven", + well_pump_type="Submersible", + well_pump_depth=8, + is_suitable_for_datalogger=True, + formation_completion_code="000EXRV", ) session.add(well) @@ -116,6 +128,20 @@ def add_well(context, session, location, name_num): return well +@add_context_object_container("well_casing_materials") +def add_well_casing_material(context, session, well): + wcm = WellCasingMaterial( + thing_id=well.id, + material="PVC", + ) + session.add(wcm) + session.commit() + session.refresh(wcm) + + context.objects["well_casing_materials"].append(wcm) + return wcm + + @add_context_object_container("well_purposes") def add_well_purpose(context, session, well, purpose_term): purpose = WellPurpose(thing=well, purpose=purpose_term) @@ -189,6 +215,54 @@ def add_spring(context, session, location, name_num): return spring +@add_context_object_container("contacts") +def add_contact(context, session): + contact = Contact( + name="Test Contact", + role="Software Developer", + organization="NMBGMR", + release_status="draft", + contact_type="Primary", + ) + session.add(contact) + session.commit() + session.refresh(contact) + + context.objects["contacts"].append(contact) + return contact + + +@add_context_object_container("permission_histories") +def add_permission_history( + context, + session, + contact_id, + permission_type, + permission_allowed, + start_date, + end_date, + notes, + target_id, + target_table, +): + permission_history = PermissionHistory( + contact_id=contact_id, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=end_date, + notes=notes, + target_id=target_id, + target_table=target_table, + ) + session.add(permission_history) + session.commit() + session.refresh(permission_history) + + context.objects["permission_histories"].append(permission_history) + return permission_history + + @add_context_object_container("sensors") def add_sensor(context, session): sensor = Sensor( @@ -317,7 +391,8 @@ def add_data_provenance( target_id, target_table, field_name, - origin_source, + origin_type=None, + origin_source=None, collection_method=None, accuracy_value=None, accuracy_unit=None, @@ -327,6 +402,7 @@ def add_data_provenance( collection_method=collection_method, target_id=target_id, target_table=target_table, + origin_type=origin_type, origin_source=origin_source, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, @@ -353,9 +429,74 @@ def add_transducer_observation(context, session, block, deployment_id, value): return obs +@add_context_object_container("aquifer_systems") +def add_aquifer_system(context, session, name, well): + aquifer_system = AquiferSystem( + name=name, + description="this is a test aquifer", + primary_aquifer_type="Artesian", + geographic_scale="Major", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(aquifer_system) + session.commit() + session.refresh(aquifer_system) + + context.objects["aquifer_systems"].append(aquifer_system) + return aquifer_system + + +@add_context_object_container("thing_aquifer_associations") +def add_thing_aquifer_association(context, session, well, aquifer_system): + association = ThingAquiferAssociation(thing=well, aquifer_system=aquifer_system) + session.add(association) + session.commit() + session.refresh(association) + + context.objects["thing_aquifer_associations"].append(association) + return association + + +@add_context_object_container("aquifer_types") +def add_aquifer_type(context, session, aquifer_type_str, thing_aquifer_association): + aquifer_type = AquiferType( + aquifer_type=aquifer_type_str, + thing_aquifer_association=thing_aquifer_association, + ) + session.add(aquifer_type) + session.commit() + session.refresh(aquifer_type) + + context.objects["aquifer_types"].append(aquifer_type) + return aquifer_type + + +@add_context_object_container("geologic_formations") +def add_geologic_formation(context, session, formation_code, well): + formation = GeologicFormation( + formation_code=formation_code, + description="This is a test geologic formation.", + lithology="Peat", + boundary="MULTIPOLYGON(((0 0, 1 1, 2 2, 3 3, 1 2, 0 0)))", + ) + session.add(formation) + session.commit() + session.refresh(formation) + + association = ThingGeologicFormationAssociation( + top_depth=1, bottom_depth=10, thing=well, geologic_formation=formation + ) + session.add(association) + session.commit() + session.refresh(association) + + context.objects["geologic_formations"].append(formation) + return formation + + def before_all(context): context.objects = {} - rebuild = False + rebuild = True # rebuild = True if rebuild: erase_and_rebuild_db() @@ -374,6 +515,28 @@ def before_all(context): sensor_1 = add_sensor(context, session) deployment = add_deployment(context, session, well_1.id, sensor_1.id) + add_well_casing_material(context, session, well_1) + + contact = add_contact(context, session) + + for permission in [ + "Datalogger Installation", + "Water Level Sample", + "Water Chemistry Sample", + ]: + add_permission_history( + context, + session, + contact_id=context.objects["contacts"][0].id, + permission_type=permission, + permission_allowed=True, + start_date=datetime(2025, 1, 1).date(), + end_date=None, + notes=f"Permission granted for {permission.lower()}.", + target_id=well_1.id, + target_table="thing", + ) + measuring_point_history_1 = add_measuring_point_history( context, session, well=well_1 ) @@ -477,6 +640,8 @@ def before_all(context): alternate_organization="NMBGMR", ) + add_well_casing_material(context, session, well_1) + group = add_group(context, session, [well_1, well_2]) elevation_method = add_data_provenance( @@ -495,12 +660,40 @@ def before_all(context): target_id=well_1.id, target_table="thing", field_name="well_depth", - origin_source="Other", + origin_type="Other", + ) + + well_completion_date_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_completion_date", + origin_type="Data Portal", + ) + + well_construction_method_source = add_data_provenance( + context, + session, + target_id=well_1.id, + target_table="thing", + field_name="well_construction_method", + origin_source="Jacob's 2013 Thesis", ) for purpose in ["Domestic", "Irrigation"]: add_well_purpose(context, session, well_1, purpose) + for name in ["Aquifer A", "Aquifer B"]: + system = add_aquifer_system(context, session, name, well_1) + add_thing_aquifer_association(context, session, well_1, system) + + for t in ["Artesian", "Fractured"]: + taa = context.objects["thing_aquifer_associations"][0] + add_aquifer_type(context, session, t, taa) + + add_geologic_formation(context, session, "000EXRV", well_1) + # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) block = add_block(context, session, parameter) @@ -519,8 +712,10 @@ def before_all(context): def after_all(context): with session_ctx() as session: for table in context.objects.values(): - for obj in table: - session.delete(obj) + for record in table: + obj = session.get(record.__class__, record.id) + if obj: + session.delete(obj) session.commit() diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py new file mode 100644 index 000000000..8b00f7eb7 --- /dev/null +++ b/tests/features/steps/well-additional-information.py @@ -0,0 +1,270 @@ +from behave import then + +from services.util import retrieve_latest_polymorphic_history_table_record + + +# ------------------------------------------------------------------------------ +# Permissions / Operational OK flags +# ------------------------------------------------------------------------------ +@then( + "the response should include whether repeat measurement permission is granted for the well" +) +def step_impl(context): + permission_type = "Water Level Sample" + assert "permissions" in context.water_well_data + + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", permission_type + ) + + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) + assert ( + water_well_data_permissions["permission_allowed"] + == permission_record.permission_allowed + ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None + + +@then("the response should include whether sampling permission is granted for the well") +def step_impl(context): + permission_type = "Water Chemistry Sample" + assert "permissions" in context.water_well_data + + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", permission_type + ) + + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) + assert ( + water_well_data_permissions["permission_allowed"] + == permission_record.permission_allowed + ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None + + +@then( + "the response should include whether datalogger installation permission is granted for the well" +) +def step_impl(context): + permission_type = "Datalogger Installation" + assert "permissions" in context.water_well_data + + permission_record = retrieve_latest_polymorphic_history_table_record( + context.objects["wells"][0], "permission_history", permission_type + ) + + water_well_data_permissions = [ + p + for p in context.water_well_data["permissions"] + if p["permission_type"] == permission_type + ][0] + assert ( + water_well_data_permissions["permission_type"] + == permission_record.permission_type + ) + assert ( + water_well_data_permissions["permission_allowed"] + == permission_record.permission_allowed + ) + assert water_well_data_permissions[ + "start_date" + ] == permission_record.start_date.strftime("%Y-%m-%d") + if permission_record.end_date: + assert water_well_data_permissions[ + "end_date" + ] == permission_record.end_date.strftime("%Y-%m-%d") + else: + assert water_well_data_permissions["end_date"] is None + + +# ------------------------------------------------------------------------------ +# Well Construction Information +# ------------------------------------------------------------------------------ + + +@then("the response should include the completion date of the well") +def step_impl(context): + assert "well_completion_date" in context.water_well_data + assert context.water_well_data["well_completion_date"] == context.objects["wells"][ + 0 + ].well_completion_date.strftime("%Y-%m-%d") + + +@then("the response should include the source of the completion information") +def step_impl(context): + assert "well_completion_date_source" in context.water_well_data + + assert ( + context.water_well_data["well_completion_date_source"] + == context.objects["wells"][0].well_completion_date_source + ) + + +@then("the response should include the driller name") +def step_impl(context): + assert "well_driller_name" in context.water_well_data + assert ( + context.water_well_data["well_driller_name"] + == context.objects["wells"][0].well_driller_name + ) + + +@then("the response should include the construction method") +def step_impl(context): + assert "well_construction_method" in context.water_well_data + assert ( + context.water_well_data["well_construction_method"] + == context.objects["wells"][0].well_construction_method + ) + + +@then("the response should include the source of the construction information") +def step_impl(context): + assert "well_construction_method_source" in context.water_well_data + assert ( + context.water_well_data["well_construction_method_source"] + == context.objects["wells"][0].well_construction_method_source + ) + + +# ------------------------------------------------------------------------------ +# Additional Well Physical Properties +# ------------------------------------------------------------------------------ + + +@then("the response should include the casing diameter in inches") +def step_impl(context): + assert "well_casing_diameter" in context.water_well_data + assert "well_casing_diameter_unit" in context.water_well_data + + assert ( + context.water_well_data["well_casing_diameter"] + == context.objects["wells"][0].well_casing_diameter + ) + assert context.water_well_data["well_casing_diameter_unit"] == "in" + + +@then("the response should include the casing depth in feet below ground surface") +def step_impl(context): + assert "well_casing_depth" in context.water_well_data + assert "well_casing_depth_unit" in context.water_well_data + + assert ( + context.water_well_data["well_casing_depth"] + == context.objects["wells"][0].well_casing_depth + ) + assert context.water_well_data["well_casing_depth_unit"] == "ft" + + +@then("the response should include the casing materials") +def step_impl(context): + assert "well_casing_materials" in context.water_well_data + assert set(context.water_well_data["well_casing_materials"]) == { + m.material for m in context.objects["wells"][0].well_casing_materials + } + + +@then("the response should include the well pump type (previously well_type field)") +def step_impl(context): + assert "well_pump_type" in context.water_well_data + assert ( + context.water_well_data["well_pump_type"] + == context.objects["wells"][0].well_pump_type + ) + + +@then("the response should include the well pump depth in feet (new field)") +def step_impl(context): + assert "well_pump_depth" in context.water_well_data + assert "well_pump_depth_unit" in context.water_well_data + + assert ( + context.water_well_data["well_pump_depth"] + == context.objects["wells"][0].well_pump_depth + ) + assert context.water_well_data["well_pump_depth_unit"] == "ft" + + +@then( + "the response should include whether the well is open and suitable for a datalogger" +) +def step_impl(context): + assert "is_suitable_for_datalogger" in context.water_well_data + assert ( + context.water_well_data["is_suitable_for_datalogger"] + == context.objects["wells"][0].is_suitable_for_datalogger + ) + + +# ------------------------------------------------------------------------------ +# Aquifer/ Geology Information +# ------------------------------------------------------------------------------ + + +@then( + "the response should include the formation as the formation zone of well completion" +) +def step_impl(context): + assert "formation_completion_code" in context.water_well_data + assert ( + context.water_well_data["formation_completion_code"] + == context.objects["wells"][0].formation_completion_code + ) + + +@then( + "the response should include the aquifer class code to classify the aquifer into aquifer system." +) +def step_impl(context): + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_system" in aquifer + assert {a.get("aquifer_system") for a in context.water_well_data["aquifers"]} == { + system.name for system in context.objects["aquifer_systems"] + } + + +@then( + "the response should include the aquifer type as the type of aquifers penetrated by the well" +) +def step_impl(context): + for aquifer in context.water_well_data["aquifers"]: + assert "aquifer_types" in aquifer + + if aquifer["aquifer_system"] == "Aquifer A": + assert set(aquifer["aquifer_types"]) == { + a.aquifer_type for a in context.objects["aquifer_types"] + } + else: + assert aquifer["aquifer_types"] == [] diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index b0adc8346..1f56161f6 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -163,7 +163,7 @@ def step_impl(context): and r.target_table == "thing" and r.target_id == context.objects["wells"][0].id ] - well_depth_source = well_depth_source_records[0].origin_source + well_depth_source = well_depth_source_records[0].origin_type assert context.water_well_data["well_depth_source"] == well_depth_source diff --git a/tests/test_thing.py b/tests/test_thing.py index 28290dada..5bd504718 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -152,6 +152,9 @@ def test_add_water_well(location, group): cleanup_post_test(Thing, data["id"]) +@pytest.mark.skip( + "This duplicates the test above. That one will need to eventually be updated" +) def test_add_water_well_with_measuring_point(location, group): """ Test creating a well with measuring_point_height and measuring_point_description. diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py new file mode 100644 index 000000000..a0ba1f02e --- /dev/null +++ b/transfers/aquifer_system_transfer.py @@ -0,0 +1,141 @@ +import time +from sqlalchemy.orm import Session +from pydantic import ValidationError + +from db import AquiferSystem +from schemas.aquifer_system import CreateAquiferSystem +from transfers.util import read_csv, replace_nans, logger + + +def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: + """ + Transfer aquifer system data from LU_AquiferClass CSV to the database. + + This creates the master list of named aquifer systems (e.g., Ogallala Aquifer). the primary_type field is set + to "Unknown" as a placeholder and will be updated during well transfer when we know what type each well encounters. + + This should be run BEFORE well_transfer.py so that aquifer records exist for wells to reference. + + Args: + session (Session): SQLAlchemy database session + limit (int, optional): Limit the number of records to transfer (for testing). + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read the CSV file + input_df = read_csv("LU_AquiferClass") + + # 2. Replace NaNs with NOne + cleaned_df = replace_nans(input_df) + + # 3. Initialize tracking variables for logging + n = len(input_df) + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + + logger.info(f"Starting transfer of {n} aquifer systems from LU_AquiferClass.") + + # 4. Process each row + for i, row in enumerate(cleaned_df.itertuples()): + # check if limit is reached + if limit and i >= limit: + logger.info(f"Reached limit of {limit} rows. Stopping migration.") + break + + # Log progress every 'step' rows + if i and not i % step: + logger.info( + f"Processing row {i} of {n}. Avg rows per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Commit progress periodically + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing aquifer system {i}: {e}") + session.rollback() + continue + + # 5. Extract aquifer code and name + aquifer_code = row.CODE + aquifer_name = row.MEANING + + if not aquifer_name: + error_msg = f"Row {i} (code: {aquifer_code}) has no aquifer name (MEANING)." + logger.critical(error_msg) + errors.append({"row": i, "code": aquifer_code, "error": error_msg}) + skipped_count += 1 + continue + + # 6. Check if aquifer system already exists + existing = ( + session.query(AquiferSystem) + .filter(AquiferSystem.name == aquifer_name) + .first() + ) + + if existing: + logger.info( + f"Aquifer '{aquifer_name}' (code: {aquifer_code}) already exists. Skipping." + ) + skipped_count += 1 + continue + + # 7. Prepare data dictionary + try: + data = CreateAquiferSystem( + name=aquifer_name, + description=None, # can be updated later + primary_aquifer_type="Unknown", # placeholder - will be updated during well transfer + ) + + # Validate data using Pydantic schema + CreateAquiferSystem.model_validate(data) + + except ValidationError as e: + errors.append( + {"code": aquifer_code, "name": aquifer_name, "error": e.errors()} + ) + logger.critical( + f"Error creating aquifer system '{aquifer_name}' (code: {aquifer_code}) (row {i}): {e.errors()}" + ) + continue + + # 8. Create database record + aquifer_system = None + try: + aquifer_data = data.model_dump() + aquifer_system = AquiferSystem(**aquifer_data) + session.add(aquifer_system) + created_count += 1 + + logger.info( + f"Created aquifer system: {aquifer_system.name} (code: {aquifer_code})" + ) + + except Exception as e: + if aquifer_system is not None: + session.expunge(aquifer_system) + errors.append({"code": aquifer_code, "name": aquifer_name, "error": str(e)}) + logger.critical( + f"Error creating aquifer system record '{aquifer_name}': {e}" + ) + continue + + # 9. Final commit + try: + session.commit() + logger.info( + f"Successfully transferred {created_count} aquifer systems, skipped {skipped_count}. " + f"Note: primary_type set to 'Unknown' and will be updated during well transfer." + ) + except Exception as e: + logger.critical(f"Error in final commit: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/data/owners_organization_mapper.json b/transfers/data/owners_organization_mapper.json index 5ce45a8bf..b4f29bd7b 100644 --- a/transfers/data/owners_organization_mapper.json +++ b/transfers/data/owners_organization_mapper.json @@ -89,6 +89,7 @@ "Pecos Trail Inn": "Pecos Trail Inn", "Pelican Spa": "Pelican Spa", "Pistachio Tree Ranch": "Pistachio Tree Ranch", + "Quemado Mutual Water and Sewage Works Association": "Quemado Municipal Water & SWA", "Rancho Encantado": "Rancho Encantado", "Rancho San Lucas": "Rancho San Lucas", "Rancho San Marcos": "Rancho San Marcos", diff --git a/transfers/geologic_formation_transfer.py b/transfers/geologic_formation_transfer.py new file mode 100644 index 000000000..7fcd73e4c --- /dev/null +++ b/transfers/geologic_formation_transfer.py @@ -0,0 +1,141 @@ +import time +from sqlalchemy.orm import Session +from pydantic import ValidationError + +from db import GeologicFormation +from schemas.geologic_formation import CreateGeologicFormation +from transfers.util import read_csv, replace_nans, logger + + +def transfer_geologic_formations(session: Session, limit: int = None) -> tuple: + """ + Transfer geologic formation data from LU_GeologicFormation CSV to the database. + + This should be run BEFORE well_transfer.py so that geologic formation records exist for wells to reference. + + Args: + session (Session): SQLAlchemy database session + limit (int, optional): Optional limit on number of records to transfer (for testing). + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read the CSV file + input_df = read_csv("LU_Formations") + + # 2. Replace NaNs with None + cleaned_df = replace_nans(input_df) + + # 3. Initialize tracking variables for logging + n = len(cleaned_df) + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + + logger.info(f"Starting transfer of {n} geologic formations") + + # 4. Process each row + for i, row in enumerate(cleaned_df.itertuples()): + # check if limit is reached + if limit and i >= limit: + logger.info(f"Reached limit of {limit} rows. Stopping migration.") + break + + # Log progress every 'step' rows + if i and not i % step: + logger.info( + f"Processing row {i} of {n}. Avg rows per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Commit progress periodically + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing geologic formations: {e}") + session.rollback() + continue + + # 5. Extract formation code and description + formation_code = row.Code + + if not formation_code: + logger.warning(f"Skipping row {i}: Missing formation code") + skipped_count += 1 + continue + + # Check if this formation already exists + existing = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if existing: + logger.info( + f"Skipping row {i}: Formation code {formation_code} already exists" + ) + skipped_count += 1 + continue + + # 6. Prepare data for creation + # Note: We only store the formation_code. Formation names will be mapped by the API using a + # formations.json file from authoritative sources (e.g., USGS). + # The description field is left as None and can be populated later if needed. + # Note: lithology is set to None here and will be updated during stratigraphy transfer + try: + data = CreateGeologicFormation( + formation_code=formation_code, + description=None, # Not storing from legacy data + lithology=None, # Will be populated from Stratigraphy.csv + ) + + # Validate the data using Pydantic schema + CreateGeologicFormation.model_validate(data) + + except ValidationError as e: + errors.append({"code": formation_code, "errors": e.errors()}) + logger.critical( + f"Validation error for row {i} with Code {formation_code}: {e.errors()}" + ) + continue + except Exception as e: + errors.append({"code": formation_code, "errors": str(e)}) + logger.critical(f"Error preparing data for {formation_code}: {e}") + continue + + # 7. Create database object + geologic_formation = None + try: + formation_data = data.model_dump() + geologic_formation = GeologicFormation(**formation_data) + session.add(geologic_formation) + created_count += 1 + + logger.info( + f"Created geologic formation: {geologic_formation.formation_code}" + ) + + except Exception as e: + if geologic_formation is not None: + session.expunge(geologic_formation) + errors.append({"code": formation_code, "error": str(e)}) + logger.critical( + f"Error creating geologic formation for {formation_code}: {e}" + ) + continue + + # 8. Final commit + try: + session.commit() + logger.info( + f"Successfully transferred {created_count} geologic formations, skipped {skipped_count}. " + f"Note: lithology is None and will be updated during stratigraphy transfer." + ) + except Exception as e: + logger.critical(f"Error during final commit of geologic formations: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py new file mode 100644 index 000000000..18daa1040 --- /dev/null +++ b/transfers/permissions_transfer.py @@ -0,0 +1,95 @@ +from sqlalchemy.orm import Session +from datetime import datetime +from pandas import isna + +from db import Thing, PermissionHistory +from transfers.util import read_csv, logger, replace_nans + +""" +Developer's notes + +According to Laila the column WellData.OpenWellLoggerOK only pertains to the +physical properties of a well (that is, if a datalogger can be installed). It +does not pertain to permissions. +""" + + +def transfer_permissions(session: Session): + """ + The transferred wells and contacts need to be transferred first + - to access the auto-generated well IDs + - to know who gave permission to which well since contact_id is required for + PermissionHistory + """ + wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) + wdf = replace_nans(wdf) + + transferred_wells = ( + session.query(Thing).filter(Thing.thing_type == "water well").all() + ) + + for well in transferred_wells: + if len(well.contacts) == 0: + logger.critical( + f"Well {well.name} has no associated contacts; skipping permission transfer." + ) + continue + else: + # Assuming the first contact is the relevant one + contact_id = well.contacts[0].id + + allow_water_level_samples = wdf.loc[ + wdf["PointID"] == well.name, "MonitorOK" + ].values + if len(allow_water_level_samples) == 0: + pass + elif isna(allow_water_level_samples[0]): + pass + else: + try: + permission_allowed = bool(allow_water_level_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Level Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Level Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + allow_water_chemistry_samples = wdf.loc[ + wdf["PointID"] == well.name, "SampleOK" + ].values + if len(allow_water_chemistry_samples) == 0: + pass + elif isna(allow_water_chemistry_samples[0]): + pass + else: + try: + permission_allowed = bool(allow_water_chemistry_samples[0]) + permission = PermissionHistory( + contact_id=contact_id, + permission_type="Water Chemistry Sample", + permission_allowed=permission_allowed, + start_date=datetime.today().date(), + target_id=well.id, + target_table="thing", + ) + session.add(permission) + logger.info( + f"Transferred Water Chemistry Sample permission for well {well.name}: {permission_allowed}." + ) + except Exception as e: + logger.error(f"Error transferring permission for well {well.name}: {e}") + session.rollback() + pass + + session.commit() diff --git a/transfers/stratigraphy_transfer.py b/transfers/stratigraphy_transfer.py new file mode 100644 index 000000000..de51e354e --- /dev/null +++ b/transfers/stratigraphy_transfer.py @@ -0,0 +1,285 @@ +""" +Transfer script for stratigraphy (lithology log) data. + +This creates ThingGeologicFormationAssociation records from the Stratigraphy CSV, which contains depth-specific +formation information for wells. It also updates the GeologicFormation.lithology field based on the +Stratigraphy.Lithology data. +""" + +import time +from sqlalchemy.orm import Session + +from db import Thing, GeologicFormation, ThingGeologicFormationAssociation +from transfers.util import ( + read_csv, + replace_nans, + filter_to_valid_point_ids, + lexicon_mapper, + logger, +) + + +def transfer_stratigraphy(session: Session, limit: int = None) -> tuple: + """ + Transfer detailed stratigraphy (lithology log) data from Stratigraphy CSV. + + The Stratigraphy CSV contains multiple rows per well, each representing a + depth interval, the formation encountered, and its lithology. + + Fields used: + - PointID: Links to the well + - UnitIdentifier: Formation code (maps to LU_Formations) + - StratTop: Top depth of the layer (feet below ground surface) + - StratBottom: Bottom depth of the layer (feet below ground surface) + - Lithology: Lithology code (maps to LU_Lithology via ABBREVIATION field) + + This should be run AFTER: + 1. transfer_geologic_formations.py (so formations exist) + 2. transfer_wells.py (so wells exist) + + Args: + session: Database session + limit: Optional limit on number of WELLS to process (for testing) + + Returns: + tuple: (input_df, cleaned_df, errors) + """ + # 1. Read and clean data + input_df = read_csv("Stratigraphy") + cleaned_df = replace_nans(input_df) + + # Step 2: Filter to only wells that exist in database + cleaned_df = filter_to_valid_point_ids(session, cleaned_df) + + n_records = len(cleaned_df) + n_wells = len(cleaned_df["PointID"].unique()) + + logger.info( + f"Starting transfer of {n_records} stratigraphy records for {n_wells} wells" + ) + + # 3. Initialize tracking variables for logging + step = 25 + start_time = time.time() + errors = [] + created_count = 0 + skipped_count = 0 + lithology_updates = 0 + + # Step 4: Group by well for efficient processing + well_groups = cleaned_df.groupby("PointID") + + for well_index, (pointid, strat_group) in enumerate(well_groups): + # Check limit (on number of wells, not records) + if limit and well_index >= limit: + logger.info(f"Reached limit of {limit} wells. Stopping.") + break + + # Progress logging every 25 wells + if well_index and not well_index % step: + logger.info( + f"Processing well {well_index} of {n_wells}, " + f"avg wells per second: {step / (time.time() - start_time):.2f}" + ) + start_time = time.time() + + # Periodic commit + try: + session.commit() + except Exception as e: + logger.critical(f"Error committing stratigraphy records: {e}") + session.rollback() + continue + + # 5. Get the well from database + thing = session.query(Thing).filter(Thing.name == pointid).first() + if not thing: + logger.warning( + f"Well {pointid} not found in database, skipping stratigraphy" + ) + skipped_count += len(strat_group) + continue + + logger.info( + f"Processing {len(strat_group)} stratigraphy layers for well {pointid}" + ) + + # 6. Process each stratigraphy record for this well + for layer_index, row in enumerate(strat_group.itertuples()): + # Validate required fields + # UnitIdentifier + if not hasattr(row, "UnitIdentifier") or not row.UnitIdentifier: + logger.critical( + f"Stratigraphy record {layer_index} for {pointid} has no UnitIdentifier, skipping" + ) + skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing UnitIdentifier", + } + ) + continue + # StratTop + if not hasattr(row, "StratTop") or row.StratTop is None: + logger.critical( + f"Stratigraphy record {layer_index} for {pointid} has no StratTop, skipping" + ) + skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing StratTop", + } + ) + continue + # StratBottom + if not hasattr(row, "StratBottom") or row.StratBottom is None: + logger.critical( + f"Stratigraphy record {layer_index} for {pointid} has no StratBottom, skipping" + ) + skipped_count += 1 + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": "Missing StratBottom", + } + ) + continue + + # Extract formation code + formation_code = row.UnitIdentifier.strip() + + # Validate depth values + try: + top_depth = float(row.StratTop) + bottom_depth = float(row.StratBottom) + except (ValueError, TypeError) as e: + error_msg = f"Invalid depth values: StratTop={row.StratTop}, StratBottom={row.StratBottom}" + logger.critical( + f"{pointid} layer {layer_index}: {error_msg}, error: {e}" + ) + errors.append( + { + "pointid": pointid, + "layer": layer_index, + "error": error_msg, + "details": str(e), # for conversion errors + } + ) + skipped_count += 1 + continue + + # Validate depth logic + if top_depth >= bottom_depth: + error_msg = ( + f"Invalid depth logic: top={top_depth} >= bottom={bottom_depth}" + ) + logger.critical(f"{pointid} layer {layer_index}: {error_msg}") + errors.append( + {"pointid": pointid, "layer": layer_index, "error": error_msg} + ) + skipped_count += 1 + continue + + if top_depth < 0: + error_msg = f"Negative top depth: {top_depth}" + logger.critical(f"{pointid} layer {layer_index}: {error_msg}") + errors.append( + {"pointid": pointid, "layer": layer_index, "error": error_msg} + ) + skipped_count += 1 + continue + + # 7. Get or create the formation + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if not formation: + # Create new formation if it doesn't exist + logger.info(f"Creating new geologic formation: {formation_code}") + formation = GeologicFormation( + formation_code=formation_code, + description=None, + lithology=None, # Will be set below + ) + session.add(formation) + session.flush() + + # 8. Update formation lithology if available and not already set + if hasattr(row, "Lithology") and row.Lithology: + try: + # Map lithology code to geologic_formation.lithology using ABBREVIATION field + lithology = lexicon_mapper.map_value( + f"LU_Lithology:{row.Lithology}" + ) + + # Update if formation does not have lithology yet + if not formation.lithology: + formation.lithology = lithology + lithology_updates += 1 + logger.info(f"Set lithology for {formation_code}: {lithology}") + elif formation.lithology != lithology: + # Log if there's a mismatch (different lithology for same formation) + logger.warning( + f"Formation {formation_code} has conflicting lithology: " + f"existing='{formation.lithology}', new='{lithology}'." + ) + except KeyError: + logger.warning( + f"Unknown lithology code '{row.Lithology}' for {pointid}, skipping lithology update" + ) + except Exception as e: + logger.warning(f"Error mapping lithology '{row.Lithology}': {e}") + + # 9. Create ThingGeologicFormationAssociation record + try: + formation_assoc = ThingGeologicFormationAssociation( + thing=thing, + geologic_formation=formation, + top_depth=top_depth, + bottom_depth=bottom_depth, + ) + session.add(formation_assoc) + created_count += 1 + + logger.info( + f" Layer {layer_index + 1}: {formation.formation_code} " + f"from {top_depth:.1f} to {bottom_depth:.1f} ft" + ) + + except Exception as e: + logger.critical( + f"Error creating stratigraphy association for {pointid}, " + f"formation {formation_code}: {e}" + ) + errors.append( + { + "pointid": pointid, + "formation": formation_code, + "layer": layer_index, + "error": str(e), + } + ) + skipped_count += 1 + continue + + # 10. Final commit + try: + session.commit() + logger.info( + f"Successfully transferred stratigraphy: " + f"{created_count} associations created, {skipped_count} skipped, " + f"{lithology_updates} lithology fields updated, {len(errors)} errors" + ) + except Exception as e: + logger.critical(f"Error in final commit: {e}") + session.rollback() + + return input_df, cleaned_df, errors diff --git a/transfers/transfer.py b/transfers/transfer.py index 77275ed35..29d76cec7 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -31,11 +31,15 @@ from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata from transfers.contact_transfer import transfer_contacts from transfers.sensor_transfer import transfer_sensors +from transfers.aquifer_system_transfer import transfer_aquifer_systems +from transfers.geologic_formation_transfer import transfer_geologic_formations from transfers.waterlevels_transfer import transfer_water_levels from transfers.well_transfer import ( transfer_wells, transfer_wellscreens, ) +from transfers.stratigraphy_transfer import transfer_stratigraphy +from transfers.permissions_transfer import transfer_permissions from transfers.asset_transfer import transfer_assets from transfers.util import timeit, timeit_direct @@ -57,6 +61,14 @@ def transfer_all(sess, limit=100): erase_and_rebuild_db() metrics = Metrics() + + # transfer aquifer systems and geologic formations first as well_transfer depend on them + message("TRANSFERRING AQUIFER SYSTEMS") + timeit_direct(transfer_aquifer_systems, sess) + + message("TRANSFERRING GEOLOGIC FORMATIONS") + timeit_direct(transfer_geologic_formations, sess) + message("TRANSFERRING WELLS") flags = { @@ -71,6 +83,9 @@ def transfer_all(sess, limit=100): results = timeit_direct(transfer_wellscreens, sess) metrics.well_screen_metrics(sess, *results) + message("TRANSFERRING STRATIGRAPHY") + timeit_direct(transfer_stratigraphy, sess) + message("TRANSFERRING SENSORS") results = timeit_direct(transfer_sensors, sess) metrics.sensor_metrics(sess, *results) @@ -124,6 +139,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING ASSETS") timeit_direct(transfer_assets, sess) + message("TRANSFERRING PERMISSIONS") + results = timeit_direct(transfer_permissions, sess) + def transfer_debugging(sess, limit=100): message("STARTING TRANSFER DEBUG", new_line_at_top=False) diff --git a/transfers/util.py b/transfers/util.py index cbf0f2b17..3d42f0d92 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -275,7 +275,7 @@ def make_location_data_provenance( ) -> list[DataProvenance]: provenance_records = [] - if row.AltitudeAccuracy or row.CoordinateAccuracy: + if row.AltitudeAccuracy: provenance = DataProvenance( target_id=location.id, target_table="location", @@ -369,7 +369,6 @@ def make_location_data_provenance( target_id=location.id, target_table="location", field_name="point", - origin_source=None, collection_method=coordinate_method, accuracy_value=accuracy_value, accuracy_unit=accuracy_unit, @@ -409,6 +408,8 @@ def _make_lu_to_lexicon_mapper(self): # Lookup tables where CODE maps to MEANING lu_tables = [ "LU_AltitudeMethod", + "LU_AquiferClass", + "LU_AquiferType", "LU_CollectionMethod", "LU_ConstructionMethod", "LU_CoordinateAccuracy", @@ -418,7 +419,9 @@ def _make_lu_to_lexicon_mapper(self): "LU_DataSource", "LU_Depth_CompletionSource", "LU_Discharge_ChemistrySource", + "LU_Formations", "LU_LevelStatus", + "LU_Lithology", "LU_MajorAnalyte", "LU_MeasurementMethod", "LU_MinorTraceAnalyte", @@ -434,8 +437,6 @@ def _make_lu_to_lexicon_mapper(self): "LU_AltitudeDatum": "code is the value, so no need for mapping", "LU_CoordinateDatum": "code is the value, so no need for mapping", "LU_FieldNoteTypes": "not being used in the transfers since there are no records", - "LU_Formations": "needs to be cleaned before it can be used", - "LU_Lithology": "needs to be cleaned before it can be used", "LU_MeasuringAgency": "the abbreviation is what is used in the new schema", } mappers = {} @@ -447,6 +448,9 @@ def _make_lu_to_lexicon_mapper(self): if lu_table == "LU_Formations": code = row.Code meaning = row.Meaning + elif lu_table == "LU_Lithology": + code = row.ABBREVIATION + meaning = row.TERM else: code = row.CODE meaning = row.MEANING diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index ee54d0216..79fa4842f 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -16,7 +16,7 @@ import json import time from datetime import datetime, UTC - +import re import pandas as pd from pandas import isna from pydantic import ValidationError @@ -36,6 +36,11 @@ StatusHistory, MonitoringFrequencyHistory, MeasuringPointHistory, + DataProvenance, + AquiferSystem, + AquiferType, + GeologicFormation, + ThingAquiferAssociation, ) from schemas.thing import CreateWell, CreateWellScreen from services.gcs_helper import get_storage_bucket @@ -117,6 +122,129 @@ def _extract_casing_materials(row) -> list[str]: return materials +pattern = re.compile( + r"\b(?Pjet|hand|submersible)\b|\b(?Pline[-\s]+shaft)\b", re.IGNORECASE +) + + +def first_matched_term(text: str): + m = pattern.search(text) + if not m: + return None + return m.group("term") or m.group("phrase") + + +PUMP_MAPPING = {"jet": "Jet", "hand": "Hand", "submersible": "Submersible"} + + +def _extract_well_pump_type(row) -> str | None: + if isna(row.ConstructionNotes): + return None + construction_notes = row.ConstructionNotes.lower() + return PUMP_MAPPING.get(first_matched_term(construction_notes), None) + + +# Parse aquifer codes +def _extract_aquifer_type_codes(aquifer_code: str) -> list[str]: + """ + Parse aquifer type codes that may contain multiple values. + + Args: + aquifer_code: Raw code from AquiferType field + + Returns: + List of individual codes + """ + if not aquifer_code: + return [] + # clean the code + code = aquifer_code.strip().upper() + # split into individual characters. This handles cases like "FC" -> ["F", "C"] + individual_codes = list(code) + return individual_codes + + +# Get or create aquifer system +def get_or_create_aquifer_system( + session: Session, aquifer_name: str, primary_type: str +) -> AquiferSystem | None: + """ + Get existing aquifer or create new one if it doesn't exist. + + With the new AquiferType model, we create ONE aquifer record per named + aquifer (e.g., one "Santa Fe Group"), not multiple variants. + + Args: + session: Database session + aquifer_name: Name of the aquifer (from AqClass or type name) + primary_type: Primary aquifer type for the aquifer_type field + """ + # Try to find existing aquifer by name + aquifer = ( + session.query(AquiferSystem).filter(AquiferSystem.name == aquifer_name).first() + ) + + if aquifer: + return aquifer + + # Create new aquifer + try: + logger.info( + f"Creating new aquifer system: {aquifer_name} (primary type: {primary_type})" + ) + + aquifer = AquiferSystem( + name=aquifer_name, + primary_aquifer_type=primary_type, # Primary type + geographic_scale=None, # Default + ) + session.add(aquifer) + session.flush() # Get the ID + return aquifer + except Exception as e: + logger.critical(f"Error creating aquifer {aquifer_name}: {e}") + return None + + +def get_or_create_geologic_formation( + session: Session, formation_code: str +) -> GeologicFormation | None: + """ + Get existing geologic formation or create new one if it doesn't exist. + + Args: + session: Database session + formation_code: The formation code from FormationZone field + + Returns: + GeologicFormation object or None if creation fails + """ + # Try to find existing formation + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if formation: + return formation + + # If not found, create new formation + try: + logger.info(f"Creating new geologic formation: {formation_code}") + formation = GeologicFormation( + formation_code=formation_code, + description=None, + lithology=None, + ) + session.add(formation) + session.flush() + return formation + except Exception as e: + logger.critical(f"Error creating formation {formation_code}: {e}") + return None + + def get_wells_to_transfer( sess: Session, flags: dict = None ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -236,6 +364,7 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_casing_materials = ( [] if isna(row.CasingDescription) else _extract_casing_materials(row) ) + well_pump_type = _extract_well_pump_type(row) # manually add the well rather than add_well from services/thing_helper.py # so that effective_start can be set on the location assocation @@ -257,6 +386,21 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None notes=( [{"content": row.Notes, "note_type": "Other"}] if row.Notes else [] ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=( + lexicon_mapper.map_value( + f"LU_ConstructionMethod:{row.ConstructionMethod}" + ) + if not isna(row.ConstructionMethod) + else None + ), + well_pump_type=well_pump_type, + is_suitable_for_datalogger=( + bool(row.OpenWellLoggerOK) + if not isna(row.OpenWellLoggerOK) + else None + ), ) CreateWell.model_validate(data) @@ -277,6 +421,8 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None "well_casing_materials", "measuring_point_height", "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" @@ -285,17 +431,6 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None well_data.pop("notes") well = Thing(**well_data) session.add(well) - # logger.info(f"Created well for {row.PointID}") - - # flush well to access its ID for status_history - # session.flush() - - # session.commit() - # session.refresh(well) - # if notes: - # for ni in notes: - # nn = well.add_note(ni['content'], ni['note_type']) - # session.add(nn) if well_purposes: for wp in well_purposes: @@ -330,6 +465,150 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None assoc.thing = well session.add(assoc) + # --- Create Aquifer Association with AquiferType records --- + if hasattr(row, "AquiferType") and not isna(row.AquiferType): + try: + # Parse codes (handles multi-character codes like "FC") + aquifer_codes = _extract_aquifer_type_codes(row.AquiferType) + + if not aquifer_codes: + logger.warning( + f"Well {row.PointID}: Empty aquifer codes after parsing '{row.AquiferType}'" + ) + else: + # Map AqClass code to aquifer name using lexicon mapper + if hasattr(row, "AqClass") and not isna(row.AqClass): + try: + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferClass:{row.AqClass}" + ) + except KeyError: + logger.warning( + f"Unknown AqClass code '{row.AqClass}' for well {row.PointID}, using first type as name" + ) + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + else: + # No AqClass - use first code's mapped name as aquifer name + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + + # Determine primary type + # This assumes the first recorded type of a compound type is the primary type of the aquifer. + # TODO: verify with AMMP + try: + primary_type = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + except KeyError: + logger.warning( + f"Unknown aquifer type code '{aquifer_codes[0]}' for well {row.PointID}." + f"Setting primary_type to 'Unknown'" + ) + primary_type = "Unknown" # Creates aquifer with placeholder + + # Get or create the aquifer + aquifer = get_or_create_aquifer_system( + session, aquifer_name, primary_type + ) + + if aquifer: + # Check if association already exists + existing_assoc = ( + session.query(ThingAquiferAssociation) + .filter( + ThingAquiferAssociation.thing_id == well.id, + ThingAquiferAssociation.aquifer_system_id == aquifer.id, + ) + .first() + ) + + if not existing_assoc: + # Create the association + aquifer_assoc = ThingAquiferAssociation( + thing=well, aquifer_system=aquifer + ) + session.add(aquifer_assoc) + session.flush() + + # Create AquiferType records for EACH characteristic + aquifer_type_names = [] + for aquifer_code in aquifer_codes: + try: + type_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_code}" + ) + aquifer_type = AquiferType( + thing_aquifer_association=aquifer_assoc, + aquifer_type=type_name, + ) + session.add(aquifer_type) + aquifer_type_names.append(type_name) + except KeyError: + logger.critical( + f"Unknown aquifer code '{aquifer_code}' from AquiferType='{row.AquiferType}' " + f"for well {well.name}. Skipping this code." + ) + errors.append( + { + "pointid": well.name, + "table": source_table, + "field": "AquiferType", + "error": f"Unknown aquifer code: {aquifer_code}", + } + ) + + logger.info( + f"Associated well {well.name} with aquifer {aquifer.name} " + f"(types: {', '.join(aquifer_type_names)})" + ) + + except Exception as e: + logger.critical( + f"Error creating aquifer associations for {well.name}: {e}" + ) + + # --- Set Formation Completion (NOT depth-based stratigraphy) --- + # This simply records which formation the well was completed in. + # For detailed depth-interval stratigraphy, see stratigraphy_transfer.py + if hasattr(row, "FormationZone") and not isna(row.FormationZone): + try: + formation_code = row.FormationZone + + # Validate formation exists + formation = ( + session.query(GeologicFormation) + .filter(GeologicFormation.formation_code == formation_code) + .first() + ) + + if formation: + # Formation exists: Set association + well.formation_completion_code = formation_code + logger.info( + f"Set completion formation for {well.name}: {formation_code}" + ) + else: + # Formation does NOT exist: Do not create new formation. Flag and log for review + logger.critical( + f"MISSING FORMATION: Formation '{formation_code}' not found for well {well.name}. Flagged for review." + ) + errors.append( + { + "pointid": well.name, + "table": source_table, + "field": "FormationZone", + "error": f"Unknown formation: {formation_code}", + } + ) + + except Exception as e: + logger.critical( + f"Error setting completion formation for {well.name}: {e}" + ) + session.commit() # add things thate need well id @@ -347,11 +626,42 @@ def transfer_wells(session: Session, flags: dict = None, limit: int = 0) -> None for dp in data_provenances: session.add(dp) + if not isna(row.CompletionSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_completion_date", + origin_type=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.CompletionSource}" + ), + ) + session.add(dp) + + if not isna(row.DataSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_construction_method", + origin_source=row.DataSource, + ) + session.add(dp) + + if not isna(row.DepthSource): + dp = DataProvenance( + target_id=well.id, + target_table="thing", + field_name="well_depth", + origin_type=lexicon_mapper.map_value( + f"LU_Depth_CompletionSource:{row.DepthSource}" + ), + ) + session.add(dp) + """ - Developer's note + Developer's note - It's not clear when the measuring point from NM_Aquifer was - determined, so I'm setting start_date to the day of the transfer + It's not clear when the measuring point from NM_Aquifer was + determined, so I'm setting start_date to the day of the transfer """ measuring_point_history = MeasuringPointHistory( thing_id=well.id,