diff --git a/.gitignore b/.gitignore index 6be8b4d9e..471b8faad 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ launcher.sh gcs_credentials.json transfers/data/assets* transfers/transfer*.log +transfer*.log # deployment files app.yaml \ No newline at end of file diff --git a/core/lexicon.json b/core/lexicon.json index 6fb9ecb51..db05e9061 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -65,7 +65,7 @@ {"categories": ["elevation_method"], "term": "Reported", "definition": "Reported"}, {"categories": ["elevation_method"], "term": "Survey-grade Global Navigation Satellite Sys, Lvl1", "definition": "Survey-grade Global Navigation Satellite Sys, Lvl1"}, {"categories": ["elevation_method"], "term": "USGS National Elevation Dataset (NED)", "definition": "USGS National Elevation Dataset (NED)"}, - {"categories": ["elevation_method", "sample_method", "coordinate_method", "current_use", "status", "organization"], "term": "Unknown", "definition": "Unknown"}, + {"categories": ["elevation_method", "sample_method", "coordinate_method", "current_use", "status", "organization", "role"], "term": "Unknown", "definition": "Unknown"}, {"categories": ["construction_method"], "term": "Air-rotary", "definition": "Air-rotary"}, {"categories": ["construction_method"], "term": "Bored or augered", "definition": "Bored or augered"}, {"categories": ["construction_method"], "term": "Cable-tool", "definition": "Cable-tool"}, @@ -348,10 +348,11 @@ {"categories": ["organization"], "term": "NMED", "definition": "New Mexico Environment Department"}, {"categories": ["organization"], "term": "NMOSE", "definition": "New Mexico Office of the State Engineer"}, {"categories": ["organization"], "term": "NMBGMR", "definition": "New Mexico Bureau of Geology and Mineral Resources"}, - {"categories": ["organization"], "term": "Bernalillo Cty", "definition": "Bernalillo County"}, + {"categories": ["organization"], "term": "Bernalillo County", "definition": "Bernalillo County"}, {"categories": ["organization"], "term": "BLM", "definition": "Bureau of Land Management"}, {"categories": ["organization"], "term": "SFC", "definition": "Santa Fe County"}, - {"categories": ["organization"], "term": "NESWCD", "definition": "Northeast Soil & Water Conservation District"}, + {"categories": ["organization"], "term": "CSF", "definition": "City of Santa Fe"}, + {"categories": ["organization"], "term": "NESWCD", "definition": "Northeastern Soil & Water Conservation District"}, {"categories": ["organization"], "term": "NMISC", "definition": "New Mexico Interstate Stream Commission"}, {"categories": ["organization"], "term": "PVACD", "definition": "Pecos Valley Artesian Conservancy District"}, {"categories": ["organization"], "term": "TSWCD", "definition": "Taos Soil & Water Conservation District"}, @@ -361,6 +362,41 @@ {"categories": ["organization"], "term": "USFS", "definition": "United States Forest Service"}, {"categories": ["organization"], "term": "NMT", "definition": "New Mexico Tech"}, {"categories": ["organization"], "term": "NPS", "definition": "National Park Service"}, + {"categories": ["organization"], "term": "NMRWA", "definition": "New Mexico Rural Water Association"}, + {"categories": ["organization"], "term": "Taos SWCD", "definition": "Taos Soil and Water Conservation District"}, + {"categories": ["organization"], "term": "Otero SWCD", "definition": "Otero Soil and Water Conservation District"}, + {"categories": ["organization"], "term": "Northeastern SWCD", "definition": "Northeastern Soil and Water Conservation District"}, + {"categories": ["organization"], "term": "A&T Pump & Well Service, LLC", "definition": "A&T Pump & Well Service, LLC"}, + {"categories": ["organization"], "term": "A. G. Wassenaar, Inc", "definition": "A. G. Wassenaar, Inc"}, + {"categories": ["organization"], "term": "AMEC", "definition": "AMEC"}, + {"categories": ["organization"], "term": "Balleau Groundwater, Inc", "definition": "Balleau Groundwater, Inc"}, + {"categories": ["organization"], "term": "CDM Smith", "definition": "CDM Smith"}, + {"categories": ["organization"], "term": "CH2M Hill", "definition": "CH2M Hill"}, + {"categories": ["organization"], "term": "Corbin Consulting, Inc", "definition": "Corbin Consulting, Inc"}, + {"categories": ["organization"], "term": "Chevron", "definition": "Chevron"}, + {"categories": ["organization"], "term": "Daniel B. Stephens & Associates, Inc", "definition": "Daniel B. Stephens & Associates, Inc"}, + {"categories": ["organization"], "term": "EnecoTech", "definition": "EnecoTech"}, + {"categories": ["organization"], "term": "Faith Engineering, Inc", "definition": "Faith Engineering, Inc"}, + {"categories": ["organization"], "term": "Foster Well Service, Inc", "definition": "Foster Well Service, Inc"}, + {"categories": ["organization"], "term": "Glorieta Geoscience, Inc", "definition": "Glorieta Geoscience, Inc"}, + {"categories": ["organization"], "term": "Golder Associates, Inc", "definition": "Golder Associates, Inc"}, + {"categories": ["organization"], "term": "Hathorn's Well Service, Inc", "definition": "Hathorn's Well Service, Inc"}, + {"categories": ["organization"], "term": "Hydroscience Associates, Inc", "definition": "Hydroscience Associates, Inc"}, + {"categories": ["organization"], "term": "IC Tech, Inc", "definition": "IC Tech, Inc"}, + {"categories": ["organization"], "term": "John Shomaker & Associates, Inc", "definition": "John Shomaker & Associates, Inc"}, + {"categories": ["organization"], "term": "Kuckleman Pump Service", "definition": "Kuckleman Pump Service"}, + {"categories": ["organization"], "term": "Los Golondrinas", "definition": "Los Golondrinas"}, + {"categories": ["organization"], "term": "Minton Engineers", "definition": "Minton Engineers"}, + {"categories": ["organization"], "term": "MJDarrconsult, Inc", "definition": "MJDarrconsult, Inc"}, + {"categories": ["organization"], "term": "Puerta del Canon Ranch", "definition": "Puerta del Canon Ranch"}, + {"categories": ["organization"], "term": "Rodgers & Company, Inc", "definition": "Rodgers & Company, Inc"}, + {"categories": ["organization"], "term": "San Pedro Creek Estates HOA", "definition": "San Pedro Creek Estates HOA"}, + {"categories": ["organization"], "term": "Statewide Drilling, Inc", "definition": "Statewide Drilling, Inc"}, + {"categories": ["organization"], "term": "Tec Drilling Limited", "definition": "Tec Drilling Limited"}, + {"categories": ["organization"], "term": "Tetra Tech, Inc", "definition": "Tetra Tech, Inc"}, + {"categories": ["organization"], "term": "Thompson Drilling, Inc", "definition": "Thompson Drilling, Inc"}, + {"categories": ["organization"], "term": "Witcher & Associates", "definition": "Witcher & Associates"}, + {"categories": ["organization"], "term": "Zeigler Geologic Consulting, LLC", "definition": "Zeigler Geologic Consulting, LLC"}, {"categories": ["collection_method"], "term": "manual", "definition": "manual sampling"}, {"categories": ["collection_method"], "term": "continuous", "definition": "continuous sampling"}, {"categories": ["role"], "term": "Owner", "definition": "Owner"}, @@ -371,9 +407,18 @@ {"categories": ["role"], "term": "Hydrologist", "definition": "Hydrologist"}, {"categories": ["role"], "term": "Hydrogeologist", "definition": "Hydrogeologist"}, {"categories": ["role"], "term": "Engineer", "definition": "Engineer"}, + {"categories": ["role"], "term": "Organization", "definition": "A contact that is an organization"}, + {"categories": ["role"], "term": "Specialist", "definition": "Specialist"}, {"categories": ["role"], "term": "Technician", "definition": "Technician"}, + {"categories": ["role"], "term": "Research Assistant", "definition": "Research Assistant"}, + {"categories": ["role"], "term": "Research Scientist", "definition": "Research Scientist"}, + {"categories": ["role"], "term": "Operator", "definition": "Operator"}, + {"categories": ["role"], "term": "Biologist", "definition": "Biologist"}, + {"categories": ["role"], "term": "Lab Manager", "definition": "Lab Manager"}, + {"categories": ["role"], "term": "Publications Manager", "definition": "Publications Manager"}, {"categories": ["email_type", "phone_type", "address_type", "contact_type"], "term": "Primary", "definition": "primary"}, {"categories": ["contact_type"], "term": "Secondary", "definition": "secondary"}, + {"categories": ["contact_type"], "term": "Field Event Participant", "definition": "A contact who has participated in a field event"}, {"categories": ["email_type", "phone_type", "address_type"], "term": "Work", "definition": "work"}, {"categories": ["email_type", "address_type"], "term": "Personal", "definition": "personal"}, {"categories": ["address_type"], "term": "Mailing", "definition": "mailing"}, diff --git a/db/contact.py b/db/contact.py index 39c0f33b5..652361bf5 100644 --- a/db/contact.py +++ b/db/contact.py @@ -39,7 +39,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): organization: Mapped[str] = lexicon_term(nullable=True) role: Mapped[str] = lexicon_term(nullable=False) contact_type: Mapped[str] = lexicon_term(nullable=False) + + # primary keys of the nm aquifer tables from which the contacts originate nma_pk_owners: Mapped[str] = mapped_column(String(100), nullable=True) + nma_pk_waterlevels: Mapped[str] = mapped_column(String(100), nullable=True) phones: Mapped[List["Phone"]] = relationship( "Phone", back_populates="contact", passive_deletes=True diff --git a/db/sample.py b/db/sample.py index c06a39523..fa1c9a437 100644 --- a/db/sample.py +++ b/db/sample.py @@ -84,6 +84,12 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): ) notes: Mapped[str] = mapped_column(nullable=True) + # --- Auditing Fields from NM_Aquifer --- + nma_pk_waterlevels: Mapped[str] = mapped_column( + nullable=True, + comment="NM_Aquifer primary key for waterlevels - to be used for transfer audits", + ) + # --- Relationship Definitions --- field_activity: Mapped["FieldActivity"] = relationship(back_populates="samples") field_event_contact: Mapped["FieldEventContactAssociation"] = relationship( diff --git a/services/lexicon_mapper.py b/services/lexicon_mapper.py deleted file mode 100644 index 56f684ea9..000000000 --- a/services/lexicon_mapper.py +++ /dev/null @@ -1,77 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from transfers.util import read_csv - - -class LexiconMapper: - def __init__(self): - self._mappers = None - - def map_value(self, value): - return self._make_lu_to_lexicon_mapper().get(value, value) - - def _make_lu_to_lexicon_mapper(self): - if self._mappers: - return self._mappers - - lu_tables = [ - # "LU_AltitudeDatum", # the code is the value, so no need for mapping - "LU_AltitudeMethod", # CODE/MEANING - "LU_CollectionMethod", # CODE/MEANING - "LU_ConstructionMethod", # CODE/MEANING - "LU_CoordinateAccuracy", # CODE/MEANING - # "LU_CoordinateDatum", # the code is the value, so no need for mapping - "LU_CoordinateMethod", # CODE/MEANING - "LU_CurrentUse", # CODE/MEANING - "LU_DataQuality", # CODE/MEANING - "LU_DataSource", # CODE/MEANING - "LU_Depth_CompletionSource", # CODE/MEANING - "LU_Discharge_ChemistrySource", # CODE/MEANING - # "LU_FieldNoteTypes", # not being used in the transfers since there are no records - # "LU_Formations", # needs to be cleaned before it can be used - "LU_LevelStatus", # CODE/MEANING - # "LU_Lithology", # needs to be cleaned before it can be used - "LU_MajorAnalyte", # CODE/MEANING - "LU_MeasurementMethod", # CODE/MEANING - # "LU_MeasuringAgency", # the abreviation is what is used in the new schema - "LU_MinorTraceAnalyte", # CODE/MEANING - "LU_MonitoringStatus", # CODE/MEANING - "LU_SampleType", # CODE/MEANING - "LU_SiteType", # CODE/MEANING - "LU_Status", # CODE/MEANING - ] - - mappers = {} - - for lu_table in lu_tables: - table = read_csv(lu_table) - - for i, row in table.iterrows(): - if lu_table == "LU_Formations": - code = row.Code - meaning = row.Meaning - else: - code = row.CODE - meaning = row.MEANING - - mappers.update({f"{lu_table}:{code}": meaning}) - self._mappers = mappers - return mappers - - -lexicon_mapper = LexiconMapper() - -# ============= EOF ============================================= diff --git a/tests/test_contact.py b/tests/test_contact.py index 67e81ea43..5959e48a1 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -786,7 +786,7 @@ def test_patch_contact_409_bad_contact_type(third_contact): assert data["detail"][0]["loc"] == ["body", "contact_type"] assert ( data["detail"][0]["msg"] - == "Invalid contact_type. Valid terms are: Primary | Secondary" + == "Invalid contact_type. Valid terms are: Primary | Secondary | Field Event Participant" ) assert data["detail"][0]["type"] == "value_error" assert data["detail"][0]["input"] == {"contact_type": payload["contact_type"]} diff --git a/transfers/data/measured_by_mapper.json b/transfers/data/measured_by_mapper.json new file mode 100644 index 000000000..2c3e113c6 --- /dev/null +++ b/transfers/data/measured_by_mapper.json @@ -0,0 +1,281 @@ +{ + "A&T Pump & Well Serv": [null, "A&T Pump & Well Service, LLC", "Organization"], + "AGW": [null, "A. G. Wassenaar, Inc", "Organization"], + "AGW Consultants": [null, "A. G. Wassenaar, Inc", "Organization"], + "AGW Consultants, Inc": [null, "A. G. Wassenaar, Inc", "Organization"], + "AGWC": [null, "A. G. Wassenaar, Inc", "Organization"], + "Turner, AGW": [null, "A. G. Wassenaar, Inc", "Unknown"], + "AMEC": [null, "AMEC Earth & Environmental", "Organization"], + "ARCADIS": [null, "ARCADIS", "Organization"], + "Arcadis": [null, "ARCADIS", "Organization"], + "Balleau": [null, "Balleau Groundwater, Inc", "Organization"], + "Balleau BW": [null, "Balleau Groundwater, Inc", "Organization"], + "Balleau GW": [null, "Balleau Groundwater, Inc", "Organization"], + "Cook": ["Cook", "Balleau Groundwater, Inc", "Unknown"], + "Bureau": [null, "NMBGMR", "Organization"], + "CDM": [null, "CDM Smith", "Organization"], + "CH2M Hill": [null, "CH2M Hill", "Organization"], + "IC Tech": [null, "IC Tech, Inc", "Organization"], + "IC Tech for OSE": [null, "IC Tech, Inc", "Organization"], + "IC Tech, Inc.": [null, "IC Tech, Inc", "Organization"], + "ICTech for OSE": [null, "IC Tech, Inc", "Organization"], + "ICTech-OSE": [null, "IC Tech, Inc", "Organization"], + "Jerome Salazar": ["Jerome Salazar", "Chervon", "Unknown"], + "Chevron personnel": [null, "Chevron", "Organization"], + "Calvert": ["Calvert", "Daniel B. Stephens & Associates, Inc.", "Unknown"], + "EnecoTech": [null, "EnecoTech", "Organization"], + "Faith Engineering": [null, "Faith Engineering", "Organization"], + "Hodgins, GCI": ["Meghan Hodgins", "Glorieta Geoscience, Inc", "Geologist"], + "Kreamer, GGI": ["Kreamer", "Glorieta Geoscience, Inc", "Unknown"], + "Olson, GGI": ["Olson", "Glorieta Geoscience, Inc", "Unknown"], + "Golder Ass. For OSE": [null, "Golder Associates, Inc.", "Organization"], + "Hathorn Well Service": [null, "Hathorn's Well Service"], + "Hydroscience Assoc.": [null, "Hydroscience Associates, Inc.", "Organization"], + "Lee Foster": [null, "Foster Well Service, Inc", "Organization"], + "Mario Gonzales NMRWA": ["Mario Gonzales", "NMRWA", "Unknown"], + "Minton": [null, "Minton Engineers", "Organization"], + "Minton Engineers": [null, "Minton Engineers", "Organization"], + "Minton.": [null, "Minton Engineers", "Organization"], + "MJ Darr.": [null, "MJDarrconsult, Inc", "Organization"], + "MJ Darr consultants": [null, "MJDarrconsult, Inc", "Organization"], + "NESWCD": [null, "North East Soil and Water Conservation District", "Organization"], + "OSE, ST": [[null, "NMOSE", "Organization"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "PVACD person": [null, "PVACD", "Organization"], + "SFC/Frost": ["Frost", "SFC", "Unknown"], + "SPCE HOA": [null, "San Pedro Creek Estates HOA", "Organization"], + "Statewide Drilling": [null, "Statewide Drilling", "Organization"], + "Tec Drilling": [null, "Tec Drilling Limited", "Organization"], + "TetraTech": [null, "Tetra Tech, Inc", "Organization"], + "Thompson Drilling": [null, "Thompson Drilling, Inc", "Organization"], + "USGA": [null, "USGS", "Organization"], + "USGS/NESWCD": [[null, "USGS", "Organization"], [null, "Northeastern SWCD", "Organization"]], + "NMBGMR": [null, "NMBGMR", "Organization"], + "NMED": [null, "NMED", "Organization"], + "NMOSE": [null, "NMOSE", "Organization"], + "NPS": [null, "NPS", "Organization"], + "Otero SWCD": [null, "Otero SWCD", "Organization"], + "SFC": [null, "SFC", "Organization"], + "Taos SWCD": [null, "Taos SWCD", "Organization"], + "TWDB": [null, "TWDB", "Organization"], + "USFS": [null, "USFS", "Organization"], + "USGS": [null, "USGS", "Organization"], + "USGS WRD": [null, "USGS WRD", "Organization"], + + "GGI": [null, "Glorieta Geoscience, Inc", "Organization"], + "GGI for OSE": [null, "Glorieta Geoscience, Inc", "Organization"], + "GGI-OSE": [null, "Glorieta Geoscience, Inc", "Organization"], + "Glorieta Geoscienc": [null, "Glorieta Geoscience, Inc", "Organization"], + "Glorieta Geoscience": [null, "Glorieta Geoscience, Inc", "Organization"], + + "Driller": [null, "*MEASURING_AGENCY*", "Driller"], + "Tribble, Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "Tribble/Cruz": [["Tribble", "Daniel B. Stephens & Associates, Inc", "Unknown"], ["Cruz", "Daniel B. Stephens & Associates", "Unknown"]], + "DBSA": [null, "Daniel B. Stephens & Associates, Inc", "Organization"], + "DBStephens & Assoc": [null, "Daniel B. Stephens & Associates, Inc", "Organization"], + + "City of Santa Fe": [null, "CSF", "Organization"], + "City of Santa Fe": [null, "CSF", "Organization"], + "City of Santa Fe": [null, "CSF", "Organization"], + "CityofSantaFe": [null, "CSF", "Organization"], + + "John Shomaker": [null, "John Shomaker & Associates, Inc", "Organization"], + "John Shomaker & Associates, Inc": [null, "John Shomaker & Associates, Inc", "Organization"], + "John Shomaker Assoc.": [null, "John Shomaker & Associates, Inc", "Organization"], + "JS&A": [null, "John Shomaker & Associates, Inc", "Organization"], + "JSA": [null, "John Shomaker & Associates, Inc", "Organization"], + "JSAI": [null, "John Shomaker & Associates, Inc", "Organization"], + "Shomaker": [null, "John Shomaker & Associates, Inc", "Organization"], + + "Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + "Fleming - Shomaker": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + "Fleming/Shomaker": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + "Shomaker - Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + "Shomaker - Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + "Shomaker/Fleming": ["Fleming", "John Shomaker & Associates, Inc", "Unknown"], + + "Kuck": [null, "Kuckleman Pump Service", "Organization"], + "Kuckleman": [null, "Kuckleman Pump Service", "Organization"], + + "NMOSE?": [null, "NMOSE", "Organization"], + "OSE": [null, "NMOSE", "Organization"], + + "OSE; Doug Rappuhn": ["Doug Rappuhn", "NMOSE", "Hydrologist"], + "D.Rappuhn OSE": ["Doug Rappuhn", "NMOSE", "Hydrologist"], + + "Rodgers & Co": [null, "Rodgers & Company, Inc", "Organization"], + "Rodgers & Co.": [null, "Rodgers & Company, Inc", "Organization"], + + "Sandia National labs": [null, "SNL", "Organization"], + "SNL": [null, "SNL", "Organization"], + + "Santa Fe County": [null, "SFC", "Organization"], + "SFCounty LF staff": [null, "SFC", "Organization"], + + + "AL": ["Angela Lucero", "NMBGMR", "Hydrologist"], + "AL, GR": [["Angela Lucero", "NMBGMR", "Hydrologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "AL, SC": [["Angela Lucero", "NMBGMR", "Hydrologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "Amy Kronson": [["Amy Kronson", "NMBGMR", "Technician"]], + "Andrew Matejunas": [["Andrew Matejunas", "NMBGMR", "Research Assistant"]], + "Andy Manning": [["Andy Manning", "USGS", "Hydrogeologist"]], + "ce": ["Cathy Eisen", "NMBGMR", "Hydrogeologist"], + "CE": ["Cathy Eisen", "NMBGMR", "Hydrogeologist"], + "CE PJ": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], + "CE, GR": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "CM": ["Cris Morton", "NMBGMR", "Hydrogeologist"], + "CM, EM": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + "CM, LS, KP": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], + "CM, LS, KrPe": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Kirsten Pearthree", "NMBGMR", "Research Scientist"]], + "CM, SC": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "CM, TK": [["Cris Morton", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "Dan McGregor": ["Dan McGregor", "Bernalillo County", "Hydrogeologist"], + "CE TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "CE, TK": [["Cathy Eisen", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "EM": ["Ethan Mamer", "NMBGMR", "Hydrogeologist"], + "EM, AL": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"],["Angela Lucero", "NMBGMR", "Hydrologist"]], + "EM, CM": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "EM,CM": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "EM, JB": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Joseph Beman", "NMBGMR", "Technician"]], + "EM, KP": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], + "EM, LS": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Laila Sturgis", "NMBGMR", "Hydrogeologist"]], + "EM, MF": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Marissa Fichera", "NMBGMR", "Hydrogeologist"]], + "EM, SMC": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]], + "EM, TK": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "EM, TN": [["Ethan Mamer", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "Gary Goss": ["Gary Goss", "*MEASURING_AGENCY*", "Hydrogeologist"], + "Anders Lundahl": ["Anders Lundahl", "*MEASURING_AGENCY*", "Specialist"], + "Anders Lundalh": ["Anders Lundahl", "*MEASURING_AGENCY*", "Specialist"], + "Bob Borton": ["Bob Borton", "NMBGMR", "Geologist"], + "Borton": ["Bob Borton", "NMBGMR", "Geologist"], + "Dennis Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], + "Dennis R. Cooper": ["Dennis Cooper", "NMOSE", "Engineer"], + "GCR": ["Geoff Rawling", "NMBGMR", "Hydrogeologist"], + "GCR/ST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "GCRST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "GR/ST": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "Rawling/Wagner": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "GCR/ST/JM": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Joe Marcoline", "NMBGMR", "Unknown"]], + "GR": ["Geoff Rawling", "NMBGMR", "Hydrogeologist"], + "GR, AL": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], + "GR, CE": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "GR, SC": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "GR, TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "GR/TK": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "GR/LL": [["Geoff Rawling", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], + "JB": ["Joseph Beman", "NMBGMR", "Technician"], + "JEB": ["Joseph Beman", "NMBGMR", "Technician"], + "Jim Corbin": ["Jim Corbin", "Corbin Consulting, Inc", "Unknown"], + "JM": ["Joe Marcoline", "NMED", "Unknown"], + "Joe Marcoline": ["Joe Marcoline", "NMED", "Unknown"], + "Johnson": ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], + "Johnson - Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], + "Johnson-Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], + "Johnson/Kuck": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Kuck", "Unknown", "Unknown"]], + "Johnson-Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], + "Johnson/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], + "Lyman": [["John Lyman", "Unknown", "Unknown"]], + "PJ/Lyman": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["John Lyman", "Unknown", "Unknown"]], + "Jose Varela Lopez": ["Jose Varela Lopez", "Puerta del Canon Ranch", "Operator"], + "K. McLain": ["Katie McLain", "NMBGMR", "Hydrogeologist"], + "K. McLain, M. Hein": [["Katie McLain", "NMBGMR", "Hydrogeologist"], ["Marina Hein", "NMT", "Biologist"]], + "K.Summers": ["Kelly Summers", "NMBGMR", "Hydrologist"], + "WK Summers": ["Kelly Summers", "NMBGMR", "Hydrologist"], + "Kelsey McNamara": ["Kelsey McNamara", "NMBGMR", "Geologist"], + "Kitty": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], + "Kitty Pokorny": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], + "KP": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], + "KP, MF": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Marissa Fichera", "NMBGMR", "Hydrogeologist"]], + "KP, ST": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "KP, TK": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "KR": ["Kylian Robinson", "NMED", "Hydrogeologist"], + "Kylian Robinson": ["Kylian Robinson", "NMED", "Hydrogeologist"], + "Leroy Romero": ["Leroy Romero", "Los Golondrinas", "Unknown"], + "LL, TN": [["Lewis Land", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "LS": ["Laila Sturgis", "NMBGMR", "Hydrogeologist"], + "LS, TK": [["Laila Sturgis", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "M. Hein": ["Marina Hein", "NMT", "Biologist"], + "MH, KM": [["Marina Hein", "NMT", "Biologist"], ["Katie McLain", "NMBGMR", "Hydrogeologist"]], + "Patricia Rosacker": ["Patricia Rosacker", "CSF", "Lab Manager"], + "PB, PJ": [["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], + "PB, PJ, TK": [["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "Pepin": ["Jeff Pepin", "USGS", "Hydrologist"], + "Pepin/Kelley": [["Jeff Pepin", "USGS", "Hydrologist"], ["Shari Kelley", "NMBGMR", "Geologist"]], + "Mark Person": ["Mark Person", "NMT", "Hydrologist"], + "PJ": ["Peggy Johnson", "NMBGMR", "Hydrogeologist"], + "PJ PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], + "PJ, PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], + "PJ TK PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], + "PJ, TK, PB": [["Peggy Johnson", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"]], + "RL Borton": [["R. L. Borton", "NMOSE", "Unknown"]], + "RP": ["RP", "NMOSE", "Unknown"], + "Sara Chudnoff": ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], + "SMC": ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], + "SMC, EM": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + "SMC, SC": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Hydrogeologist"]], + "SMC, TK": [["Sara Chudnoff", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "SC": ["Scott Christenson", "NMBGMR", "Technician"], + "SCC": ["Scott Christenson", "NMBGMR", "Technician"], + "SD": ["Scott Christenson", "NMBGMR", "Technician"], + "SC, AL": [["Scott Christenson", "NMBGMR", "Technician"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], + "SC, CM": [["Scott Christenson", "NMBGMR", "Technician"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "SC, DL": [["Scott Christenson", "NMBGMR", "Technician"], ["Dan Lavery", "NMBGMR", "Hydrogeologist"]], + "SC, EM": [["Scott Christenson", "NMBGMR", "Technician"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + "SC, GR": [["Scott Christenson", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "SC, KP": [["Scott Christenson", "NMBGMR", "Technician"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], + "SC, SMC": [["Scott Christenson", "NMBGMR", "Technician"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]], + "SC, ST": [["Scott Christenson", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]], + "SC, TK": [["Scott Christenson", "NMBGMR", "Technician"], ["Trevor Kludt", "NMBGMR", "Hydrogeologist"]], + "SC, TN": [["Scott Christenson", "NMBGMR", "Technician"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "SK": ["Shari Kelley", "NMBGMR", "Geologist"], + "SK, SC, GR": [["Shari Kelley", "NMBGMR", "Geologist"], ["Scott Christenson", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "SR": ["Stephanie Roussel", "USGS", "Hydrologist"], + "Stephanie Roussel": ["Stephanie Roussel", "USGS", "Hydrologist"], + "SR, EM": [["Stephanie Roussel", "USGS", "Hydrologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + " Wagner": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], + "ST": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], + "Stacy Timmons": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], + "Timmons": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], + "Wagner": ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], + "ST, CE": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "ST, Joe Marcoline": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Joe Marcoline", "NMED", "Unknown"]], + "ST/JM": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Joe Marcoline", "NMED", "Unknown"]], + "ST, KP": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]], + "ST, SK": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Shari Kelley", "NMBGMR", "Geologist"]], + "ST, SK, Person": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Shari Kelley", "NMBGMR", "Geologist"], ["Mark Person", "NMT", "Hydrologist"]], + "ST, SMC": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]], + "ST/BF": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "ST/BFK": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "ST/BTN": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "ST/GCR": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "ST/GR": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "Wagner/Rawling": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "ST/JW": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Jim Witcher", "Witcher & Associates", "Hydrogeologist"]], + "ST/LL": [["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], + "TK": ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], + "Trevor Kludt": ["Trevor Kludt", "NMBGMR", "Hydrogeologist"], + "TK BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "TK, BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "TK/BF": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Brigitte Felix", "NMBGMR", "Publications Manager"]], + "tk cm": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "TK, CM": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cris Morton", "NMBGMR", "Hydrogeologist"]], + "TK KR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], + "TK, KR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kylian Robinson", "NMED", "Hydrogeologist"]], + "TK, AL": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Angela Lucero", "NMBGMR", "Hydrologist"]], + "TK, CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK,CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, EM": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Ethan Mamer", "NMBGMR", "Hydrogeologist"]], + "TK, GR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK, GCR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK/GR": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK/RG": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]], + "TK, KrPe": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Kirsten Pearthree", "NMBGMR", "Research Scientist"]], + "TK, PB, PJ": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Paul Bauer", "NMBGMR", "Geologist"], ["Peggy Johnson", "NMBGMR", "Hydrogeologist"]], + "TK, SC": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Scott Christenson", "NMBGMR", "Technician"]], + "TK, ST, CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, ST; CE": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]], + "TK, TN": [["Trevor Kludt", "NMBGMR", "Hydrogeologist"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]], + "TN": ["Talon Newton", "NMBGMR", "Hydrogeologist"], + "TN, LL": [["Talon Newton", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]], + "Wasiolek": ["Maryann Wasiolek", "Hydroscience Associates, Inc", "Hydrogeologist"], + "Wasiolek rpt 1983": ["Maryann Wasiolek", "Hydroscience Associates, Inc", "Hydrogeologist"] +} \ No newline at end of file diff --git a/transfers/transfer.py b/transfers/transfer.py index 4675969d9..56977e9bd 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -14,7 +14,6 @@ # limitations under the License. # =============================================================================== import os -import time from dotenv import load_dotenv @@ -25,19 +24,13 @@ from db import Base from db.engine import session_ctx -from transfers.asset_transfer import transfer_assets from transfers.group_transfer import transfer_groups from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata from transfers.contact_transfer import transfer_contacts from transfers.sensor_transfer import init_sensor from transfers.waterlevels_transfer import transfer_water_levels from transfers.well_transfer import transfer_wells, transfer_wellscreens -from transfers.thing_transfer import ( - transfer_springs, - transfer_perennial_stream, - transfer_ephemeral_stream, - transfer_met, -) + from transfers.util import timeit, timeit_direct from transfers.logger import logger, save_log_to_bucket @@ -106,9 +99,6 @@ def transfer_all(sess, limit=100): message("TRANSFERRING CONTACTS") timeit_direct(transfer_contacts, sess) - message("TRANSFERRING WATER LEVELS") - timeit_direct(transfer_water_levels, sess) - """ Developer's notes @@ -126,6 +116,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING GROUPS") timeit_direct(transfer_groups, sess) + message("TRANSFERRING WATER LEVELS") + timeit_direct(transfer_water_levels, sess) + # message("TRANSFERRING ASSETS") # timeit_direct(transfer_assets, sess) diff --git a/transfers/util.py b/transfers/util.py index d7770a807..fd0cc7cd8 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -29,13 +29,14 @@ from constants import SRID_WGS84, SRID_UTM_ZONE_13N from db import Thing, Location from services.gcs_helper import get_storage_bucket -from services.lexicon_mapper import lexicon_mapper + +# from services.lexicon_mapper import lexicon_mapper from services.util import ( transform_srid, get_epqs_elevation_from_point, - get_state_from_point, - get_county_from_point, - get_quad_name_from_point, + # get_state_from_point, + # get_county_from_point, + # get_quad_name_from_point, ) from transfers.logger import logger @@ -92,7 +93,9 @@ def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: reader = csv.reader(f) _ = next(reader) valid_datasources = [row[0] for row in reader if row[1] == "Yes"] - logger.info(f"Valid WellData Datasources: {valid_datasources}") + logger.info("Valid WellData Datasources:") + logger.info("\n".join(f" {vd}" for vd in valid_datasources)) + logger.info(f" {vd}") return df[df["DataSource"].isin(valid_datasources)] @@ -106,7 +109,9 @@ def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: reader = csv.reader(f) _ = next(reader) valid_measuring_agencies = [row[0] for row in reader if row[1] == "Yes"] - logger.info(f"Valid Measuring Agencies: {valid_measuring_agencies}") + logger.info("Valid Measuring Agencies:") + for vma in valid_measuring_agencies: + logger.info(f" {vma}") return df[df["MeasuringAgency"].isin(valid_measuring_agencies)] @@ -312,4 +317,94 @@ def wrapper(*args, **kwargs): return wrapper +class LexiconMapper: + def __init__(self): + self._mappers = None + + def map_value(self, value): + return self._make_lu_to_lexicon_mapper().get(value, value) + + def _make_lu_to_lexicon_mapper(self): + if self._mappers: + return self._mappers + + # Lookup tables where CODE maps to MEANING + lu_tables = [ + "LU_AltitudeMethod", + "LU_CollectionMethod", + "LU_ConstructionMethod", + "LU_CoordinateAccuracy", + "LU_CoordinateMethod", + "LU_CurrentUse", + "LU_DataQuality", + "LU_DataSource", + "LU_Depth_CompletionSource", + "LU_Discharge_ChemistrySource", + "LU_LevelStatus", + "LU_MajorAnalyte", + "LU_MeasurementMethod", + "LU_MinorTraceAnalyte", + "LU_MonitoringStatus", + "LU_SampleType", + "LU_SiteType", + "LU_Status", + ] + + # Lookup tables intentionally skipped (kept for documentation only) + # Each entry explains why the table is excluded + _lu_tables_skipped = { + "LU_AltitudeDatum": "code is the value, so no need for mapping", + "LU_CoordinateDatum": "code is the value, so no need for mapping", + "LU_FieldNoteTypes": "not being used in the transfers since there are no records", + "LU_Formations": "needs to be cleaned before it can be used", + "LU_Lithology": "needs to be cleaned before it can be used", + "LU_MeasuringAgency": "the abbreviation is what is used in the new schema", + } + # "LU_AltitudeDatum", # the code is the value, so no need for mapping + "LU_AltitudeMethod", # CODE/MEANING + "LU_CollectionMethod", # CODE/MEANING + "LU_ConstructionMethod", # CODE/MEANING + "LU_CoordinateAccuracy", # CODE/MEANING + # "LU_CoordinateDatum", # the code is the value, so no need for mapping + "LU_CoordinateMethod", # CODE/MEANING + "LU_CurrentUse", # CODE/MEANING + "LU_DataQuality", # CODE/MEANING + "LU_DataSource", # CODE/MEANING + "LU_Depth_CompletionSource", # CODE/MEANING + "LU_Discharge_ChemistrySource", # CODE/MEANING + # "LU_FieldNoteTypes", # not being used in the transfers since there are no records + # "LU_Formations", # needs to be cleaned before it can be used + "LU_LevelStatus", # CODE/MEANING + # "LU_Lithology", # needs to be cleaned before it can be used + "LU_MajorAnalyte", # CODE/MEANING + "LU_MeasurementMethod", # CODE/MEANING + # "LU_MeasuringAgency", # the abreviation is what is used in the new schema + "LU_MinorTraceAnalyte", # CODE/MEANING + "LU_MonitoringStatus", # CODE/MEANING + "LU_SampleType", # CODE/MEANING + "LU_SiteType", # CODE/MEANING + "LU_Status", # CODE/MEANING + ] + + mappers = {} + + for lu_table in lu_tables: + table = read_csv(lu_table) + + for i, row in table.iterrows(): + if lu_table == "LU_Formations": + code = row.Code + meaning = row.Meaning + else: + code = row.CODE + meaning = row.MEANING + + mappers.update({f"{lu_table}:{code}": meaning}) + self._mappers = mappers + return mappers + + +lexicon_mapper = LexiconMapper() + + # ============= EOF ============================================= diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 0772b5911..913313119 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -16,19 +16,38 @@ import time import uuid from datetime import datetime +import json import pandas as pd -from db import Thing, Sample, Observation, FieldEvent, FieldActivity -from services.lexicon_mapper import lexicon_mapper +from db import ( + Thing, + Sample, + Observation, + FieldEvent, + FieldActivity, + FieldEventContactAssociation, + Contact, +) from transfers.util import ( filter_to_valid_point_ids, logger, read_csv, convert_mt_to_utc, filter_by_valid_measuring_agency, + lexicon_mapper, ) +# constants +SPACE_2 = " " * 2 +SPACE_4 = " " * 4 + +# keep a dictionary of created Contacts to avoid repeated SQL queries +CREATED_CONTACTS = {} + +with open("transfers/data/measured_by_mapper.json", "r") as f: + measured_by_mapper = json.load(f) + def transfer_water_levels(session): @@ -56,11 +75,9 @@ def transfer_water_levels(session): ) session.commit() - if pd.isna(row.DepthToWater) or pd.isna(row.DateMeasured): + if pd.isna(row.DateMeasured): logger.critical( - f"transfer_water_levels. Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to " - f"missing " - f"data." + f"transfer_water_levels. Skipping row PointID={row.PointID}, objectid={row.OBJECTID} because there is no DateMeasured" ) continue @@ -95,24 +112,22 @@ def transfer_water_levels(session): measurement is the same as the date/time of the field event. """ - # if pd.isna(row.MeasuringAgency): - # collecting_organization = "Unknown" - # else: - # collecting_organization = row.MeasuringAgency - - # if pd.isna(row.MeasuredBy): - # sampler_name = "Unknown" - # else: - # sampler_name = row.MeasuredBy + # --- FieldEvent --- field_event = FieldEvent( thing=thing, event_date=dt_utc, - # collecting_organization=collecting_organization, release_status=release_status, ) session.add(field_event) + session.flush() + + logger.info( + f"{SPACE_2}Created field event: ID {field_event.id} | Date {field_event.event_date} | Thing ID {field_event.thing.id} | Thing Name {field_event.thing.name}" + ) + + # --- FieldActivity --- field_activity = FieldActivity( field_event=field_event, @@ -120,6 +135,230 @@ def transfer_water_levels(session): release_status=release_status, ) session.add(field_activity) + session.flush() + + logger.info( + f"{SPACE_4}Created field activity: ID {field_activity.id} | Type {field_activity.activity_type}" + ) + # --- Contact/FieldEventContactAssociation --- + # AMP feedback: + # - is Duke Engring the same as Duke University? Is it from their engineering school? + # - speak with AMP to help identify all initials + if pd.isna(row.MeasuredBy): + measured_by = None + else: + measured_by = row.MeasuredBy + + if pd.isna(row.MeasuringAgency): + measuring_agency = "Unknown" + else: + measuring_agency = row.MeasuringAgency + + # sometimes multiple contacts need to be created, so they'll be stored in a list + # the nth name corresponds with the nth organization + contact_names = [] + contact_organizations = [] + roles = [] + + # ns --> names + # os --> organizations + # rs --> roles + + # TODO: get help figuring out (AMP) + if measured_by in measured_by_mapper.keys(): + args = measured_by_mapper[measured_by] + if isinstance(args[0], list): + ns, os, rs = zip(*args) + else: + ns = [args[0]] + os = [args[1]] + rs = [args[2]] + elif measured_by is None: + ns = [None] + os = ["Unknown"] + rs = ["Unknown"] + elif measured_by in [ + "Anthony Chavez", + "BEI", + "BF/RG", + "Borchert", + "Borton & Cooper", + "CDWR", + "Chaves/Cruz", + "Chavez/Cruz", + "CM, AK", + "Coons", + "Cooper", + "Corbin", + "Crocker", + "Cruz", + "Cruz-Tribble", + "Cruz/Frost", + "D.Bird", + "D.D.", + "D.Duncan", + "Dames & Moore", + "Dames/Moore", + "Dave Snider", + "David N Jenkins", + "David N. Jenkins", + "DC", + "Decker", + "DL, TK", + "DR", + "DR, ST", + "Duke Engring", + "Duncan", + "EA", + "EA/HB", + "Frost", + "G.Boylan", + "GLR, SC", + "GLR, SK, SC", + "GR, MM", + "GR/PW", + "GR/RG", + "HB", + "Heaton", + "Horner-Crocker", + "HR", + "Hydrogeologic Serv", + "J.Evans", + "J.Frost", + "Jenkins", + "Johnson/Cruz", + "Johnson/Robbins", + "Kilmer/Jenkins", + "KP, MR", + "KP, MT", + "Lazarus", + "Mike Rodgers", + "Mourant", + "MWB Consultant", + "Myers report", + "Rankin", + "Sandia Drillers", + "SC, MR", + "SdC", + "SM&Assoc", + "SMA", + "Spiegel", + "Spiegel & Baldwin", + "SPRI", + "Steve", + "T.Decker", + "Topol", + "URS", + "UTM", + "VeneKlasen", + "Vista del Oro", + ]: + # set name to measured_by so that water level is logged to that + # person even if they are not known. this allows future updates + ns = [measured_by] + os = ["Unknown"] + rs = ["Unknown"] + logger.warning( + f"{SPACE_4}The following record has not been mapped to a Contact: {row.MeasuredBy} // {row.MeasuringAgency} for PointID {row.PointID} (which comes from the WaterLevels table)" + ) + + elif measured_by in [ + "?", + "Consultant", + "Consulting Pro.", + "Gamma log unit", + "Pump company", + "PumpService", + "REPORTED", + "Theis report", + "Unknown", + "Unknown; reported", + "Water operator", + "WWTP", + "WWTP personnel", + ]: + # Unknowns + ns = [None] + os = [measuring_agency] + rs = ["Unknown"] + + else: + logger.critical( + f"Skipping the following record because it has no mappings: {row.MeasuredBy} // {row.MeasuringAgency} for PointID {row.PointID}" + ) + continue + """ + Developer's notes + + Use existing contact for the thing if measured by is the owner + """ + field_event_contacts = [] + if measured_by not in ["Owner", "Owner report", "Well owner"]: + contact_names.extend(ns) + contact_organizations.extend(os) + roles.extend(rs) + for i, c in enumerate(contact_names): + if c not in CREATED_CONTACTS.keys(): + # create new contact if not already created + name = contact_names[i] + organization = contact_organizations[i] + role = roles[i] + + contact = Contact( + name=name, + role=role, + contact_type="Field Event Participant", + organization=organization, + nma_pk_waterlevels=row.GlobalID, + ) + session.add(contact) + session.flush() # to get the contact.id + + logger.info( + f"{SPACE_4}Created contact: ID {contact.id} | Name {contact.name} | Role {contact.role} | Organization {contact.organization} | nma_pk_waterlevels {contact.nma_pk_waterlevels}" + ) + + CREATED_CONTACTS[c] = contact + else: + contact = CREATED_CONTACTS[c] + field_event_contacts.append(contact) + else: + contact = thing.contacts[0] + field_event_contacts.append(contact) + + """ + Developer's notes + + Assumes that the first listed contact is the lead and the + person who took the sample. The subsequent contact will be + participants in the field event + """ + for i, fec in enumerate(field_event_contacts): + if i == 0: + field_event_contact = FieldEventContactAssociation( + field_event=field_event, + contact=fec, + field_contact_role="Lead", + ) + sampler = field_event_contact + else: + field_event_contact = FieldEventContactAssociation( + field_event=field_event, + contact=fec, + field_contact_role="Participant", + ) + session.add(field_event_contact) + session.flush() + logger.info( + f"{SPACE_4}Created field event contact: ID {field_event_contact.id} | Contact Name {field_event_contact.contact.name} | Field Contact Role {field_event_contact.field_contact_role}" + ) + + if pd.isna(row.DepthToWater): + logger.warning( + f"{SPACE_4}No sample and observation have been made for WaterLevels record with GlobalID {row.GlobalID} because DepthToWater is NULL" + ) + continue + # --- Sample --- if not pd.isna(row.MeasurementMethod): sample_method = lexicon_mapper.map_value( @@ -130,8 +369,9 @@ def transfer_water_levels(session): # todo: use create schema to validate data sample = Sample( + nma_pk_waterlevels=row.GlobalID, field_activity=field_activity, - # sampler_name=sampler_name, + field_event_contact=sampler, sample_date=dt_utc, sample_matrix="water", sample_name=str( @@ -143,10 +383,10 @@ def transfer_water_levels(session): depth_bottom=None, ) session.add(sample) - - # TODO: update for auto-collectors in the Sensor table, like e-probes - # update the deployment table here - sensor_id = None + session.flush() + logger.info( + f"{SPACE_4}Created sample: ID {sample.id} | Date {sample.sample_date} | Matrix {sample.sample_matrix} | Method {sample.sample_method}" + ) if not pd.isna(row.LevelStatus): level_status = lexicon_mapper.map_value( @@ -156,19 +396,25 @@ def transfer_water_levels(session): level_status = None # TODO: use create schema to validate data + + # TODO: after sensors have been added to the database update sensor_id (or sensor) for waterlevels that come from db sensors (like e probes?) observation = Observation( - sensor_id=sensor_id, - sample=sample, nma_pk_waterlevels=row.GlobalID, - value=row.DepthToWater, - measuring_point_height=row.MPHeight, + sample=sample, + sensor_id=None, + analysis_method_id=None, + observation_datetime=dt_utc, observed_property="groundwater level", + value=row.DepthToWater, unit="ft", + measuring_point_height=row.MPHeight, level_status=level_status, - observation_datetime=dt_utc, ) - session.add(observation) + session.flush() + logger.info( + f"{SPACE_4}Created observation: ID {observation.id} | DT {observation.observation_datetime} | Value {observation.value} | MPHeight {observation.measuring_point_height} | nma_pk_waterlevels {observation.nma_pk_waterlevels}" + ) session.commit()