BDMS-158: Company to Organization Mapping & Other Updates

.gitignore

@@ -24,6 +24,7 @@ migrate.sh
 launcher.sh
 gcs_credentials.json
 transfers/data/assets*
+transfers/data/nma_csv_cache/*
 transfers/transfer*.log
 transfer*.log
 tests/features/*.feature

db/contact.py

@@ -15,7 +15,7 @@
 # ===============================================================================
 from typing import List, TYPE_CHECKING
 
-from sqlalchemy import Integer, ForeignKey, String
+from sqlalchemy import Integer, ForeignKey, String, UniqueConstraint
 from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy
 from sqlalchemy.orm import relationship, Mapped, mapped_column
 from sqlalchemy_utils import TSVectorType
@@ -113,6 +113,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin):
         TSVectorType("name", "role", "organization", "nma_pk_owners")
     )
 
+    __table_args__ = (
+        UniqueConstraint("name", "organization", name="uq_contact_name_organization"),
+    )
+
 
 class Phone(Base, AutoBaseMixin, ReleaseMixin):
     contact_id: Mapped[int] = mapped_column(

tests/__init__.py

@@ -13,21 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-import os
-
 # Load .env file BEFORE importing anything else
 # Use override=True to override conflicting shell environment variables
 from dotenv import load_dotenv
 
 load_dotenv(override=True)
 
-# Set timezone to UTC for consistent datetime handling in tests
-os.environ["TZ"] = "UTC"
+# this should not be needed since all Pydantic serializes all datetimes as UTC
+# furthermore, tzset is not supported on Windows, so this breaks cross-platform compatibility
+# # Set timezone to UTC for consistent datetime handling in tests
+# os.environ["TZ"] = "UTC"
 
-# Also set time.tzset() to apply the timezone change
-import time
+# # Also set time.tzset() to apply the timezone change
+# import time
 
-time.tzset()
+# time.tzset()
 
 from transfers.transfer import erase_and_initalize
 from fastapi.testclient import TestClient

transfers/contact_transfer.py

@@ -14,7 +14,15 @@
 # limitations under the License.
 # ===============================================================================
 from pydantic import ValidationError
-
+import json
+
+from transfers.util import (
+    read_csv,
+    filter_to_valid_point_ids,
+    replace_nans,
+    get_transfers_data_path,
+)
+from transfers.logger import logger
 from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address
 from transfers.logger import logger
 from transfers.util import read_csv, filter_to_valid_point_ids, replace_nans
@@ -43,6 +51,10 @@ def extract_owner_role(comment):
 
 def transfer_contacts(session):
 
+    co_to_org_mapper_path = get_transfers_data_path("owners_organization_mapper.json")
+    with open(co_to_org_mapper_path, "r") as f:
+        co_to_org_mapper = json.load(f)
+
     odf = read_csv("OwnersData")
     odf = odf.drop(["OBJECTID", "GlobalID"], axis=1)
     ldf = read_csv("OwnerLink")
@@ -66,7 +78,7 @@ def transfer_contacts(session):
 
         # TODO: use contact_helper.add_contact
         try:
-            _add_first_contact(session, row, thing)
+            _add_first_contact(session, row, thing, co_to_org_mapper)
             session.commit()
             session.flush()
             logger.info(f"added first contact for PointID {row.PointID}")
@@ -82,7 +94,7 @@ def transfer_contacts(session):
             session.rollback()
 
         try:
-            _add_second_contact(session, row, thing)
+            _add_second_contact(session, row, thing, co_to_org_mapper)
             session.commit()
             session.flush()
             logger.info(f"added second contact for PointID {row.PointID}")
@@ -98,21 +110,23 @@ def transfer_contacts(session):
             session.rollback()
 
 
-def _add_first_contact(session, row, thing):
+def _add_first_contact(session, row, thing, co_to_org_mapper):
     # TODO: extract role from OwnerComment
     # role = extract_owner_role(row.OwnerComment)
     role = "Owner"
     release_status = "private"
 
     name = _make_name(row.FirstName, row.LastName)
 
+    organization = co_to_org_mapper.get(row.Company, row.Company)
+
     contact_data = {
         "thing_id": thing.id,
         "release_status": release_status,
         "name": name,
         "role": role,
         "contact_type": "Primary",
-        "organization": row.Company,
+        "organization": organization,
         "nma_pk_owners": row.OwnerKey,
         "addresses": [],
         "emails": [],
@@ -185,18 +199,20 @@ def _add_first_contact(session, row, thing):
             contact.addresses.append(address)
 
 
-def _add_second_contact(session, row, thing):
+def _add_second_contact(session, row, thing, co_to_org_mapper):
 
     release_status = "private"
     name = _make_name(row.SecondFirstName, row.SecondLastName)
 
+    organization = co_to_org_mapper.get(row.Company, row.Company)
+
     contact_data = {
         "thing_id": thing.id,
         "release_status": release_status,
         "name": name,
         "role": "Owner",
         "contact_type": "Secondary",
-        "organization": row.Company,
+        "organization": organization,
         "nma_pk_owners": row.OwnerKey,
         "addresses": [],
         "emails": [],

transfers/data/measured_by_mapper.json

@@ -38,7 +38,7 @@
     "Minton.": [null, "Minton Engineers", "Organization"],
     "MJ Darr.": [null, "MJDarrconsult, Inc", "Organization"],
     "MJ Darr consultants": [null, "MJDarrconsult, Inc", "Organization"],
-    "NESWCD": [null, "Northeastern Soil and Water Conservation District", "Organization"],
+    "NESWCD": [null, "Northeastern SWCD", "Organization"],
     "OSE, ST": [[null, "NMOSE", "Organization"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]],
     "PVACD person": [null, "PVACD", "Organization"],
     "Sandia Drillers": [null, "Sandia Well Service, Inc", "Organization"],
@@ -267,7 +267,7 @@
     "Kilmer/Jenkins": [["Kilmer", "Unknown", "Unknown"], ["David N. Jenkins", "Unknown", "Unknown"]],
     "KP": ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"],
     "KP, MF": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Marissa Fichera", "NMBGMR", "Hydrogeologist"]],
-    "KP, MR": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["MR", "NMBGMR", "Unknown"]],
+    "KP, MR": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Madeline Richards", "NMT", "Graduate Student"]],
     "KP, MT": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["MT", "Unknown", "Unknown"]],
     "KP, ST": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]],
     "KP, TK": [["Kitty Pokorny", "NMBGMR", "Hydrogeologist"], ["Trevor Kludt", "NMBGMR", "Technician"]],
@@ -311,7 +311,7 @@
     "SC, GR": [["Scott Christenson", "NMBGMR", "Technician"], ["Geoff Rawling", "NMBGMR", "Hydrogeologist"]],
     "SC, KP": [["Scott Christenson", "NMBGMR", "Technician"], ["Kitty Pokorny", "NMBGMR", "Hydrogeologist"]],
     "SC, MA": [["Scott Christenson", "NMBGMR", "Technician"], ["MA", "Unknown", "Unknown"]],
-    "SC, MR": [["Scott Christenson", "NMBGMR", "Technician"], ["MR", "NMBGMR", "Unknown"]],
+    "SC, MR": [["Scott Christenson", "NMBGMR", "Technician"], ["Madeline Richards", "NMT", "Graduate Student"]],
     "SC, SMC": [["Scott Christenson", "NMBGMR", "Technician"], ["Sara Chudnoff", "NMBGMR", "Hydrogeologist"]],
     "SC, ST": [["Scott Christenson", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"]],
     "SC, TK": [["Scott Christenson", "NMBGMR", "Technician"], ["Trevor Kludt", "NMBGMR", "Technician"]],
@@ -368,7 +368,7 @@
     "TK, ST, CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]],
     "TK, ST; CE": [["Trevor Kludt", "NMBGMR", "Technician"], ["Stacy Timmons", "NMBGMR", "Hydrogeologist"], ["Cathy Eisen", "NMBGMR", "Hydrogeologist"]],
     "TK, JAA": [["Trevor Kludt", "NMBGMR", "Technician"], ["JAA", "NMBGMR", "Unknown"]],
-    "TK, MR": [["Trevor Kludt", "NMBGMR", "Technician"], ["MR", "Unknown", "Unknown"]],
+    "TK, MR": [["Trevor Kludt", "NMBGMR", "Technician"], ["Madeline Richards", "NMT", "Graduate Student"]],
     "TK, TN": [["Trevor Kludt", "NMBGMR", "Technician"], ["Talon Newton", "NMBGMR", "Hydrogeologist"]],
     "TN": ["Talon Newton", "NMBGMR", "Hydrogeologist"],
     "TN, LL": [["Talon Newton", "NMBGMR", "Hydrogeologist"], ["Lewis Land", "NMBGMR", "Hydrogeologist"]],

transfers/data/owners_organization_mapper.json

@@ -0,0 +1,159 @@
+{
+	"Abeyta Engineering": "Abeyta Engineering, Inc",
+	"Adobe Ranch": "Adobe Ranch",
+	"Agua Fria Water Assn.": "Agua Fria Community Water Association",
+	"Apache Gap Ranch": "Apache Gap Ranch",
+	"Aspendale Baptist Camp": "Aspendale Mountain Retreat",
+	"Augustin Ranch LLC": "Augustin Plains Ranch LLC",
+	"B + B Cattle": "B & B Cattle Co",
+	"BLM Taos Office": "BLM Taos Office",	
+	"Berridge Distributing Company": "Berridge Distributing Company",
+	"Bishop's Lodge": "Bishop's Lodge",
+	"Bonanza Creek Ranch": "Bonanza Creek Ranch",
+	"Bug Scuffle Water Association": "Bug Scuffle Water Association",
+	"Bureau of Land Management": "BLM",
+	"Camp Wehinahpay": "Wehinahpay Mountain Camp",
+	"Campbell Ranch": "Campbell Ranch",
+	"Capitol Ford Auto Body": "Capitol Ford Santa Fe",
+	"Cemex Plant": "Cemex, Inc",
+	"Cerro Community Center": "Cerro Community Center",
+	"Chabad Jewish Center": "Santa Fe Jewish Center",
+	"Chupadero MDWCA": "Chupadero MDWCA",
+	"Cielo Lumbre HOA": "Cielo Lumbre HOA",
+	"Circle Cross Ranch": "Circle Cross Ranch",
+	"City of Alamogordo": "City of Alamogordo",
+	"City of Portales, Public Works Dept.": "City of Portales, Public Works Dept.",
+	"City of Santa Fe WWTP": "City of Santa Fe WWTP",
+	"City of Santa Fe, Municipal Recreation Complex": "City of Santa Fe, Municipal Recreation Complex",
+	"City of Santa Fe, Sangre de Cristo Water Co.": "City of Santa Fe, Sangre de Cristo Water Co.",
+	"City of Socorro": "City of Socorro",
+	"Commonwealth Conservancy": "Commonwealth Conservancy",
+	"Country Club Garden MHP": "Country Club Garden Mobile Home Park",
+	"Crossroads Cattle Co., Ltd.": "Crossroads Cattle Co., Ltd",
+	"Double H Ranch": "Double H Ranch",
+	"E.A. Meadows East": "E.A. Meadows East",
+	"El Camino Realty": "El Camino Realty, Inc",
+	"El Dorado Utilities": "Eldorado Area Water & Sanitation District",
+	"El Gancho Restaurant": "Bourbon Grill at El Gancho",
+	"El Prado HOA": "El Prado HOA",
+	"El Rancho de las Golondrinas": "El Rancho de las Golondrinas",
+	"El Rito MDWCA": "El Rito Canyon MDWCA",
+	"Eldorado Water and Sanitation District": "Eldorado Area Water & Sanitation District",
+	"Encantado Enterprises": "Encantado Enterprises",
+	"Estrella Concepts LLC": "Estrella Concepts LLC",
+	"Farr Cattle Company (Farr Ranch": "Farr Cattle Company",
+	"Fire Department": "Sixteen Springs Fire Department",
+	"Fire Water Lodge": "Fire Water Lodge",
+	"Ford County Land and Cattle": "Ford County Land & Cattle Company, Inc",
+	"Friendly Construction, Inc.": "Friendly Construction, Inc",
+	"Hacienda Del Cerezo, Ltd.": "Hacienda Del Cerezo",
+	"Hefker Vega Ranch": "Hefker Vega Ranch",
+	"High Nogal Ranch": "High Nogal Ranch",
+	"Holloman Air Force Base": "Holloman Air Force Base",
+	"Hyde Park Estates MDWCA": "Hyde Park Estates MDWCA",
+	"Hyde Park Estates WUA": "Hyde Park Estates MDWCA",
+	"Ideal Mobile Home & RV": "Desert Village RV & Mobile Home Park",
+	"K. Schmitt Trust": "K. Schmitt Trust",
+	"La Cienega Mutual Domestic Assn.": "La Cienega MDWCA",
+	"La Vista HOA": "La Vista HOA",
+	"Lamy MDWUA": "Lamy MDWCA",
+	"Land Ventures LLC": "Land Ventures LLC",
+	"Las Lagunitas": "Las Lagunitas",
+	"Las Lagunitas Homeowners Assn.": "Las Lagunitas HOA",
+	"Living World Ministries": "Living World Ministries",
+	"Los Atrevidos, Inc.": "Los Atrevidos, Inc",
+	"Los Prados HOA": "Los Prados HOA",
+	"Malaga Water System": "Malaga MDWCA & SWA",
+	"Mangas Outfitters": "Mangas Outfitters",
+	"Media Gravel Pit": "Medina Gravel Pit",
+	"Mendenhall Trading Co.": "Mendenhall Trading Co",
+	"Mesa Verde Ranch": "Mesa Verde Ranch",
+	"NESWCD": "Northeastern SWCD",
+	"TSWCD": "Taos SWCD",
+	"NM Environment Dept.": "NMED",
+	"NM Game & Fish Dept.": "NMDGF",
+	"NM Office of the State Engineer": "NMOSE",
+	"NM State Highway Dept.": "NMDOT",
+	"NMSU College of Agriculture": "NMSU College of Agriculture",
+	"Naiche Development": "Naiche Development",
+	"National Radio Astronomy Observatory": "NRAO",
+	"New Mexico Spaceport Authority": "NMSA",
+	"New Mexico Tech": "NMT",
+	"Nogal Mutual Domestic Water Consumers Association": "Nogal MDWCA",
+	"O Bar O Ranch": "O Bar O Ranch",
+	"OMI Wastewater Treatment Plant": "OMI Wastewater Treatment Plant",
+	"Old Road Ranch Pardners Ltd.": "Old Road Ranch Pardners Ltd",
+	"PNM Service Center": "PNM Service Center",
+	"PNM, Sangre de Cristo": "PNM Service Center",
+	"Peace Tabernacle Church": "Peace Tabernacle Church",
+	"Pecos Trail Inn": "Pecos Trail Inn",
+	"Pelican Spa": "Pelican Spa",
+	"Pistachio Tree Ranch": "Pistachio Tree Ranch",
+	"Rancho Encantado": "Rancho Encantado",
+	"Rancho San Lucas": "Rancho San Lucas",
+	"Rancho San Marcos": "Rancho San Marcos",
+	"Rancho Viejo Partnership": "Rancho Viejo Partnership",
+	"Ranney Ranch": "Ranney Ranch",
+	"Rio En Medio MDWCA": "Rio En Medio MDWCA",
+	"San Acacia MDWCA": "San Acacia MDWCA",
+	"San Juan Residences": "San Juan Residences",
+	"Sandia National Laboratories": "SNL",
+	"Sangre de Cristo Center": "Sangre de Cristo Center",
+	"Sangre de Cristo Estates": "Sangre de Cristo Estates",
+	"Santa Fe Community College": "Santa Fe Community College",
+	"Santa Fe County": "SFC",
+	"Santa Fe County, Fire Facilities": "SFC, Fire Facilities",
+	"Santa Fe County, Utilities Dept.": "SFC, Utilities Dept.",
+	"Santa Fe County, Valle Vista Water Utility, Inc.": "Santa Fe County, Valle Vista Water Utility, Inc.",
+	"Santa Fe Downs": "Santa Fe Downs",
+	"Santa Fe Horse Park": "Santa Fe Horse Park",
+	"Santa Fe Municipal Airport": "Santa Fe Municipal Airport",
+	"Santa Fe Opera": "Santa Fe Opera",
+	"Santa Fe Waldorf School": "Santa Fe Waldorf School",
+	"Shidoni Foundry, Inc.": "Shidoni Foundry and Gallery",
+	"Sierra Grande Lodge": "Sierra Grande Lodge",
+	"Sierra Vista Retirement Community": "Sierra Vista Retirement Community",
+	"Slash Triangle Ranch": "Slash Triangle Ranch",
+	"Stagecoach Motel": "Stagecoach Motel",
+	"State of New Mexico": "State of New Mexico",
+	"Stephenson Ranch": "Stephenson Ranch",
+	"Sun Broadcasting Network": "Sun Broadcasting Network",
+	"Tano Rd LLC": "Tano Rd LLC",
+	"Taos Municipal Schools, UNM Taos": "UNM-Taos",
+	"Tee Pee Ranch/Tee Pee Subdivision": "Tee Pee Ranch/Tee Pee Subdivision",
+	"Tent Rock, Inc.": "Tent Rock, Inc",
+	"Tesuque": "Tesuque MDWCA",
+	"Tesuque MDWCA": "Tesuque MDWCA",
+	"The Great Cloud Zen Center": "The Great Cloud Zen Center",
+	"Three Rivers Ranch": "Three Rivers Ranch",
+	"Timberon Water and Sanitation District": "Timberon Water and Sanitation District",
+	"Town of Magdalena": "Town of Magdalena",
+	"Town of Taos": "Town of Taos",
+	"Town of Taos, National Guard Armory": "Town of Taos, National Guard Armory",
+	"Trinity Ranch": "Trinity Ranch",
+	"Tularosa Basin Natl. Desalination Research Facil.": "Tularosa Basin National Desalination Research Facility",
+	"Turquoise Trail Charter School": "Turquoise Trail Charter School",
+	"US Bureau of Indian Affairs, Indian School": "US Bureau of Indian Affairs, Santa Fe Indian School",
+	"USFS, Carson NF, Taos Office": "USFS, Carson NF, Taos Office",
+	"USFS, Cibola NF, Magdalena Ranger District": "USFS, Cibola NF, Magdalena Ranger District",
+	"USFS, Santa Fe NF, Espanola Ranger District": "USFS, Santa Fe NF, Espanola Ranger District",
+	"USFS/Bluewater Ranch": "USFS",
+	"Ute Mountain Farms": "Ute Mountain Farms",
+	"VA Hospital": "VA Hospital",
+	"Valle Vista Water Utility": "Valle Vista Water Utility",
+	"Velte": "Velte",
+	"Vereda Serena Property": "Vereda Serena Property",
+	"Village of Corona": "Village of Corona",
+	"Village of Floyd": "Village of Floyd",
+	"Village of Melrose": "Village of Melrose",
+	"Village of Vaughn": "Village of Vaughn",
+	"Vista Land Company": "Vista Land Company",
+	"Vista Redonda MDWCA": "Vista Redonda MDWCA",
+	"Vista Redondo MDWCA": "Vista Redonda MDWCA",
+	"Vista de Oro Water Co-op": "Vista de Oro de Placitas Water Users Coop",
+	"Walker Ranch": "Walker Ranch",
+	"Wild & Woolley Trailer Ranch": "Wild & Woolley Trailer Ranch",
+	"Winter Brothers/U.S. Government": "Winter Brothers",
+	"Yates Petroleum": "Yates Petroleum Corporation",
+	"Zamora Accounting Services": "Zamora Accounting Services"
+}