NO TICKET: maintain contact schema | skip second contact if no info | persist invalid legacy phone data

db/contact.py

@@ -17,7 +17,7 @@
 
 from sqlalchemy import Integer, ForeignKey, String, UniqueConstraint
 from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy
-from sqlalchemy.orm import relationship, Mapped, mapped_column
+from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr
 from sqlalchemy_utils import TSVectorType
 
 from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term
@@ -68,6 +68,11 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin):
     addresses: Mapped[List["Address"]] = relationship(
         "Address", back_populates="contact", cascade="all, delete, delete-orphan"
     )
+    # One-To-One: A Contact can have one NMA Phone record.
+    incomplete_nma_phones: Mapped[List["IncompleteNMAPhone"]] = relationship(
+        "IncompleteNMAPhone", back_populates="contact", cascade="all, delete-orphan"
+    )
+
     # One-To-Many: A Contact can grant many Permissions.
     permissions: Mapped[List["Permission"]] = relationship(
         "Permission", back_populates="contact", cascade="all, delete, delete-orphan"
@@ -118,12 +123,32 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin):
     )
 
 
+class IncompleteNMAPhone(Base, AutoBaseMixin):
+    """
+    This table stores data from NM_Aquifer that is not complete and cannot be transferred to the Phone model due to validation issues.
+    This is often due to missing area codes, but could be other issues as well.
+    """
+
+    @declared_attr
+    def __tablename__(self):
+        return "incomplete_nma_phone"
+
+    contact_id: Mapped[int] = mapped_column(
+        ForeignKey("contact.id", ondelete="CASCADE"), nullable=False
+    )
+
+    phone_number: Mapped[str] = mapped_column(String(20), nullable=False)
+
+    contact: Mapped["Contact"] = relationship(
+        "Contact", back_populates="incomplete_nma_phones", passive_deletes=True
+    )
+
+
 class Phone(Base, AutoBaseMixin, ReleaseMixin):
     contact_id: Mapped[int] = mapped_column(
         ForeignKey("contact.id", ondelete="CASCADE"), nullable=False
     )
-    nma_phone_number: Mapped[str] = mapped_column(String(20), nullable=True)
-    phone_number: Mapped[str] = mapped_column(String(20), nullable=True)
+    phone_number: Mapped[str] = mapped_column(String(20), nullable=False)
     phone_type: Mapped[str] = lexicon_term(nullable=False)
 
     contact: Mapped["Contact"] = relationship(

schemas/contact.py

@@ -113,7 +113,6 @@ class CreatePhone(BaseCreateModel, ValidatePhone):
     contact_id: int | None = None  # set to None for when made via POST /contact
     phone_number: str
     phone_type: PhoneType = "Primary"  # Default to 'Primary'
-    nma_phone_number: str | None = None
 
 
 class CreateAddress(BaseCreateModel):
@@ -175,7 +174,6 @@ class PhoneResponse(BaseItemResponse):
 
     phone_number: str | None = None
     phone_type: str  # e.g., 'mobile', 'landline', etc.
-    nma_phone_number: str | None = None
 
 
 class EmailResponse(BaseItemResponse):
@@ -210,11 +208,19 @@ class ContactResponse(BaseResponseModel):
     organization: str | None
     role: Role
     contact_type: ContactType
+    incomplete_nma_phones: List[str] = []
     emails: List[EmailResponse] = []
     phones: List[PhoneResponse] = []
     addresses: List[AddressResponse] = []
     things: List[ThingResponse] = []  # List of related things
 
+    @field_validator("incomplete_nma_phones", mode="before")
+    def make_incomplete_nma_phone_str(cls, v: list) -> list:
+        if len(v) == 0:
+            return []
+        else:
+            return [p.phone_number for p in v]
+
 
 # class ThingContactAssociationResponse(BaseUpdateModel):
 #     """

services/contact_helper.py

@@ -39,6 +39,7 @@ def get_db_contacts(
         joinedload(Contact.thing_associations).joinedload(
             ThingContactAssociation.thing
         ),
+        joinedload(Contact.incomplete_nma_phones),
     )
 
     if thing_id:

tests/conftest.py

@@ -351,6 +351,36 @@ def contact(water_well_thing):
         session.commit()
 
 
+@pytest.fixture()
+def incomplete_nma_phone_1(contact):
+    with session_ctx() as session:
+        nma_phone = IncompleteNMAPhone(
+            phone_number="9999999",
+            contact_id=contact.id,
+        )
+        session.add(nma_phone)
+        session.commit()
+        session.refresh(nma_phone)
+        yield nma_phone
+        session.delete(nma_phone)
+        session.commit()
+
+
+@pytest.fixture()
+def incomplete_nma_phone_2(contact):
+    with session_ctx() as session:
+        nma_phone = IncompleteNMAPhone(
+            phone_number="8888888",
+            contact_id=contact.id,
+        )
+        session.add(nma_phone)
+        session.commit()
+        session.refresh(nma_phone)
+        yield nma_phone
+        session.delete(nma_phone)
+        session.commit()
+
+
 @pytest.fixture()
 def address(contact):
     with session_ctx() as session:

tests/test_contact.py

@@ -367,7 +367,9 @@ def test_add_phone_409_contact_not_found(contact):
 # GET tests ======================================================
 
 
-def test_get_contacts(contact, email, address, phone):
+def test_get_contacts(
+    contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2
+):
     response = client.get("/contact")
     assert response.status_code == 200
     data = response.json()
@@ -382,6 +384,11 @@ def test_get_contacts(contact, email, address, phone):
     assert data["items"][0]["release_status"] == contact.release_status
     assert data["items"][0]["organization"] == contact.organization
 
+    assert len(data["items"][0]["incomplete_nma_phones"]) == 2
+    assert sorted(data["items"][0]["incomplete_nma_phones"]) == sorted(
+        [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number]
+    )
+
     assert len(data["items"][0]["emails"]) == 1
     assert data["items"][0]["emails"][0]["id"] == email.id
     assert data["items"][0]["emails"][0]["created_at"] == email.created_at.astimezone(
@@ -427,7 +434,9 @@ def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing):
     assert data["items"][0]["id"] == contact.id
 
 
-def test_get_contact_by_id(contact, email, address, phone):
+def test_get_contact_by_id(
+    contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2
+):
     response = client.get(f"/contact/{contact.id}")
     assert response.status_code == 200
     data = response.json()
@@ -441,6 +450,11 @@ def test_get_contact_by_id(contact, email, address, phone):
     assert data["release_status"] == contact.release_status
     assert data["organization"] == contact.organization
 
+    assert len(data["incomplete_nma_phones"]) == 2
+    assert sorted(data["incomplete_nma_phones"]) == sorted(
+        [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number]
+    )
+
     assert len(data["emails"]) == 1
     assert data["emails"][0]["id"] == email.id
     assert data["emails"][0]["created_at"] == email.created_at.astimezone(

transfers/contact_transfer.py

@@ -17,7 +17,15 @@
 
 from pydantic import ValidationError
 
-from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address
+from db import (
+    Thing,
+    Contact,
+    ThingContactAssociation,
+    Email,
+    Phone,
+    Address,
+    IncompleteNMAPhone,
+)
 from transfers.logger import logger
 from transfers.util import (
     get_transfers_data_path,
@@ -101,6 +109,16 @@ def transfer_contacts(session):
                 errors.append({"pointid": row.PointID, "error": e})
 
             try:
+                if (
+                    row.SecondFirstName is None
+                    and row.SecondLastName is None
+                    and row.SecondCtctEmail is None
+                    and row.SecondCtctPhone is None
+                ):
+                    logger.warning(
+                        f"No second contact info for PointID {row.PointID}, skipping."
+                    )
+                    continue
                 _add_second_contact(session, row, thing, co_to_org_mapper)
                 session.commit()
                 # session.flush()
@@ -158,26 +176,32 @@ def _add_first_contact(session, row, thing, co_to_org_mapper):
             contact.emails.append(email)
 
     if row.Phone:
-        phone = _make_phone(
+        phone, complete = _make_phone(
             "first",
             row.OwnerKey,
             phone_number=row.Phone,
             phone_type="Primary",
             release_status=release_status,
         )
         if phone:
-            contact.phones.append(phone)
+            if complete:
+                contact.phones.append(phone)
+            else:
+                contact.incomplete_nma_phones.append(phone)
 
     if row.CellPhone:
-        phone = _make_phone(
+        phone, complete = _make_phone(
             "first",
             row.OwnerKey,
             phone_number=row.CellPhone,
             phone_type="Mobile",
             release_status=release_status,
         )
         if phone:
-            contact.phones.append(phone)
+            if complete:
+                contact.phones.append(phone)
+            else:
+                contact.incomplete_nma_phones.append(phone)
 
     if row.MailingAddress:
         address = _make_address(
@@ -244,15 +268,18 @@ def _add_second_contact(session, row, thing, co_to_org_mapper):
             contact.emails.append(email)
 
     if row.SecondCtctPhone:
-        phone = _make_phone(
+        phone, complete = _make_phone(
             "second",
             row.OwnerKey,
             phone_number=row.SecondCtctPhone,
             phone_type="Primary",
             release_status=release_status,
         )
         if phone:
-            contact.phones.append(phone)
+            if complete:
+                contact.phones.append(phone)
+            else:
+                contact.incomplete_nma_phones.append(phone)
 
 
 # helpers
@@ -290,14 +317,13 @@ def _make_phone(first_second, ownerkey, **kw):
             kw["phone_number"] = kw["phone_number"].strip()
 
         phone = CreatePhone(**kw)
-        return Phone(**phone.model_dump())
+        return Phone(**phone.model_dump()), True
     except ValidationError as e:
         try:
             if "phone_number" in kw:
-                pn = kw.pop("phone_number")
-                kw["nma_phone_number"] = pn.strip()
-            phone = CreatePhone(**kw)
-            return Phone(**phone.model_dump())
+                incomplete_phone = IncompleteNMAPhone(phone_number=kw["phone_number"])
+                logger.info(f"Salvaged incomplete phone number for OwnerKey {ownerkey}")
+                return incomplete_phone, False
         except ValidationError:
 
             logger.critical(

transfers/logger.py

@@ -40,7 +40,7 @@
 if not os.path.exists(root):
     os.mkdir(root)
 
-log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log"
+log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%H_%M_%S}.log"
 
 
 logging.basicConfig(

transfers/metrics.py

@@ -46,7 +46,7 @@ def __init__(self):
         if not os.path.exists(root):
             os.mkdir(root)
 
-        self.path = root / f"metrics_{datetime.now()}.csv"
+        self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H_%M_%S')}.csv"
         delimiter = "|" if self.include_errors else ","
         self._writer = csv.writer(self.path.open("a"), delimiter=delimiter)
         self._writer.writerow(
@@ -116,7 +116,7 @@ def _transducer_metrics(
         self._write_errors(errors)
 
     def _make_metrics(self, name, input_n, cleaned_n, count):
-        percent_issue = (cleaned_n - count) / cleaned_n * 100 if cleaned_n == 0 else 0
+        percent_issue = (cleaned_n - count) / cleaned_n * 100 if cleaned_n != 0 else 0
         return [name, input_n, cleaned_n, count, percent_issue]
 
     def _handle_metrics(