From cfee8dd0495016e3eab5a23f55a83b94d7016ec6 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 30 Oct 2025 17:14:01 -0600 Subject: [PATCH 01/12] fix: maintain invalid legacy phones in separate classes --- db/contact.py | 32 +++++++++++++++++++++++++++++++- schemas/contact.py | 16 ++++++++++++++-- services/contact_helper.py | 2 ++ tests/conftest.py | 32 ++++++++++++++++++++++++++++++++ tests/test_contact.py | 8 ++++++-- 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/db/contact.py b/db/contact.py index a674820ab..d2ee94778 100644 --- a/db/contact.py +++ b/db/contact.py @@ -68,6 +68,13 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): addresses: Mapped[List["Address"]] = relationship( "Address", back_populates="contact", cascade="all, delete, delete-orphan" ) + # One-To-One: A Contact can have one NMA Phone record. + nma_phone: Mapped["NMAPhone"] = relationship( + "NMAPhone", back_populates="contact", cascade="all, delete-orphan" + ) + nma_cell_phone: Mapped["NMACellPhone"] = relationship( + "NMACellPhone", back_populates="contact", cascade="all, delete-orphan" + ) # One-To-Many: A Contact can grant many Permissions. permissions: Mapped[List["Permission"]] = relationship( "Permission", back_populates="contact", cascade="all, delete, delete-orphan" @@ -118,11 +125,34 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) +class NMAPhone(Base, AutoBaseMixin): + contact_id: Mapped[int] = mapped_column( + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + ) + + phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + + contact: Mapped["Contact"] = relationship( + "Contact", back_populates="nma_phone", passive_deletes=True + ) + + +class NMACellPhone(Base, AutoBaseMixin): + contact_id: Mapped[int] = mapped_column( + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + ) + + phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + + contact: Mapped["Contact"] = relationship( + "Contact", back_populates="nma_cell_phone", passive_deletes=True + ) + + class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - nma_phone_number: Mapped[str] = mapped_column(String(20), nullable=True) phone_number: Mapped[str] = mapped_column(String(20), nullable=True) phone_type: Mapped[str] = lexicon_term(nullable=False) diff --git a/schemas/contact.py b/schemas/contact.py index ca4838409..6d6a92027 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -113,7 +113,6 @@ class CreatePhone(BaseCreateModel, ValidatePhone): contact_id: int | None = None # set to None for when made via POST /contact phone_number: str phone_type: PhoneType = "Primary" # Default to 'Primary' - nma_phone_number: str | None = None class CreateAddress(BaseCreateModel): @@ -175,7 +174,6 @@ class PhoneResponse(BaseItemResponse): phone_number: str | None = None phone_type: str # e.g., 'mobile', 'landline', etc. - nma_phone_number: str | None = None class EmailResponse(BaseItemResponse): @@ -210,11 +208,25 @@ class ContactResponse(BaseResponseModel): organization: str | None role: Role contact_type: ContactType + nma_phone: str | None + nma_cell_phone: str | None emails: List[EmailResponse] = [] phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] things: List[ThingResponse] = [] # List of related things + @field_validator("nma_phone", mode="before") + def make_nma_phone_str(cls, v) -> str | None: + if v is not None: + return v.phone_number + return None + + @field_validator("nma_cell_phone", mode="before") + def make_nma_cell_phone_str(cls, v) -> str | None: + if v is not None: + return v.phone_number + return None + # class ThingContactAssociationResponse(BaseUpdateModel): # """ diff --git a/services/contact_helper.py b/services/contact_helper.py index f50a3b3e1..e4c2f4d01 100644 --- a/services/contact_helper.py +++ b/services/contact_helper.py @@ -39,6 +39,8 @@ def get_db_contacts( joinedload(Contact.thing_associations).joinedload( ThingContactAssociation.thing ), + joinedload(Contact.nma_phone), + joinedload(Contact.nma_cell_phone), ) if thing_id: diff --git a/tests/conftest.py b/tests/conftest.py index fae8aedaa..2b5191894 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -333,6 +333,8 @@ def contact(water_well_thing): role="Owner", contact_type="Primary", organization="NMBGMR", + # nma_primary_phone_number="9999999", + # nma_secondary_phone_number="8888888", ) session.add(contact) session.commit() @@ -351,6 +353,36 @@ def contact(water_well_thing): session.commit() +@pytest.fixture() +def nma_phone(contact): + with session_ctx() as session: + nma_phone = NMAPhone( + phone_number="9999999", + contact_id=contact.id, + ) + session.add(nma_phone) + session.commit() + session.refresh(nma_phone) + yield nma_phone + session.delete(nma_phone) + session.commit() + + +@pytest.fixture() +def nma_cell_phone(contact): + with session_ctx() as session: + nma_cell_phone = NMACellPhone( + phone_number="8888888", + contact_id=contact.id, + ) + session.add(nma_cell_phone) + session.commit() + session.refresh(nma_cell_phone) + yield nma_cell_phone + session.delete(nma_cell_phone) + session.commit() + + @pytest.fixture() def address(contact): with session_ctx() as session: diff --git a/tests/test_contact.py b/tests/test_contact.py index 4c25f4473..9a1b766cc 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -367,7 +367,7 @@ def test_add_phone_409_contact_not_found(contact): # GET tests ====================================================== -def test_get_contacts(contact, email, address, phone): +def test_get_contacts(contact, email, address, phone, nma_phone, nma_cell_phone): response = client.get("/contact") assert response.status_code == 200 data = response.json() @@ -381,6 +381,8 @@ def test_get_contacts(contact, email, address, phone): assert data["items"][0]["contact_type"] == contact.contact_type assert data["items"][0]["release_status"] == contact.release_status assert data["items"][0]["organization"] == contact.organization + assert data["items"][0]["nma_phone"] == nma_phone.phone_number + assert data["items"][0]["nma_cell_phone"] == nma_cell_phone.phone_number assert len(data["items"][0]["emails"]) == 1 assert data["items"][0]["emails"][0]["id"] == email.id @@ -427,7 +429,7 @@ def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing): assert data["items"][0]["id"] == contact.id -def test_get_contact_by_id(contact, email, address, phone): +def test_get_contact_by_id(contact, email, address, phone, nma_phone, nma_cell_phone): response = client.get(f"/contact/{contact.id}") assert response.status_code == 200 data = response.json() @@ -440,6 +442,8 @@ def test_get_contact_by_id(contact, email, address, phone): assert data["contact_type"] == contact.contact_type assert data["release_status"] == contact.release_status assert data["organization"] == contact.organization + assert data["nma_phone"] == nma_phone.phone_number + assert data["nma_cell_phone"] == nma_cell_phone.phone_number assert len(data["emails"]) == 1 assert data["emails"][0]["id"] == email.id From 1c431096f4baf54701c2e170802b24b1cd420c79 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 09:15:09 -0600 Subject: [PATCH 02/12] feat: store nma phones in own class for strict schemas --- db/contact.py | 20 +++----------------- schemas/contact.py | 20 +++++++------------- services/contact_helper.py | 3 +-- tests/conftest.py | 16 +++++++--------- tests/test_contact.py | 18 ++++++++++++------ 5 files changed, 30 insertions(+), 47 deletions(-) diff --git a/db/contact.py b/db/contact.py index d2ee94778..003f5b304 100644 --- a/db/contact.py +++ b/db/contact.py @@ -69,12 +69,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): "Address", back_populates="contact", cascade="all, delete, delete-orphan" ) # One-To-One: A Contact can have one NMA Phone record. - nma_phone: Mapped["NMAPhone"] = relationship( + nma_phones: Mapped[List["NMAPhone"]] = relationship( "NMAPhone", back_populates="contact", cascade="all, delete-orphan" ) - nma_cell_phone: Mapped["NMACellPhone"] = relationship( - "NMACellPhone", back_populates="contact", cascade="all, delete-orphan" - ) + # One-To-Many: A Contact can grant many Permissions. permissions: Mapped[List["Permission"]] = relationship( "Permission", back_populates="contact", cascade="all, delete, delete-orphan" @@ -133,19 +131,7 @@ class NMAPhone(Base, AutoBaseMixin): phone_number: Mapped[str] = mapped_column(String(20), nullable=True) contact: Mapped["Contact"] = relationship( - "Contact", back_populates="nma_phone", passive_deletes=True - ) - - -class NMACellPhone(Base, AutoBaseMixin): - contact_id: Mapped[int] = mapped_column( - ForeignKey("contact.id", ondelete="CASCADE"), nullable=False - ) - - phone_number: Mapped[str] = mapped_column(String(20), nullable=True) - - contact: Mapped["Contact"] = relationship( - "Contact", back_populates="nma_cell_phone", passive_deletes=True + "Contact", back_populates="nma_phones", passive_deletes=True ) diff --git a/schemas/contact.py b/schemas/contact.py index 6d6a92027..d54bdc47d 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -208,24 +208,18 @@ class ContactResponse(BaseResponseModel): organization: str | None role: Role contact_type: ContactType - nma_phone: str | None - nma_cell_phone: str | None + nma_phones: List[str] = [] emails: List[EmailResponse] = [] phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] things: List[ThingResponse] = [] # List of related things - @field_validator("nma_phone", mode="before") - def make_nma_phone_str(cls, v) -> str | None: - if v is not None: - return v.phone_number - return None - - @field_validator("nma_cell_phone", mode="before") - def make_nma_cell_phone_str(cls, v) -> str | None: - if v is not None: - return v.phone_number - return None + @field_validator("nma_phones", mode="before") + def make_nma_phone_str(cls, v: list) -> list: + if len(v) == 0: + return [] + else: + return [p.phone_number for p in v] # class ThingContactAssociationResponse(BaseUpdateModel): diff --git a/services/contact_helper.py b/services/contact_helper.py index e4c2f4d01..ac75fba89 100644 --- a/services/contact_helper.py +++ b/services/contact_helper.py @@ -39,8 +39,7 @@ def get_db_contacts( joinedload(Contact.thing_associations).joinedload( ThingContactAssociation.thing ), - joinedload(Contact.nma_phone), - joinedload(Contact.nma_cell_phone), + joinedload(Contact.nma_phones), ) if thing_id: diff --git a/tests/conftest.py b/tests/conftest.py index 2b5191894..f7dc15f4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -333,8 +333,6 @@ def contact(water_well_thing): role="Owner", contact_type="Primary", organization="NMBGMR", - # nma_primary_phone_number="9999999", - # nma_secondary_phone_number="8888888", ) session.add(contact) session.commit() @@ -354,7 +352,7 @@ def contact(water_well_thing): @pytest.fixture() -def nma_phone(contact): +def nma_phone_1(contact): with session_ctx() as session: nma_phone = NMAPhone( phone_number="9999999", @@ -369,17 +367,17 @@ def nma_phone(contact): @pytest.fixture() -def nma_cell_phone(contact): +def nma_phone_2(contact): with session_ctx() as session: - nma_cell_phone = NMACellPhone( + nma_phone = NMAPhone( phone_number="8888888", contact_id=contact.id, ) - session.add(nma_cell_phone) + session.add(nma_phone) session.commit() - session.refresh(nma_cell_phone) - yield nma_cell_phone - session.delete(nma_cell_phone) + session.refresh(nma_phone) + yield nma_phone + session.delete(nma_phone) session.commit() diff --git a/tests/test_contact.py b/tests/test_contact.py index 9a1b766cc..e7a8afc14 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -367,7 +367,7 @@ def test_add_phone_409_contact_not_found(contact): # GET tests ====================================================== -def test_get_contacts(contact, email, address, phone, nma_phone, nma_cell_phone): +def test_get_contacts(contact, email, address, phone, nma_phone_1, nma_phone_2): response = client.get("/contact") assert response.status_code == 200 data = response.json() @@ -381,8 +381,11 @@ def test_get_contacts(contact, email, address, phone, nma_phone, nma_cell_phone) assert data["items"][0]["contact_type"] == contact.contact_type assert data["items"][0]["release_status"] == contact.release_status assert data["items"][0]["organization"] == contact.organization - assert data["items"][0]["nma_phone"] == nma_phone.phone_number - assert data["items"][0]["nma_cell_phone"] == nma_cell_phone.phone_number + + assert len(data["items"][0]["nma_phones"]) == 2 + assert sorted(data["items"][0]["nma_phones"]) == sorted( + [nma_phone_1.phone_number, nma_phone_2.phone_number] + ) assert len(data["items"][0]["emails"]) == 1 assert data["items"][0]["emails"][0]["id"] == email.id @@ -429,7 +432,7 @@ def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing): assert data["items"][0]["id"] == contact.id -def test_get_contact_by_id(contact, email, address, phone, nma_phone, nma_cell_phone): +def test_get_contact_by_id(contact, email, address, phone, nma_phone_1, nma_phone_2): response = client.get(f"/contact/{contact.id}") assert response.status_code == 200 data = response.json() @@ -442,8 +445,11 @@ def test_get_contact_by_id(contact, email, address, phone, nma_phone, nma_cell_p assert data["contact_type"] == contact.contact_type assert data["release_status"] == contact.release_status assert data["organization"] == contact.organization - assert data["nma_phone"] == nma_phone.phone_number - assert data["nma_cell_phone"] == nma_cell_phone.phone_number + + assert len(data["nma_phones"]) == 2 + assert sorted(data["nma_phones"]) == sorted( + [nma_phone_1.phone_number, nma_phone_2.phone_number] + ) assert len(data["emails"]) == 1 assert data["emails"][0]["id"] == email.id From 0f5f45febef2899edd72897a07acfa3985fdb029 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 10:09:53 -0600 Subject: [PATCH 03/12] refactor: prepend 'incomplete' to legacy nma_phone --- db/contact.py | 13 +++++++++---- schemas/contact.py | 6 +++--- services/contact_helper.py | 2 +- tests/conftest.py | 8 ++++---- tests/test_contact.py | 20 ++++++++++++-------- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/db/contact.py b/db/contact.py index 003f5b304..e5413ac4a 100644 --- a/db/contact.py +++ b/db/contact.py @@ -69,8 +69,8 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): "Address", back_populates="contact", cascade="all, delete, delete-orphan" ) # One-To-One: A Contact can have one NMA Phone record. - nma_phones: Mapped[List["NMAPhone"]] = relationship( - "NMAPhone", back_populates="contact", cascade="all, delete-orphan" + incomplete_nma_phones: Mapped[List["IncompleteNMAPhone"]] = relationship( + "IncompleteNMAPhone", back_populates="contact", cascade="all, delete-orphan" ) # One-To-Many: A Contact can grant many Permissions. @@ -123,7 +123,12 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) -class NMAPhone(Base, AutoBaseMixin): +class IncompleteNMAPhone(Base, AutoBaseMixin): + """ + This table stores data from NM_Aquifer that is not complete and cannot be transferred to the Phone model due to validation issues. + This is often due to missing area codes, but could be other issues as well. + """ + contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) @@ -131,7 +136,7 @@ class NMAPhone(Base, AutoBaseMixin): phone_number: Mapped[str] = mapped_column(String(20), nullable=True) contact: Mapped["Contact"] = relationship( - "Contact", back_populates="nma_phones", passive_deletes=True + "Contact", back_populates="incomplete_nma_phones", passive_deletes=True ) diff --git a/schemas/contact.py b/schemas/contact.py index d54bdc47d..d43cd4aaf 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -208,14 +208,14 @@ class ContactResponse(BaseResponseModel): organization: str | None role: Role contact_type: ContactType - nma_phones: List[str] = [] + incomplete_nma_phones: List[str] = [] emails: List[EmailResponse] = [] phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] things: List[ThingResponse] = [] # List of related things - @field_validator("nma_phones", mode="before") - def make_nma_phone_str(cls, v: list) -> list: + @field_validator("incomplete_nma_phones", mode="before") + def make_incomplete_nma_phone_str(cls, v: list) -> list: if len(v) == 0: return [] else: diff --git a/services/contact_helper.py b/services/contact_helper.py index ac75fba89..942293e70 100644 --- a/services/contact_helper.py +++ b/services/contact_helper.py @@ -39,7 +39,7 @@ def get_db_contacts( joinedload(Contact.thing_associations).joinedload( ThingContactAssociation.thing ), - joinedload(Contact.nma_phones), + joinedload(Contact.incomplete_nma_phones), ) if thing_id: diff --git a/tests/conftest.py b/tests/conftest.py index f7dc15f4d..34944f957 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -352,9 +352,9 @@ def contact(water_well_thing): @pytest.fixture() -def nma_phone_1(contact): +def incomplete_nma_phone_1(contact): with session_ctx() as session: - nma_phone = NMAPhone( + nma_phone = IncompleteNMAPhone( phone_number="9999999", contact_id=contact.id, ) @@ -367,9 +367,9 @@ def nma_phone_1(contact): @pytest.fixture() -def nma_phone_2(contact): +def incomplete_nma_phone_2(contact): with session_ctx() as session: - nma_phone = NMAPhone( + nma_phone = IncompleteNMAPhone( phone_number="8888888", contact_id=contact.id, ) diff --git a/tests/test_contact.py b/tests/test_contact.py index e7a8afc14..6939c704d 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -367,7 +367,9 @@ def test_add_phone_409_contact_not_found(contact): # GET tests ====================================================== -def test_get_contacts(contact, email, address, phone, nma_phone_1, nma_phone_2): +def test_get_contacts( + contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2 +): response = client.get("/contact") assert response.status_code == 200 data = response.json() @@ -382,9 +384,9 @@ def test_get_contacts(contact, email, address, phone, nma_phone_1, nma_phone_2): assert data["items"][0]["release_status"] == contact.release_status assert data["items"][0]["organization"] == contact.organization - assert len(data["items"][0]["nma_phones"]) == 2 - assert sorted(data["items"][0]["nma_phones"]) == sorted( - [nma_phone_1.phone_number, nma_phone_2.phone_number] + assert len(data["items"][0]["incomplete_nma_phones"]) == 2 + assert sorted(data["items"][0]["incomplete_nma_phones"]) == sorted( + [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number] ) assert len(data["items"][0]["emails"]) == 1 @@ -432,7 +434,9 @@ def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing): assert data["items"][0]["id"] == contact.id -def test_get_contact_by_id(contact, email, address, phone, nma_phone_1, nma_phone_2): +def test_get_contact_by_id( + contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2 +): response = client.get(f"/contact/{contact.id}") assert response.status_code == 200 data = response.json() @@ -446,9 +450,9 @@ def test_get_contact_by_id(contact, email, address, phone, nma_phone_1, nma_phon assert data["release_status"] == contact.release_status assert data["organization"] == contact.organization - assert len(data["nma_phones"]) == 2 - assert sorted(data["nma_phones"]) == sorted( - [nma_phone_1.phone_number, nma_phone_2.phone_number] + assert len(data["incomplete_nma_phones"]) == 2 + assert sorted(data["incomplete_nma_phones"]) == sorted( + [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number] ) assert len(data["emails"]) == 1 From 273e13799939f6e0b06664937369d0daf25dac6b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 10:29:11 -0600 Subject: [PATCH 04/12] fix: replace : with - for valid file name --- transfers/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/metrics.py b/transfers/metrics.py index eeb072f17..70c160a90 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -35,7 +35,7 @@ def __init__(self): if not os.path.exists(root): os.mkdir(root) - self.path = root / f"metrics_{datetime.now()}.csv" + self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv" self._writer = csv.writer(self.path.open("a"), delimiter="|") self._writer.writerow(["model", "transferred", "input_count", "cleaned_count"]) From c8e19077f49d9caab1d57c4dbcf31e43cd4a7e51 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 10:37:44 -0600 Subject: [PATCH 05/12] refactor: name incomplete nma phone incomplete_nma_phone in db --- db/contact.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db/contact.py b/db/contact.py index e5413ac4a..59d21e4de 100644 --- a/db/contact.py +++ b/db/contact.py @@ -17,7 +17,7 @@ from sqlalchemy import Integer, ForeignKey, String, UniqueConstraint from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy -from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr from sqlalchemy_utils import TSVectorType from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -129,6 +129,10 @@ class IncompleteNMAPhone(Base, AutoBaseMixin): This is often due to missing area codes, but could be other issues as well. """ + @declared_attr + def __tablename__(self): + return "incomplete_nma_phone" + contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) From 94f157507647c19144080ac55faaab540c22a83c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 11:44:05 -0600 Subject: [PATCH 06/12] refactor: use classes for invalid legacy phones --- transfers/contact_transfer.py | 42 +++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index ad2726d7c..cc557c629 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -17,7 +17,15 @@ from pydantic import ValidationError -from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address +from db import ( + Thing, + Contact, + ThingContactAssociation, + Email, + Phone, + Address, + IncompleteNMAPhone, +) from transfers.logger import logger from transfers.util import ( get_transfers_data_path, @@ -151,7 +159,7 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): contact.emails.append(email) if row.Phone: - phone = _make_phone( + phone, complete = _make_phone( "first", row.OwnerKey, phone_number=row.Phone, @@ -159,10 +167,13 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) if row.CellPhone: - phone = _make_phone( + phone, completeness = _make_phone( "first", row.OwnerKey, phone_number=row.CellPhone, @@ -170,7 +181,10 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) if row.MailingAddress: address = _make_address( @@ -237,7 +251,7 @@ def _add_second_contact(session, row, thing, co_to_org_mapper): contact.emails.append(email) if row.SecondCtctPhone: - phone = _make_phone( + phone, complete = _make_phone( "second", row.OwnerKey, phone_number=row.SecondCtctPhone, @@ -245,7 +259,10 @@ def _add_second_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) # helpers @@ -283,14 +300,15 @@ def _make_phone(first_second, ownerkey, **kw): kw["phone_number"] = kw["phone_number"].strip() phone = CreatePhone(**kw) - return Phone(**phone.model_dump()) + return Phone(**phone.model_dump()), True except ValidationError as e: try: if "phone_number" in kw: - pn = kw.pop("phone_number") - kw["nma_phone_number"] = pn.strip() - phone = CreatePhone(**kw) - return Phone(**phone.model_dump()) + incomplete_phone = IncompleteNMAPhone(phone_number=kw["phone_number"]) + logger.info( + f"Salvaged incomplete phone number for OwnerKey {ownerkey}: {kw['phone_number']}" + ) + return incomplete_phone, False except ValidationError: logger.critical( From a2153b23805264bfaf9416f0e67d00248725a69e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 11:45:35 -0600 Subject: [PATCH 07/12] refactor: rename metric and log files for readability --- transfers/logger.py | 2 +- transfers/metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transfers/logger.py b/transfers/logger.py index 14623c88e..745582c71 100644 --- a/transfers/logger.py +++ b/transfers/logger.py @@ -40,7 +40,7 @@ if not os.path.exists(root): os.mkdir(root) -log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" +log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%H_%M_%S}.log" logging.basicConfig( diff --git a/transfers/metrics.py b/transfers/metrics.py index 70c160a90..6b7da3209 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -35,7 +35,7 @@ def __init__(self): if not os.path.exists(root): os.mkdir(root) - self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv" + self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H-%M-%S')}.csv" self._writer = csv.writer(self.path.open("a"), delimiter="|") self._writer.writerow(["model", "transferred", "input_count", "cleaned_count"]) From 09d7753909fbd4c18b35cdac4486cf618d7e0493 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 11:54:06 -0600 Subject: [PATCH 08/12] fix: make phone_number non-nullable --- db/contact.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/contact.py b/db/contact.py index 59d21e4de..7855814fb 100644 --- a/db/contact.py +++ b/db/contact.py @@ -137,7 +137,7 @@ def __tablename__(self): ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + phone_number: Mapped[str] = mapped_column(String(20), nullable=False) contact: Mapped["Contact"] = relationship( "Contact", back_populates="incomplete_nma_phones", passive_deletes=True @@ -148,7 +148,7 @@ class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + phone_number: Mapped[str] = mapped_column(String(20), nullable=False) phone_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( From 03b7523632c9fee3ae8d31895747a5ec43d7dfdb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 12:02:56 -0600 Subject: [PATCH 09/12] refactor: use _ as time separator --- transfers/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/metrics.py b/transfers/metrics.py index 6b7da3209..fe053697b 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -35,7 +35,7 @@ def __init__(self): if not os.path.exists(root): os.mkdir(root) - self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H-%M-%S')}.csv" + self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H_%M_%S')}.csv" self._writer = csv.writer(self.path.open("a"), delimiter="|") self._writer.writerow(["model", "transferred", "input_count", "cleaned_count"]) From da0bcc73f49f51afebff7b8fcd33ee616b0f975a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 12:11:19 -0600 Subject: [PATCH 10/12] fix: don't log salvaged phone number --- transfers/contact_transfer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index cc557c629..a222801ad 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -305,9 +305,7 @@ def _make_phone(first_second, ownerkey, **kw): try: if "phone_number" in kw: incomplete_phone = IncompleteNMAPhone(phone_number=kw["phone_number"]) - logger.info( - f"Salvaged incomplete phone number for OwnerKey {ownerkey}: {kw['phone_number']}" - ) + logger.info(f"Salvaged incomplete phone number for OwnerKey {ownerkey}") return incomplete_phone, False except ValidationError: From ea5f7f98ea3848ba86d87240830ab65952ad4a0c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 31 Oct 2025 14:35:53 -0600 Subject: [PATCH 11/12] feat: skip second contact if no info --- transfers/contact_transfer.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index a222801ad..18dc041ca 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -102,6 +102,16 @@ def transfer_contacts(session): session.rollback() errors.append({"pointid": row.PointID, "error": e}) try: + if ( + row.SecondFirstName is None + and row.SecondLastName is None + and row.SecondCtctEmail is None + and row.SecondCtctPhone is None + ): + logger.warning( + f"No second contact info for PointID {row.PointID}, skipping." + ) + continue _add_second_contact(session, row, thing, co_to_org_mapper) session.commit() session.flush() From 9a4bd96562d6eb5c8d5beaf6a33632a9a0a2bdc3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 4 Nov 2025 08:32:26 -0700 Subject: [PATCH 12/12] fix: change completeness to complete --- transfers/contact_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 18dc041ca..bd827079a 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -183,7 +183,7 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): contact.incomplete_nma_phones.append(phone) if row.CellPhone: - phone, completeness = _make_phone( + phone, complete = _make_phone( "first", row.OwnerKey, phone_number=row.CellPhone,