diff --git a/db/contact.py b/db/contact.py index a674820ab..7855814fb 100644 --- a/db/contact.py +++ b/db/contact.py @@ -17,7 +17,7 @@ from sqlalchemy import Integer, ForeignKey, String, UniqueConstraint from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy -from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr from sqlalchemy_utils import TSVectorType from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -68,6 +68,11 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): addresses: Mapped[List["Address"]] = relationship( "Address", back_populates="contact", cascade="all, delete, delete-orphan" ) + # One-To-One: A Contact can have one NMA Phone record. + incomplete_nma_phones: Mapped[List["IncompleteNMAPhone"]] = relationship( + "IncompleteNMAPhone", back_populates="contact", cascade="all, delete-orphan" + ) + # One-To-Many: A Contact can grant many Permissions. permissions: Mapped[List["Permission"]] = relationship( "Permission", back_populates="contact", cascade="all, delete, delete-orphan" @@ -118,12 +123,32 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): ) +class IncompleteNMAPhone(Base, AutoBaseMixin): + """ + This table stores data from NM_Aquifer that is not complete and cannot be transferred to the Phone model due to validation issues. + This is often due to missing area codes, but could be other issues as well. + """ + + @declared_attr + def __tablename__(self): + return "incomplete_nma_phone" + + contact_id: Mapped[int] = mapped_column( + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + ) + + phone_number: Mapped[str] = mapped_column(String(20), nullable=False) + + contact: Mapped["Contact"] = relationship( + "Contact", back_populates="incomplete_nma_phones", passive_deletes=True + ) + + class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - nma_phone_number: Mapped[str] = mapped_column(String(20), nullable=True) - phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + phone_number: Mapped[str] = mapped_column(String(20), nullable=False) phone_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( diff --git a/schemas/contact.py b/schemas/contact.py index ca4838409..d43cd4aaf 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -113,7 +113,6 @@ class CreatePhone(BaseCreateModel, ValidatePhone): contact_id: int | None = None # set to None for when made via POST /contact phone_number: str phone_type: PhoneType = "Primary" # Default to 'Primary' - nma_phone_number: str | None = None class CreateAddress(BaseCreateModel): @@ -175,7 +174,6 @@ class PhoneResponse(BaseItemResponse): phone_number: str | None = None phone_type: str # e.g., 'mobile', 'landline', etc. - nma_phone_number: str | None = None class EmailResponse(BaseItemResponse): @@ -210,11 +208,19 @@ class ContactResponse(BaseResponseModel): organization: str | None role: Role contact_type: ContactType + incomplete_nma_phones: List[str] = [] emails: List[EmailResponse] = [] phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] things: List[ThingResponse] = [] # List of related things + @field_validator("incomplete_nma_phones", mode="before") + def make_incomplete_nma_phone_str(cls, v: list) -> list: + if len(v) == 0: + return [] + else: + return [p.phone_number for p in v] + # class ThingContactAssociationResponse(BaseUpdateModel): # """ diff --git a/services/contact_helper.py b/services/contact_helper.py index f50a3b3e1..942293e70 100644 --- a/services/contact_helper.py +++ b/services/contact_helper.py @@ -39,6 +39,7 @@ def get_db_contacts( joinedload(Contact.thing_associations).joinedload( ThingContactAssociation.thing ), + joinedload(Contact.incomplete_nma_phones), ) if thing_id: diff --git a/tests/conftest.py b/tests/conftest.py index fae8aedaa..34944f957 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -351,6 +351,36 @@ def contact(water_well_thing): session.commit() +@pytest.fixture() +def incomplete_nma_phone_1(contact): + with session_ctx() as session: + nma_phone = IncompleteNMAPhone( + phone_number="9999999", + contact_id=contact.id, + ) + session.add(nma_phone) + session.commit() + session.refresh(nma_phone) + yield nma_phone + session.delete(nma_phone) + session.commit() + + +@pytest.fixture() +def incomplete_nma_phone_2(contact): + with session_ctx() as session: + nma_phone = IncompleteNMAPhone( + phone_number="8888888", + contact_id=contact.id, + ) + session.add(nma_phone) + session.commit() + session.refresh(nma_phone) + yield nma_phone + session.delete(nma_phone) + session.commit() + + @pytest.fixture() def address(contact): with session_ctx() as session: diff --git a/tests/test_contact.py b/tests/test_contact.py index 4c25f4473..6939c704d 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -367,7 +367,9 @@ def test_add_phone_409_contact_not_found(contact): # GET tests ====================================================== -def test_get_contacts(contact, email, address, phone): +def test_get_contacts( + contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2 +): response = client.get("/contact") assert response.status_code == 200 data = response.json() @@ -382,6 +384,11 @@ def test_get_contacts(contact, email, address, phone): assert data["items"][0]["release_status"] == contact.release_status assert data["items"][0]["organization"] == contact.organization + assert len(data["items"][0]["incomplete_nma_phones"]) == 2 + assert sorted(data["items"][0]["incomplete_nma_phones"]) == sorted( + [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number] + ) + assert len(data["items"][0]["emails"]) == 1 assert data["items"][0]["emails"][0]["id"] == email.id assert data["items"][0]["emails"][0]["created_at"] == email.created_at.astimezone( @@ -427,7 +434,9 @@ def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing): assert data["items"][0]["id"] == contact.id -def test_get_contact_by_id(contact, email, address, phone): +def test_get_contact_by_id( + contact, email, address, phone, incomplete_nma_phone_1, incomplete_nma_phone_2 +): response = client.get(f"/contact/{contact.id}") assert response.status_code == 200 data = response.json() @@ -441,6 +450,11 @@ def test_get_contact_by_id(contact, email, address, phone): assert data["release_status"] == contact.release_status assert data["organization"] == contact.organization + assert len(data["incomplete_nma_phones"]) == 2 + assert sorted(data["incomplete_nma_phones"]) == sorted( + [incomplete_nma_phone_1.phone_number, incomplete_nma_phone_2.phone_number] + ) + assert len(data["emails"]) == 1 assert data["emails"][0]["id"] == email.id assert data["emails"][0]["created_at"] == email.created_at.astimezone( diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 5d16e73ff..36c7107b7 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -17,7 +17,15 @@ from pydantic import ValidationError -from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address +from db import ( + Thing, + Contact, + ThingContactAssociation, + Email, + Phone, + Address, + IncompleteNMAPhone, +) from transfers.logger import logger from transfers.util import ( get_transfers_data_path, @@ -101,6 +109,16 @@ def transfer_contacts(session): errors.append({"pointid": row.PointID, "error": e}) try: + if ( + row.SecondFirstName is None + and row.SecondLastName is None + and row.SecondCtctEmail is None + and row.SecondCtctPhone is None + ): + logger.warning( + f"No second contact info for PointID {row.PointID}, skipping." + ) + continue _add_second_contact(session, row, thing, co_to_org_mapper) session.commit() # session.flush() @@ -158,7 +176,7 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): contact.emails.append(email) if row.Phone: - phone = _make_phone( + phone, complete = _make_phone( "first", row.OwnerKey, phone_number=row.Phone, @@ -166,10 +184,13 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) if row.CellPhone: - phone = _make_phone( + phone, complete = _make_phone( "first", row.OwnerKey, phone_number=row.CellPhone, @@ -177,7 +198,10 @@ def _add_first_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) if row.MailingAddress: address = _make_address( @@ -244,7 +268,7 @@ def _add_second_contact(session, row, thing, co_to_org_mapper): contact.emails.append(email) if row.SecondCtctPhone: - phone = _make_phone( + phone, complete = _make_phone( "second", row.OwnerKey, phone_number=row.SecondCtctPhone, @@ -252,7 +276,10 @@ def _add_second_contact(session, row, thing, co_to_org_mapper): release_status=release_status, ) if phone: - contact.phones.append(phone) + if complete: + contact.phones.append(phone) + else: + contact.incomplete_nma_phones.append(phone) # helpers @@ -290,14 +317,13 @@ def _make_phone(first_second, ownerkey, **kw): kw["phone_number"] = kw["phone_number"].strip() phone = CreatePhone(**kw) - return Phone(**phone.model_dump()) + return Phone(**phone.model_dump()), True except ValidationError as e: try: if "phone_number" in kw: - pn = kw.pop("phone_number") - kw["nma_phone_number"] = pn.strip() - phone = CreatePhone(**kw) - return Phone(**phone.model_dump()) + incomplete_phone = IncompleteNMAPhone(phone_number=kw["phone_number"]) + logger.info(f"Salvaged incomplete phone number for OwnerKey {ownerkey}") + return incomplete_phone, False except ValidationError: logger.critical( diff --git a/transfers/logger.py b/transfers/logger.py index 14623c88e..745582c71 100644 --- a/transfers/logger.py +++ b/transfers/logger.py @@ -40,7 +40,7 @@ if not os.path.exists(root): os.mkdir(root) -log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" +log_filename = root / f"transfer_{datetime.now():%Y-%m-%dT%H_%M_%S}.log" logging.basicConfig( diff --git a/transfers/metrics.py b/transfers/metrics.py index 424085dc3..ffbd3da31 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -46,7 +46,7 @@ def __init__(self): if not os.path.exists(root): os.mkdir(root) - self.path = root / f"metrics_{datetime.now()}.csv" + self.path = root / f"metrics_{datetime.now().strftime('%Y-%m-%dT%H_%M_%S')}.csv" delimiter = "|" if self.include_errors else "," self._writer = csv.writer(self.path.open("a"), delimiter=delimiter) self._writer.writerow( @@ -116,7 +116,7 @@ def _transducer_metrics( self._write_errors(errors) def _make_metrics(self, name, input_n, cleaned_n, count): - percent_issue = (cleaned_n - count) / cleaned_n * 100 if cleaned_n == 0 else 0 + percent_issue = (cleaned_n - count) / cleaned_n * 100 if cleaned_n != 0 else 0 return [name, input_n, cleaned_n, count, percent_issue] def _handle_metrics(