Transfer refactor for future reusability improvements

.gitignore

@@ -6,6 +6,13 @@ dist/
 wheels/
 *.egg-info
 
+# Test coverage reports
+*.cover
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
 # Virtual environments
 .venv
 requirements.txt
@@ -25,11 +32,11 @@ launcher.sh
 gcs_credentials.json
 transfers/data/assets*
 transfers/data/nma_csv_cache/*
+transfers/data/*.csv
 transfers/transfer*.log
 transfer*.log
 transfers/data/nma_csv_cache/*
 !transfers/data/nma_csv_cache/.gitkeep
-tests/features/*.feature
 transfers/metrics/*
 transfers/logs/*
 run_bdd-local.sh

.pre-commit-config.yaml

@@ -25,6 +25,8 @@ repos:
         types: [python] # Specify relevant file types for your tests
         pass_filenames: false
         always_run: true
+        args:
+          - -x
 
   # - repo: https://github.com/pre-commit/mirrors-mypy
   #   rev: v1.10.0  # Use the latest stable version or pin to your preference

core/enums.py

@@ -24,7 +24,9 @@
 )
 CasingMaterial: type[Enum] = build_enum_from_lexicon_category("casing_material")
 CollectionMethod: type[Enum] = build_enum_from_lexicon_category("collection_method")
-ConstructionMethod: type[Enum] = build_enum_from_lexicon_category("construction_method")
+WellConstructionMethod: type[Enum] = build_enum_from_lexicon_category(
+    "well_construction_method"
+)
 ContactType: type[Enum] = build_enum_from_lexicon_category("contact_type")
 CoordinateMethod: type[Enum] = build_enum_from_lexicon_category("coordinate_method")
 WellPurpose: type[Enum] = build_enum_from_lexicon_category("well_purpose")
@@ -68,8 +70,14 @@
 Vertical_datum: type[Enum] = build_enum_from_lexicon_category("vertical_datum")
 ScreenType: type[Enum] = build_enum_from_lexicon_category("screen_type")
 SensorType: type[Enum] = build_enum_from_lexicon_category("sensor_type")
+WellPumpType: type[Enum] = build_enum_from_lexicon_category("well_pump_type")
+PermissionType: type[Enum] = build_enum_from_lexicon_category("permission_type")
 GroupType: type[Enum] = build_enum_from_lexicon_category("group_type")
 MonitoringFrequency: type[Enum] = build_enum_from_lexicon_category(
     "monitoring_frequency"
 )
+AquiferType: type[Enum] = build_enum_from_lexicon_category("aquifer_type")
+GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale")
+Lithology: type[Enum] = build_enum_from_lexicon_category("lithology")
+FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code")
 # ============= EOF =============================================

db/__init__.py

@@ -33,7 +33,7 @@
 from db.notes import *
 from db.observation import *
 from db.parameter import *
-from db.permission import *
+from db.permission_history import *
 from db.publication import *
 from db.regulatory_limit import *
 from db.sample import *
@@ -43,6 +43,11 @@
 from db.transducer import *
 from db.measuring_point_history import *
 from db.data_provenance import *
+from db.aquifer_system import *
+from db.geologic_formation import *
+from db.thing_aquifer_association import *
+from db.thing_geologic_formation_association import *
+from db.aquifer_type import *
 
 from sqlalchemy import (
     func,

db/aquifer_system.py

@@ -0,0 +1,84 @@
+"""
+SQLAlchemy model for the AquiferSystem table.
+
+This is a master reference table for aquifer systems and hydrogeologic units.
+"""
+
+from typing import List, TYPE_CHECKING
+
+from sqlalchemy import Text, Index
+from sqlalchemy.orm import relationship, Mapped, mapped_column
+from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy
+from geoalchemy2 import Geometry
+
+from db.base import Base, AutoBaseMixin, ReleaseMixin
+from db.lexicon import lexicon_term
+
+from constants import SRID_WGS84
+
+if TYPE_CHECKING:
+    from db.thing import WellScreen, ThingAquiferAssociation, Thing
+    from db.aquifer_type import AquiferType
+
+
+class AquiferSystem(Base, AutoBaseMixin, ReleaseMixin):
+    __versioned__ = {}
+
+    name: Mapped[str] = mapped_column(
+        nullable=False,
+        unique=True,
+        comment="The full, human-readable name of the aquifer system (e.g., 'Ogallala Aquifer').",
+    )
+    description: Mapped[str] = mapped_column(
+        Text,
+        nullable=True,
+        comment="A detailed description of the aquifer system, its characteristics, and its significance.",
+    )
+    # Lexicon terms were retrieved from NMAquifer's 'LU_AquiferType' table.
+    primary_aquifer_type: Mapped[str] = lexicon_term(
+        nullable=False,
+        comment="A controlled vocabulary field to classify the aquifer system as a whole (e.g., 'Unconfined', 'Confined', 'Perched').",
+    )
+    geographic_scale: Mapped[str] = lexicon_term(
+        nullable=True,
+        comment="A controlled vocabulary field to classify the aquifer's geographic scale (e.g., 'Major', 'Regional', 'Local').",
+    )
+    boundary: Mapped[Geometry] = mapped_column(
+        Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True),
+        nullable=True,
+        comment="A spatial representation of the aquifer system's boundary.",
+    )
+    # Hierarchical relationship fields (may be implemented in future iterations)
+    # Example: High Plains Aquifer (parent) contains Ogallala Aquifer (child)
+    # parent_id = Column(Integer, ForeignKey('aquifer_system.id'))
+    # parent = relationship('AquiferSystem', remote_side=[id], backref='subsystems')
+
+    # --- Relationships ---
+    # One-To-Many: An AquiferSystem can be associated with many wells (Things) via the ThingAquiferAssociation join table.
+    thing_associations: Mapped[List["ThingAquiferAssociation"]] = relationship(
+        "ThingAquiferAssociation",
+        back_populates="aquifer_system",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+    )
+
+    # One-To-Many: An AquiferSystem can be the target for many individual WellScreens.
+    well_screens: Mapped[List["WellScreen"]] = relationship(
+        "WellScreen",
+        back_populates="aquifer_system",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+    )
+
+    # --- Association Proxies ---
+    # Proxy to directly access Things (wells) associated with this AquiferSystem.
+    things: AssociationProxy[List["Thing"]] = association_proxy(
+        "thing_associations", "thing"
+    )
+    # Proxy to directly access all AquiferTypes associated with this AquiferSystem.
+    aquifer_types: AssociationProxy[List["AquiferType"]] = association_proxy(
+        "thing_associations", "aquifer_types"
+    )
+
+    # --- Table Arguments ---
+    __table_args__ = (Index("ix_aquifersystem_name", "name"),)

db/aquifer_type.py

@@ -0,0 +1,58 @@
+"""
+SQLAlchemy model for the AquiferType table.
+
+This table stores the specific aquifer characteristics/types associated with
+a Thing-AquiferSystem relationship. It allows capturing that a single aquifer
+can have multiple characteristics simultaneously.
+
+Example:
+    A well in the "Ogallala" aquifer might tap portions that are both
+    "Fractured" AND "Confined". This would create:
+    - One AquiferSystem: "Ogallala"
+    - One ThingAquiferAssociation: linking well to Ogallala
+    - Two AquiferType records: "Fractured" and "Confined"
+"""
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import relationship, Mapped, mapped_column
+
+from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term
+
+if TYPE_CHECKING:
+    from db.thing_aquifer_association import ThingAquiferAssociation
+
+
+class AquiferType(Base, AutoBaseMixin, ReleaseMixin):
+    """
+    Represents the specific aquifer types/characteristics for a
+    Thing-AquiferSystem association.
+
+    This allows modeling the fact that:
+    - A single aquifer can have multiple characteristics
+    - Different wells may tap different characteristics of the same aquifer
+    - Characteristics are attributes of the relationship, not the aquifer itself
+
+    Fields from WellData CSV:
+        - AquiferType: May contain multiple codes (e.g., "FC" = Fractured + Confined)
+        - Each code becomes a separate AquiferType record
+    """
+
+    # --- Columns ---
+    thing_aquifer_association_id: Mapped[int] = mapped_column(
+        ForeignKey("thing_aquifer_association.id", ondelete="CASCADE"),
+        nullable=False,
+        comment="Links to the Thing-Aquifer association this type describes.",
+    )
+    aquifer_type: Mapped[str] = lexicon_term(
+        nullable=False,
+        comment="Controlled vocabulary for aquifer hydrologic properties. "
+        "Examples: 'Unconfined', 'Confined', 'Perched', 'Fractured', 'Unconsolidated'.",
+    )
+
+    # --- Relationships ---
+    # Many-to-One: Multiple aquifer types can belong to one association
+    thing_aquifer_association: Mapped["ThingAquiferAssociation"] = relationship(
+        "ThingAquiferAssociation", back_populates="aquifer_types"
+    )

db/contact.py

@@ -26,7 +26,7 @@
     from db.field import FieldEventParticipant, FieldEvent
     from db.thing import Thing
     from db.publication import Author, AuthorContactAssociation
-    from db.permission import Permission
+    from db.permission_history import PermissionHistory
 
 
 class ThingContactAssociation(Base, AutoBaseMixin):
@@ -74,8 +74,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin):
     )
 
     # One-To-Many: A Contact can grant many Permissions.
-    permissions: Mapped[List["Permission"]] = relationship(
-        "Permission", back_populates="contact", cascade="all, delete, delete-orphan"
+    permissions: Mapped[List["PermissionHistory"]] = relationship(
+        "PermissionHistory",
+        back_populates="contact",
+        cascade="all, delete, delete-orphan",
     )
     # One-To-Many: A Contact can be associated with many Authors (in Publications).
     author_associations: Mapped[List["AuthorContactAssociation"]] = relationship(

db/data_provenance.py

@@ -19,7 +19,7 @@
 from sqlalchemy import Integer, Index, and_
 from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign
 
-from db.base import Base, AutoBaseMixin, ReleaseMixin, pascal_to_snake
+from db.base import Base, AutoBaseMixin, ReleaseMixin
 
 from db import lexicon_term
 
@@ -53,9 +53,13 @@ class DataProvenance(AutoBaseMixin, ReleaseMixin, Base):
     )
     # Values from the following NMAquifer tables are included as `origin_source` terms in the lexicon:
     # 'LU_DataSource', 'LU_Depth_CompletionSource'.
-    origin_source: Mapped[str] = lexicon_term(
+    origin_type: Mapped[str] = lexicon_term(
         nullable=True,
-        comment="Indicates the origin source of the data (e.g'Driller's Log', 'Well Report'.",
+        comment="Indicates the type of origin the data (e.g'Driller's Log', 'Well Report'.",
+    )
+    origin_source: Mapped[str] = mapped_column(
+        nullable=True,
+        comment="The specific source of the data (e.g., 'J. Brown Thesis, \"I like APIs\", Pomona College, 1994').",
     )
     # Values from the following NMAquifer tables are included as `collection_method` terms in the lexicon:
     # 'LU_AltitudeMethod','LU_CoordinateMethod'.
@@ -116,7 +120,7 @@ def data_provenance(cls):
             "DataProvenance",
             primaryjoin=and_(
                 cls.id == foreign(DataProvenance.target_id),
-                DataProvenance.target_table == pascal_to_snake(cls.__name__),
+                DataProvenance.target_table == cls.__tablename__,
             ),
             lazy="selectin",
             viewonly=True,

db/geologic_formation.py

@@ -0,0 +1,82 @@
+"""
+SQLAlchemy model for the GeologicFormation table.
+
+This table is a master reference table for geologic formations. Its purpose is to store definitions and descriptions
+of various geologic formations that can be referenced by other tables in the database.
+"""
+
+from typing import List, TYPE_CHECKING
+
+from sqlalchemy import Text, Index
+from sqlalchemy.orm import relationship, Mapped, mapped_column
+from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy
+from geoalchemy2 import Geometry
+
+from db.base import Base, AutoBaseMixin, ReleaseMixin
+from db.lexicon import lexicon_term
+
+from constants import SRID_WGS84
+
+if TYPE_CHECKING:
+    from db.thing import Thing, WellScreen
+    from db.thing_geologic_formation_association import (
+        ThingGeologicFormationAssociation,
+    )
+
+
+class GeologicFormation(Base, AutoBaseMixin, ReleaseMixin):
+    __versioned__ = {}
+
+    # TODO: Let the API map formation codes to names using a formations.json file that can be periodically updated
+    #  from the authoritative source (.e.g USGS). A placeholder `formations.json` file had been added to the `core`
+    #  directory.
+    # name: Mapped[str] = mapped_column(
+    #     nullable=False,
+    #     unique=True,
+    #     comment="The full, human-readable name of the geologic formation (e.g., 'Navajo Sandstone').",
+    # )
+    formation_code: Mapped[str] = lexicon_term(
+        nullable=True,
+        unique=True,
+        comment="A short code or abbreviation for the geologic formation (e.g., '120ELRT').",
+    )
+    description: Mapped[str] = mapped_column(
+        Text,
+        nullable=True,
+        comment="A detailed description of the geologic formation, its characteristics, and its significance.",
+    )
+    # TODO: Implement controlled vocabularies for `lithology` using NMAquifer's 'LU_Lithology' table.
+    #  This should be implemented after AMMP reviews and cleans up their formation terms and codes.
+    lithology: Mapped[str] = lexicon_term(
+        nullable=True,
+        comment="A controlled vocabulary for the primary, dominant rock type"
+        "(e.g., 'Tuff', 'Sandstone', 'Alluvium', 'Shale').",
+    )
+    boundary: Mapped[Geometry] = mapped_column(
+        Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True),
+        nullable=True,
+        comment="A spatial representation of the geologic formation's extent.",
+    )
+
+    # --- Relationships ---
+    # One-To-Many (Association Object): A GeologicFormation can be associated with many Things (e.g., wells) via the
+    # ThingGeologicFormationAssociation join table.
+    thing_associations: Mapped[List["ThingGeologicFormationAssociation"]] = (
+        relationship(
+            "ThingGeologicFormationAssociation",
+            back_populates="geologic_formation",
+            cascade="all, delete-orphan",
+            passive_deletes=True,
+        )
+    )
+    # One-To-Many: A GeologicFormation can have many physical WellScreens installed in it.
+    well_screens: Mapped[List["WellScreen"]] = relationship(
+        "WellScreen", back_populates="geologic_formation", passive_deletes=True
+    )
+
+    # --- Association Proxies ---
+    # Provides direct access to Things (wells) that penetrate this formation.
+    things: AssociationProxy["Thing"] = association_proxy("thing_associations", "thing")
+
+    # --- Table Arguments ---
+    __table_args__ = (Index("ix_geologicformation_formation_code", "formation_code"),)

db/location.py

@@ -23,6 +23,7 @@
     String,
     ForeignKey,
     DateTime,
+    Date,
     func,
     Text,
 )
@@ -61,6 +62,18 @@ class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin, DataProvenanceMixi
     nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True)
     nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True)
 
+    # --- AMPAPI Date Fields (Migration-Only, Read-Only Post-Migration) ---
+    nma_date_created: Mapped[datetime.date] = mapped_column(
+        Date,
+        nullable=True,
+        comment="Original AMPAPI DateCreated (read-only, populated only during migration)",
+    )
+    nma_site_date: Mapped[datetime.date] = mapped_column(
+        Date,
+        nullable=True,
+        comment="Original AMPAPI SiteDate (read-only, populated only during migration)",
+    )
+
     # --- Relationship Definitions ---
     thing_associations: Mapped[list["LocationThingAssociation"]] = relationship(
         back_populates="location", cascade="all, delete-orphan"