From 7d97f478ec639ea64d55a17a418fb5406de81e17 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Fri, 17 Oct 2025 09:42:18 -0600 Subject: [PATCH 01/24] feat: Create new `Notes` table A polymorphic Notes table was needed to store all unstructured notes. This commit creates the necessary polymorphic Notes table. Its purpose is to store all unstructured notes, categorized by a note_type. It should be used when a record might need more than one note, when the notes need to be categorized, or when you need the ability to search across all notes in the system. This is different from a dedicated notes field on a specific table, which should be used to store a single, intrinsic attribute of the record itself. --- db/__init__.py | 1 + db/notes.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 db/notes.py diff --git a/db/__init__.py b/db/__init__.py index 7a7b20fa6..82c6637a4 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -30,6 +30,7 @@ from db.group import * from db.lexicon import * from db.location import * +from db.notes import * from db.observation import * from db.parameter import * from db.permission import * diff --git a/db/notes.py b/db/notes.py new file mode 100644 index 000000000..0125ee2b5 --- /dev/null +++ b/db/notes.py @@ -0,0 +1,81 @@ +""" +SQLAlchemy model for the Notes table. + +This is a polymorphic table for storing all unstructured notes, categorized by +a note_type. + +The Notes table should be used when a record might need more than one note, +when the notes need to be categorized, or when you need the ability to +search across all notes in the system. This is different from a dedicated +notes field on a specific table, which should be used to store a single, +intrinsic attribute of the record itself. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, Text, and_, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.thing import Thing + from db.location import Location + + +class Notes(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a single, categorized note that can be attached to various + parent objects throughout the database. + """ + + # --- Polymorphic Columns --- + notable_id: Mapped[int] = mapped_column( + Integer, + nullable=False, + comment="The ID of the parent record this note is about (e.g., a `thing_id`, `location_id`, etc).", + ) + # TODO: create new lexicon category and lexicon terms for notable_type + notable_type: Mapped[str] = lexicon_term( + nullable=False, + comment="The type of the note associated with this record.", + ) + + # --- Columns --- + # TODO: create new lexicon category and lexicon terms for note_type + note_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field that defines the specific category of the note (e.g. 'Access Instructions`, ", + ) + content: Mapped[str] = mapped_column(Text, nullable=False) + + # --- Polymorphic Parent Relationships (Internal) --- + # These are viewonly relationships used by the 'target' property below. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin=and_(foreign(notable_id) == Thing.id, notable_type == "Thing"), + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin=and_( + foreign(notable_id) == Location.id, notable_type == "Location" + ), + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record without + needing to check the 'notable_type' field manually. + """ + return getattr(self, f"_{self.notable_type.lower()}_target") + + # --- Table Arguments --- + # A composite index to optimize retrieval of all note records for a specific parent object. + + __table_args__ = (Index("ix_notes_polymorphic_link", "notable_id", "notable_type"),) From 8ab40ff30d90717bb73b3ebe309e9dc18df42d2b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 20 Oct 2025 11:23:14 -0600 Subject: [PATCH 02/24] doc: Updated `Notes` model docstring for clarity. --- db/notes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/notes.py b/db/notes.py index 0125ee2b5..fee5e5cf3 100644 --- a/db/notes.py +++ b/db/notes.py @@ -7,8 +7,8 @@ The Notes table should be used when a record might need more than one note, when the notes need to be categorized, or when you need the ability to search across all notes in the system. This is different from a dedicated -notes field on a specific table, which should be used to store a single, -intrinsic attribute of the record itself. +notes field on a specific table, which should be used to store a simple, +single-purpose attribute of the record itself. """ from typing import TYPE_CHECKING From 7aff37db6fbd47cb72e26705634aa9919e16bb7f Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 20 Oct 2025 11:57:56 -0600 Subject: [PATCH 03/24] feat: Add 'notable_type' category and values This commit introduces a new category, `notable_type`, to the `lexicon.json` file. This category will serve as the central, authoritative source for the controlled vocabulary used in the polymorphic `noteable_type` columns --- core/lexicon.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/lexicon.json b/core/lexicon.json index e1ffd9cf4..2ac67c5c6 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -23,6 +23,7 @@ {"name": "limit_type", "description": null}, {"name": "measurement_method", "description": null}, {"name": "monitoring_status", "description": null}, + {"name": "noteable_type", "description": null}, {"name": "parameter_name", "description": null}, {"name": "organization", "description": null}, {"name": "parameter_type", "description": null}, @@ -525,6 +526,11 @@ {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, - {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"} + {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, + {"categories": ["noteable_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, + {"categories": ["noteable_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, + {"categories": ["noteable_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, + {"categories": ["noteable_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, + {"categories": ["noteable_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."} ] } \ No newline at end of file From 1c9db25d0406de60fb25600730a4eadc0d12ea93 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 20 Oct 2025 11:59:45 -0600 Subject: [PATCH 04/24] doc: Remove TODO comments Removed the TODO comments about updating the lexicon file with 'notable_type' category and values. They have been added. --- db/notes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/db/notes.py b/db/notes.py index fee5e5cf3..4fbfb1cec 100644 --- a/db/notes.py +++ b/db/notes.py @@ -35,14 +35,12 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="The ID of the parent record this note is about (e.g., a `thing_id`, `location_id`, etc).", ) - # TODO: create new lexicon category and lexicon terms for notable_type notable_type: Mapped[str] = lexicon_term( nullable=False, comment="The type of the note associated with this record.", ) # --- Columns --- - # TODO: create new lexicon category and lexicon terms for note_type note_type: Mapped[str] = lexicon_term( nullable=False, comment="A controlled vocabulary field that defines the specific category of the note (e.g. 'Access Instructions`, ", From 8936e1aec345b4ff15b93dff1f0dcdb8b37b7e5d Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 20 Oct 2025 12:32:28 -0600 Subject: [PATCH 05/24] feat: Add polymorphic NotesMixin to support multi-category notes Added NotesMixin to db/base.py that provides polymorphic note capabilities to any model that inherits from it. The mixin offers: - A polymorphic one-to-many relationship to the Notes table using the notable_id/notable_type pattern - Efficient loading with selectin strategy to prevent N+1 query issues - A convenient add_note() factory method for creating new notes with proper polymorphic associations - Clean separation between read (relationship) and write (method) operations This mixin enables consistent note-taking across multiple entity types while maintaining type safety and relationship integrity. --- db/base.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/db/base.py b/db/base.py index ba2a45be8..3414842bb 100644 --- a/db/base.py +++ b/db/base.py @@ -54,8 +54,14 @@ ) from sqlalchemy_searchable import make_searchable from sqlalchemy_continuum import make_versioned +from sqlalchemy.inspection import inspect import re +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from db.notes import Notes + make_versioned() @@ -210,6 +216,64 @@ def permissions(self): ) +class NotesMixin: + """ + Mixin for models that can have multiple types or categories of notes. + It automatically creates a polymorphic One-to-Many relationship to the + Notes table. + """ + + @declared_attr + def notes(cls): + """ + The high-performance, declarative relationship for reading notes. + This provides a polymorphic one-to-many link to the Notes table. + + PERFORMANCE NOTE: Use with `selectinload` in queries to prevent the + N+1 query problem when accessing notes for multiple parent objects. + """ + # Dynamically get the primary key column name of the inheriting class + pk_name = inspect(cls).primary_key[0].name + + return relationship( + "Notes", + primaryjoin=f"and_({cls.__name__}.{pk_name}==Notes.notable_id, " + f"Notes.notable_type=='{cls.__name__}')", + lazy="selectin", # A good default for eager loading on demand + viewonly=True, + ) + + def add_note(self, content: str, note_type: str, created_by: str) -> "Notes": + """ + A convenient factory method to create a new Note associated with this object. + This provides a clean, object-oriented API for writing. + + NOTE: This method creates and returns a new Note object but does *not* add + it to the database session. The caller is responsible for session management. + + Args: + content: The text content of the note. + note_type: The categorized type of the note (from a controlled vocabulary). + created_by: The user or process creating the note. + + Returns: + A new, unsaved Notes object linked to this parent. + """ + # This import is inside the method to avoid circular import issues at runtime. + from db.notes import Notes + + # Dynamically get the primary key value of this specific instance + pk_name = inspect(self.__class__).primary_key[0].name + pk_value = getattr(self, pk_name) + + return Notes( + content=content, + note_type=note_type, + notable_id=pk_value, + notable_type=self.__class__.__name__, + ) + + class User(Base): """Represents a user in the system.""" From d0a7b408890d444ed6c516e2e1abd31c9aba5efd Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 20 Oct 2025 12:49:14 -0600 Subject: [PATCH 06/24] feat: Integrate NotesMixin into Location and Thing models Added the NotesMixin to both Location and Thing models to enable polymorphic note capabilities. This allows: - Associating multiple categorized notes with locations and things - Consistent note handling across entity types - Type-safe access to notes through relationship loading - Efficient loading of notes with selectin strategy The NotesMixin provides both the notes relationship and add_note() method for these models, enabling a clean API for both reading and creating notes. --- db/location.py | 5 +++-- db/thing.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/db/location.py b/db/location.py index b1113eaad..292e2dc1b 100644 --- a/db/location.py +++ b/db/location.py @@ -31,14 +31,14 @@ from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from constants import SRID_WGS84 -from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, NotesMixin from db.lexicon import lexicon_term if TYPE_CHECKING: from db.thing import Thing -class Location(Base, AutoBaseMixin, ReleaseMixin): +class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): __versioned__ = {} nma_pk_location: Mapped[UUID] = mapped_column(String(36), nullable=True) @@ -56,6 +56,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): county: Mapped[str] = mapped_column(String(100), nullable=True) state: Mapped[str] = mapped_column(String(100), nullable=True) quad_name: Mapped[str] = mapped_column(String(100), nullable=True) + # TODO: remove this 'notes' field in favor of using the polymorphic Notes table. Did not remove it yet to avoid breaking existing data model. notes: Mapped[str] = mapped_column(Text, nullable=True) nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True) nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True) diff --git a/db/thing.py b/db/thing.py index 13ce81bbb..ea4dc9efc 100644 --- a/db/thing.py +++ b/db/thing.py @@ -26,6 +26,7 @@ ReleaseMixin, StatusHistoryMixin, PermissionMixin, + NotesMixin, ) from typing import List, TYPE_CHECKING @@ -39,7 +40,9 @@ from db.group import Group, GroupThingAssociation -class Thing(Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMixin): +class Thing( + Base, AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, PermissionMixin, NotesMixin +): """ Represents a physical object of interest being monitored (e.g., a well). Stores static, core attributes of the physical installation. From c905273cf68b5db0666df8f260955b398cdc59c1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 09:42:39 -0600 Subject: [PATCH 07/24] feat: Add Notes schema and polymorphic relationships to Thing and Location models - Created new Pydantic schema for the Notes model - Added polymorphic relationship `notes: List[NoteResponse] = []` to Thing and Location schemas for flexible data modeling. --- schemas/location.py | 14 ++++++++++++- schemas/notes.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ schemas/thing.py | 4 ++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 schemas/notes.py diff --git a/schemas/location.py b/schemas/location.py index 9340b8cac..9f47be5d8 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -18,6 +18,8 @@ from pydantic import BaseModel, field_validator from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel +from schemas.notes import NoteResponse +from typing import List from services.validation.geospatial import validate_wkt_geometry @@ -40,6 +42,8 @@ class CreateLocation(BaseCreateModel, ValidateLocation): """ # name: str | None = None + # TODO: remove `notes` field once polymorphic Notes are successfully implemented. + # I didn't want to remove yet in case it breaks something with the transfer. notes: str | None = None point: str # point is required and should be in WKT format elevation: float @@ -66,6 +70,9 @@ class LocationResponse(BaseResponseModel): """ # name: str | None + # TODO: remove `notes` field once polymorphic Notes are successfully implemented. + # I didn't want to remove yet in case it breaks something with the transfer. + notes: str | None = None notes: str | None point: str elevation: float | None @@ -80,6 +87,9 @@ class LocationResponse(BaseResponseModel): county: str | None quad_name: str | None + # The new relationship to the polymorphic Notes table + notes: List[NoteResponse] = [] + @field_validator("point", mode="before") def point_to_wkt(cls, value): if isinstance(value, WKBElement): @@ -104,10 +114,12 @@ class GroupLocationResponse(BaseResponseModel): # -------- UPDATE ---------- class UpdateLocation(BaseUpdateModel, ValidateLocation): """ - Schema for updating a location. + Schema for updating a location. Notes are managed via the polymorphic Notes table. """ # name: str | None = None + # TODO: remove `notes` field once polymorphic Notes are successfully implemented. + # I didn't want to remove yet in case it breaks something with the transfer. notes: str | None = None point: str | None = None elevation: float | None = None diff --git a/schemas/notes.py b/schemas/notes.py new file mode 100644 index 000000000..9ec50b2ff --- /dev/null +++ b/schemas/notes.py @@ -0,0 +1,48 @@ +""" +Pydantic models for the Notes table. +""" + +from pydantic import BaseModel +from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel + +# -------- BASE SCHEMA: ---------- +""" +Defines the core, shared attributes of a Note for reuse. +""" + + +class BaseNote(BaseModel): + note_type: str + content: str + + +# -------- CREATE ---------- +class CreateNote(BaseCreateModel, BaseNote): + # TODO: this was a suggestion by AI, but based on our other schemas it + # seems like more should be added here... + """ + Schema for creating a new Note. The parent object's ID and type will be + taken from the URL path, not the request body. + """ + pass + + +# -------- RESPONSE ---------- +class NoteResponse(BaseResponseModel, BaseNote): + """ + Response schema for Note details. + """ + + note_id: int + notable_id: int + notable_type: str + + +# -------- UPDATE ---------- +class UpdateNote(BaseUpdateModel): + """ + Schema for updating an existing Note. All fields are optional + """ + + note_type: str | None = None + content: str | None = None diff --git a/schemas/thing.py b/schemas/thing.py index 6bf0befc1..a3014cd98 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -19,6 +19,7 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse +from schemas.notes import NoteResponse # -------- VALIDATE ---------- @@ -142,6 +143,9 @@ class BaseThingResponse(BaseResponseModel): current_location: LocationResponse | None first_visit_date: PastDate | None + # The new relationship to the polymorphic Notes table + notes: List[NoteResponse] = [] + class WellResponse(BaseThingResponse): """ From 550accec40c7c16aa3070accb8570612f95ef30d Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 10:13:07 -0600 Subject: [PATCH 08/24] fix: Resolve circular import in Notes model - Resolved SQLAlchemy inspection error caused by circular import between Notes and Thing/Location models - Updated relationship definitions in notes.py to use string-based SQL expressions instead of direct class references - Maintained TYPE_CHECKING imports for proper type hinting while avoiding runtime circular dependencies --- db/notes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db/notes.py b/db/notes.py index 4fbfb1cec..2793dfa25 100644 --- a/db/notes.py +++ b/db/notes.py @@ -13,8 +13,8 @@ from typing import TYPE_CHECKING -from sqlalchemy import Integer, Text, and_, Index -from sqlalchemy.orm import relationship, Mapped, mapped_column, foreign +from sqlalchemy import Integer, Text, Index +from sqlalchemy.orm import relationship, Mapped, mapped_column from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -51,14 +51,12 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): # These are viewonly relationships used by the 'target' property below. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin=and_(foreign(notable_id) == Thing.id, notable_type == "Thing"), + primaryjoin="and_(foreign(Notes.notable_id) == Thing.id, Notes.notable_type == 'Thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin=and_( - foreign(notable_id) == Location.id, notable_type == "Location" - ), + primaryjoin="and_(foreign(Notes.notable_id) == Location.id, Notes.notable_type == 'Location')", viewonly=True, ) From f18b715ab8365911b8e7074f694944dc0100f28b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 13:32:31 -0600 Subject: [PATCH 09/24] fix: Resolve undefined pk_value in NotesMixin.add_note method - Fixed the NotesMixin.add_note method by replacing the undefined pk_value reference with self.id - This resolves the "Unresolved reference 'pk_value'" error in base.py - Ensures proper note creation through the convenient factory method - Complements previous circular import fixes in the polymorphic relationship structure The change maintains the intended functionality of creating notes associated with model instances while fixing a reference error that was preventing proper execution. --- db/base.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/db/base.py b/db/base.py index 3414842bb..82cb66323 100644 --- a/db/base.py +++ b/db/base.py @@ -54,7 +54,6 @@ ) from sqlalchemy_searchable import make_searchable from sqlalchemy_continuum import make_versioned -from sqlalchemy.inspection import inspect import re from typing import TYPE_CHECKING @@ -232,14 +231,14 @@ def notes(cls): PERFORMANCE NOTE: Use with `selectinload` in queries to prevent the N+1 query problem when accessing notes for multiple parent objects. """ - # Dynamically get the primary key column name of the inheriting class - pk_name = inspect(cls).primary_key[0].name + # All parent tables use 'id' as their primary key. + pk_name = "id" return relationship( "Notes", - primaryjoin=f"and_({cls.__name__}.{pk_name}==Notes.notable_id, " + primaryjoin=f"and_({cls.__name__}.{pk_name}==foreign(Notes.notable_id), " f"Notes.notable_type=='{cls.__name__}')", - lazy="selectin", # A good default for eager loading on demand + lazy="selectin", viewonly=True, ) @@ -247,29 +246,14 @@ def add_note(self, content: str, note_type: str, created_by: str) -> "Notes": """ A convenient factory method to create a new Note associated with this object. This provides a clean, object-oriented API for writing. - - NOTE: This method creates and returns a new Note object but does *not* add - it to the database session. The caller is responsible for session management. - - Args: - content: The text content of the note. - note_type: The categorized type of the note (from a controlled vocabulary). - created_by: The user or process creating the note. - - Returns: - A new, unsaved Notes object linked to this parent. """ # This import is inside the method to avoid circular import issues at runtime. from db.notes import Notes - # Dynamically get the primary key value of this specific instance - pk_name = inspect(self.__class__).primary_key[0].name - pk_value = getattr(self, pk_name) - return Notes( content=content, note_type=note_type, - notable_id=pk_value, + notable_id=self.id, notable_type=self.__class__.__name__, ) From 6425d06f479738aebcc9a8b3ade68d64f5817c13 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 14:56:59 -0600 Subject: [PATCH 10/24] refactor: Update 'noteable_type' category to `note_type` The lexicon incorrectly categorized a type of note as `noteable_type`. It has been corrected to `note_type` This resolves the error where tests for the location model were unable to locate the proper "note_type". --- core/lexicon.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 2ac67c5c6..07dbfd8f1 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -23,7 +23,7 @@ {"name": "limit_type", "description": null}, {"name": "measurement_method", "description": null}, {"name": "monitoring_status", "description": null}, - {"name": "noteable_type", "description": null}, + {"name": "note_type", "description": null}, {"name": "parameter_name", "description": null}, {"name": "organization", "description": null}, {"name": "parameter_type", "description": null}, @@ -527,10 +527,10 @@ {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"}, - {"categories": ["noteable_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, - {"categories": ["noteable_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, - {"categories": ["noteable_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, - {"categories": ["noteable_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, - {"categories": ["noteable_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."} + {"categories": ["note_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, + {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, + {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, + {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, + {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."} ] } \ No newline at end of file From e814635e7259ea017a6bba5b5fc48af087efe829 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 15:10:46 -0600 Subject: [PATCH 11/24] fix: Standardize location notes as List[NoteResponse] across schemas - Comment out string-type notes fields from CreateLocation and UpdateLocation schemas - Ensure LocationResponse consistently uses notes as List[NoteResponse] - Fix validation error where API was receiving string notes but expecting list type - Complete transition to polymorphic notes relationship for location entities - Update test data to create proper Note objects instead of using string values This resolves the ResponseValidationError where the API expected notes to be a list but received a string value ('these are some test notes'). --- schemas/location.py | 22 +++++++++------------- schemas/thing.py | 1 - tests/test_location.py | 26 +++++++++++++++++++++----- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/schemas/location.py b/schemas/location.py index 9f47be5d8..6f60b648a 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -42,9 +42,9 @@ class CreateLocation(BaseCreateModel, ValidateLocation): """ # name: str | None = None - # TODO: remove `notes` field once polymorphic Notes are successfully implemented. - # I didn't want to remove yet in case it breaks something with the transfer. - notes: str | None = None + # TODO: AI suggested managing notes via a separate /locations/{id}/notes endpoint. + # I don't know if we want to do that, but am leaving this comment for future reference. + # notes: str | None = None point: str # point is required and should be in WKT format elevation: float release_status: str | None = "draft" @@ -70,10 +70,9 @@ class LocationResponse(BaseResponseModel): """ # name: str | None - # TODO: remove `notes` field once polymorphic Notes are successfully implemented. - # I didn't want to remove yet in case it breaks something with the transfer. - notes: str | None = None - notes: str | None + # The 'notes' field is now a List of NoteResponse objects, + # matching the polymorphic relationship in the database model. + notes: List[NoteResponse] = [] point: str elevation: float | None horizontal_datum: str = "WGS84" @@ -87,9 +86,6 @@ class LocationResponse(BaseResponseModel): county: str | None quad_name: str | None - # The new relationship to the polymorphic Notes table - notes: List[NoteResponse] = [] - @field_validator("point", mode="before") def point_to_wkt(cls, value): if isinstance(value, WKBElement): @@ -118,9 +114,9 @@ class UpdateLocation(BaseUpdateModel, ValidateLocation): """ # name: str | None = None - # TODO: remove `notes` field once polymorphic Notes are successfully implemented. - # I didn't want to remove yet in case it breaks something with the transfer. - notes: str | None = None + # TODO: AI suggested managing notes via a separate API endpoint, /notes/{note_id}. + # I don't know if we want to do that, but am leaving this comment for future reference. + # notes: str | None = None point: str | None = None elevation: float | None = None release_status: str | None = None diff --git a/schemas/thing.py b/schemas/thing.py index a3014cd98..64710dbd3 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -142,7 +142,6 @@ class BaseThingResponse(BaseResponseModel): thing_type: str current_location: LocationResponse | None first_visit_date: PastDate | None - # The new relationship to the polymorphic Notes table notes: List[NoteResponse] = [] diff --git a/tests/test_location.py b/tests/test_location.py index 628c1c352..755cd8f89 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -43,7 +43,12 @@ def override_dependencies_fixture(): def test_add_location(): payload = { # "name": "test location", - "notes": "these are some test notes", + "notes": [ + { + "note_type": "Access", + "content": "These are some test access notes.", + } + ], "point": "POINT (-106.607784 35.118924)", "elevation": 1558.8, "release_status": "draft", @@ -59,7 +64,9 @@ def test_add_location(): assert "id" in data assert "created_at" in data # assert data["name"] == payload["name"] - assert data["notes"] == payload["notes"] + assert len(data["notes"]) == 1 + assert data["notes"][0]["note_type"] == "Access" + assert data["notes"][0]["content"] == "These are some test access notes." assert data["point"] == payload["point"] assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] @@ -81,7 +88,9 @@ def test_add_location(): def test_update_location(location): payload = { # "name": "patched name", - "notes": "these are some patched notes", + "notes": [ + {"note_type": "Access", "content": "These are some patched access notes."} + ], "point": "POINT (-106.904107 34.068198)", "elevation": 1408.3, "release_status": "draft", @@ -95,7 +104,9 @@ def test_update_location(location): data = response.json() assert data["id"] == location.id # assert data["name"] == payload["name"] - assert data["notes"] == payload["notes"] + assert len(data["notes"]) == 1 + assert data["notes"][0]["note_type"] == "Access" + assert data["notes"][0]["content"] == "These are some patched access notes." assert data["point"] == payload["point"] assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] @@ -144,7 +155,12 @@ def test_get_locations(location): "+00:00", "Z" ) # assert data["items"][0]["name"] == location.name - assert data["items"][0]["notes"] == location.notes + assert isinstance(data["items"][0]["notes"], list) + # If you know the exact number of notes expected: + # assert len(data["items"][0]["notes"]) == expected_count + # If you want to check content of a specific note: + # if data["items"][0]["notes"]: + # assert data["items"][0]["notes"][0]["content"] == expected_content assert data["items"][0]["point"] == to_shape(location.point).wkt assert data["items"][0]["elevation"] == location.elevation assert data["items"][0]["release_status"] == location.release_status From 32672465ca539661b7e5fd9224c3572af13d75fd Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 10 Nov 2025 16:47:10 -0700 Subject: [PATCH 12/24] feat: enhance note management by adding release status and updating note handling in CRUD operations --- core/initializers.py | 9 ++++----- db/base.py | 17 ++++++++++++----- db/location.py | 2 +- db/notes.py | 9 +++++---- schemas/location.py | 8 +++++--- schemas/notes.py | 4 +--- services/crud_helper.py | 25 ++++++++++++++++++++++++- tests/__init__.py | 7 +------ tests/conftest.py | 11 ++++++++++- tests/test_location.py | 3 ++- 10 files changed, 65 insertions(+), 30 deletions(-) diff --git a/core/initializers.py b/core/initializers.py index 3da41018b..1449e4463 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -94,11 +94,10 @@ def init_parameter(path: str = None) -> None: def erase_and_rebuild_db(session: Session): from sqlalchemy import text - with session.bind.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() + session.execute(text("DROP SCHEMA public CASCADE")) + session.execute(text("CREATE SCHEMA public")) + session.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + session.commit() Base.metadata.drop_all(session.bind) Base.metadata.create_all(session.bind) diff --git a/db/base.py b/db/base.py index 82cb66323..279973dfd 100644 --- a/db/base.py +++ b/db/base.py @@ -36,6 +36,9 @@ 7. An `AuditMixin` to add standard audit columns to tables. """ +import re +from typing import TYPE_CHECKING + from sqlalchemy import ( Column, DateTime, @@ -52,11 +55,8 @@ mapped_column, relationship, ) -from sqlalchemy_searchable import make_searchable from sqlalchemy_continuum import make_versioned -import re - -from typing import TYPE_CHECKING +from sqlalchemy_searchable import make_searchable if TYPE_CHECKING: from db.notes import Notes @@ -242,7 +242,13 @@ def notes(cls): viewonly=True, ) - def add_note(self, content: str, note_type: str, created_by: str) -> "Notes": + def add_note( + self, + content: str, + note_type: str, + release_status: str = "draft", + created_by: str = None, + ) -> "Notes": """ A convenient factory method to create a new Note associated with this object. This provides a clean, object-oriented API for writing. @@ -255,6 +261,7 @@ def add_note(self, content: str, note_type: str, created_by: str) -> "Notes": note_type=note_type, notable_id=self.id, notable_type=self.__class__.__name__, + release_status=release_status, ) diff --git a/db/location.py b/db/location.py index 682f49bd8..f95e8b552 100644 --- a/db/location.py +++ b/db/location.py @@ -56,7 +56,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): state: Mapped[str] = mapped_column(String(100), nullable=True) quad_name: Mapped[str] = mapped_column(String(100), nullable=True) # TODO: remove this 'notes' field in favor of using the polymorphic Notes table. Did not remove it yet to avoid breaking existing data model. - notes: Mapped[str] = mapped_column(Text, nullable=True) + # notes: Mapped[str] = mapped_column(Text, nullable=True) nma_notes_location: Mapped[str] = mapped_column(Text, nullable=True) nma_coordinate_notes: Mapped[str] = mapped_column(Text, nullable=True) elevation_accuracy: Mapped[float] = mapped_column(nullable=True) diff --git a/db/notes.py b/db/notes.py index 2793dfa25..4bd39f742 100644 --- a/db/notes.py +++ b/db/notes.py @@ -35,10 +35,11 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): nullable=False, comment="The ID of the parent record this note is about (e.g., a `thing_id`, `location_id`, etc).", ) - notable_type: Mapped[str] = lexicon_term( - nullable=False, - comment="The type of the note associated with this record.", - ) + notable_type: Mapped[str] = mapped_column() + # notable_type: Mapped[str] = lexicon_term( + # nullable=False, + # comment="The type of the note associated with this record.", + # ) # --- Columns --- note_type: Mapped[str] = lexicon_term( diff --git a/schemas/location.py b/schemas/location.py index 19617c452..0bcd226f3 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from typing import List + from geoalchemy2 import WKBElement from geoalchemy2.shape import to_shape from pydantic import BaseModel, field_validator from core.enums import ElevationMethod, CoordinateMethod from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel -from schemas.notes import NoteResponse -from typing import List +from schemas.notes import NoteResponse, CreateNote, UpdateNote from services.validation.geospatial import validate_wkt_geometry @@ -46,6 +47,7 @@ class CreateLocation(BaseCreateModel, ValidateLocation): # TODO: AI suggested managing notes via a separate /locations/{id}/notes endpoint. # I don't know if we want to do that, but am leaving this comment for future reference. # notes: str | None = None + notes: List[CreateNote] = [] point: str # point is required and should be in WKT format elevation: float elevation_accuracy: float | None = None @@ -115,7 +117,7 @@ class UpdateLocation(BaseUpdateModel, ValidateLocation): # name: str | None = None # TODO: AI suggested managing notes via a separate API endpoint, /notes/{note_id}. # I don't know if we want to do that, but am leaving this comment for future reference. - # notes: str | None = None + notes: List[UpdateNote] = [] point: str | None = None elevation: float | None = None elevation_accuracy: float | None = None diff --git a/schemas/notes.py b/schemas/notes.py index 9ec50b2ff..c96f6c736 100644 --- a/schemas/notes.py +++ b/schemas/notes.py @@ -2,7 +2,6 @@ Pydantic models for the Notes table. """ -from pydantic import BaseModel from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel # -------- BASE SCHEMA: ---------- @@ -11,7 +10,7 @@ """ -class BaseNote(BaseModel): +class BaseNote: note_type: str content: str @@ -33,7 +32,6 @@ class NoteResponse(BaseResponseModel, BaseNote): Response schema for Note details. """ - note_id: int notable_id: int notable_type: str diff --git a/services/crud_helper.py b/services/crud_helper.py index 6ef4d80e4..249dd2e18 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -18,6 +18,7 @@ from sqlalchemy.orm import Session, DeclarativeBase from starlette.status import HTTP_204_NO_CONTENT +from db import NotesMixin from services.query_helper import simple_get_by_id @@ -35,10 +36,22 @@ def model_adder(session, table, model, user=None, **kwargs): md["created_by_id"] = user["sub"] md["created_by_name"] = user["name"] + notes = None + if issubclass(table, NotesMixin): + notes = md.pop("notes", None) + obj = table(**md) + session.add(obj) session.commit() session.refresh(obj) + + if notes: + for n in notes: + note = obj.add_note(**n) + session.add(note) + obj.notes.append(note) + return obj @@ -60,7 +73,17 @@ def model_patcher( """ for key, value in payload.model_dump(exclude_unset=True).items(): - setattr(item, key, value) + if isinstance(item, NotesMixin) and key == "notes": + # delete all notes and re-add + for note in item.notes: + session.delete(note) + + for note in value: + note = item.add_note(**note) + item.notes.append(note) + session.add(note) + else: + setattr(item, key, value) if user: item.updated_by_id = user["sub"] diff --git a/tests/__init__.py b/tests/__init__.py index 1dbba0836..7d9a155d2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -34,19 +34,14 @@ from fastapi_pagination import add_pagination from starlette.middleware.cors import CORSMiddleware -from core.initializers import init_lexicon, init_parameter, register_routes +from core.initializers import register_routes from db import Base, Parameter from db.engine import session_ctx from core.app import app - -# Base.metadata.drop_all(engine) -# Base.metadata.create_all(engine) with session_ctx() as session: erase_and_initalize(session) -init_lexicon() -init_parameter() register_routes(app) app.add_middleware( diff --git a/tests/conftest.py b/tests/conftest.py index 34944f957..72270ee0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ def location(): with session_ctx() as session: loc = Location( # name="first location", - notes="these are some test notes", + # notes="these are some test notes", point="POINT(-107.949533 33.809665)", elevation=2464.9, release_status="draft", @@ -24,10 +24,19 @@ def location(): # county="Catron", # quad_name="Luera Mountains West", ) + session.add(loc) session.commit() session.refresh(loc) + + note = loc.add_note("these are some test notes", "Other") + session.add(note) + session.commit() + session.refresh(loc) + yield loc + + session.delete(note) session.delete(loc) session.commit() diff --git a/tests/test_location.py b/tests/test_location.py index d2eb8a304..31d9d7a38 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -140,7 +140,8 @@ def test_patch_location_404_not_found(location): bad_location_id = 99999 location_notes_patch = "patched notes" response = client.patch( - f"/location/{bad_location_id}", json={"notes": location_notes_patch} + f"/location/{bad_location_id}", + json={"notes": [{"content": location_notes_patch, "note_type": "Other"}]}, ) data = response.json() assert response.status_code == 404 From fc31bc8fed69012a2cf851b25961a96d66db415d Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 10 Nov 2025 16:56:29 -0700 Subject: [PATCH 13/24] fix: update note handling in CRUD operations to ensure session commits and refreshes --- services/crud_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/crud_helper.py b/services/crud_helper.py index 249dd2e18..810bcc7b3 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -50,8 +50,9 @@ def model_adder(session, table, model, user=None, **kwargs): for n in notes: note = obj.add_note(**n) session.add(note) - obj.notes.append(note) + session.commit() + session.refresh(obj) return obj @@ -80,7 +81,6 @@ def model_patcher( for note in value: note = item.add_note(**note) - item.notes.append(note) session.add(note) else: setattr(item, key, value) From 40ed513e88bd5ac125abf6b50350c65099edf20b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 10 Nov 2025 10:30:50 -0700 Subject: [PATCH 14/24] feat: get polymorphic record via function --- services/util.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/services/util.py b/services/util.py index cb3d8826c..eb77a3721 100644 --- a/services/util.py +++ b/services/util.py @@ -5,6 +5,7 @@ import httpx from constants import SRID_WGS84 +from db import Base TRANSFORMERS = {} @@ -115,6 +116,41 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] +def retrieve_polymorphic_table_record( + target_record: Base, + polymorphic_relationship: str, + polymorphic_type: str, + latest=True, +) -> Base: + """ + Retrieve a record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. + Parameters: + ---------- + target_record : Base + The parent record from which to retrieve the polymorphic child record. + polymorphic_relationship : str + The name of the relationship attribute on the parent record that corresponds to the polymorphic table. + polymorphic_type : str + The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). + latest : bool, optional + If True, retrieves the latest record based on start_date. Defaults to True. + """ + if polymorphic_relationship == "permissions": + type_field = "permission_type" + elif polymorphic_relationship == "status_history": + type_field = "status_type" + + polymorphic_records = getattr(target_record, polymorphic_relationship) + type_polymorphic_records = [ + r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + ] + sorted_type_polymorphic_records = sorted( + type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + ) + return sorted_type_polymorphic_records[0] + + if __name__ == "__main__": x = -106.904107 y = 34.068198 From 23832ea8ff904a0821e8c351ba78d0a75bcad588 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 10 Nov 2025 15:32:39 -0700 Subject: [PATCH 15/24] refactor: latest record must have null end date --- services/util.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/services/util.py b/services/util.py index eb77a3721..e9e286a6e 100644 --- a/services/util.py +++ b/services/util.py @@ -116,15 +116,16 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_polymorphic_table_record( +def retrieve_latest_polymorphic_table_record( target_record: Base, polymorphic_relationship: str, polymorphic_type: str, - latest=True, ) -> Base: """ - Retrieve a record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. + Retrieve the latest record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. This + requires end_date to be None + Parameters: ---------- target_record : Base @@ -143,10 +144,12 @@ def retrieve_polymorphic_table_record( polymorphic_records = getattr(target_record, polymorphic_relationship) type_polymorphic_records = [ - r for r in polymorphic_records if getattr(r, type_field) == polymorphic_type + r + for r in polymorphic_records + if getattr(r, type_field) == polymorphic_type and r.end_date is None ] sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=latest + type_polymorphic_records, key=lambda r: r.start_date, reverse=True ) return sorted_type_polymorphic_records[0] From ce1f52252747ea7db1a02aa85b3f0e3d0a48c648 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 10 Nov 2025 15:38:32 -0700 Subject: [PATCH 16/24] refactor: move polymorphic record retrival to tests --- services/util.py | 39 --------------------------------------- tests/__init__.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/services/util.py b/services/util.py index e9e286a6e..cb3d8826c 100644 --- a/services/util.py +++ b/services/util.py @@ -5,7 +5,6 @@ import httpx from constants import SRID_WGS84 -from db import Base TRANSFORMERS = {} @@ -116,44 +115,6 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float | None: return data["value"] -def retrieve_latest_polymorphic_table_record( - target_record: Base, - polymorphic_relationship: str, - polymorphic_type: str, -) -> Base: - """ - Retrieve the latest record from a polymorphic table. This function assumes that the - parent class has the correct mixin to support retrieval via an attribute. This - requires end_date to be None - - Parameters: - ---------- - target_record : Base - The parent record from which to retrieve the polymorphic child record. - polymorphic_relationship : str - The name of the relationship attribute on the parent record that corresponds to the polymorphic table. - polymorphic_type : str - The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). - latest : bool, optional - If True, retrieves the latest record based on start_date. Defaults to True. - """ - if polymorphic_relationship == "permissions": - type_field = "permission_type" - elif polymorphic_relationship == "status_history": - type_field = "status_type" - - polymorphic_records = getattr(target_record, polymorphic_relationship) - type_polymorphic_records = [ - r - for r in polymorphic_records - if getattr(r, type_field) == polymorphic_type and r.end_date is None - ] - sorted_type_polymorphic_records = sorted( - type_polymorphic_records, key=lambda r: r.start_date, reverse=True - ) - return sorted_type_polymorphic_records[0] - - if __name__ == "__main__": x = -106.904107 y = 34.068198 diff --git a/tests/__init__.py b/tests/__init__.py index 7d9a155d2..c67491ba2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -104,4 +104,42 @@ def cleanup_patch_test(model: Base, payload: dict, original_data: Base) -> None: session.commit() +def retrieve_latest_polymorphic_table_record( + target_record: Base, + polymorphic_relationship: str, + polymorphic_type: str, +) -> Base: + """ + Retrieve the latest record from a polymorphic table. This function assumes that the + parent class has the correct mixin to support retrieval via an attribute. This + requires end_date to be None + + Parameters: + ---------- + target_record : Base + The parent record from which to retrieve the polymorphic child record. + polymorphic_relationship : str + The name of the relationship attribute on the parent record that corresponds to the polymorphic table. + polymorphic_type : str + The specific type of the polymorphic record to retrieve (e.g., 'Use Status' or 'Monitoring Status' for StatusHistory). + latest : bool, optional + If True, retrieves the latest record based on start_date. Defaults to True. + """ + if polymorphic_relationship == "permissions": + type_field = "permission_type" + elif polymorphic_relationship == "status_history": + type_field = "status_type" + + polymorphic_records = getattr(target_record, polymorphic_relationship) + type_polymorphic_records = [ + r + for r in polymorphic_records + if getattr(r, type_field) == polymorphic_type and r.end_date is None + ] + sorted_type_polymorphic_records = sorted( + type_polymorphic_records, key=lambda r: r.start_date, reverse=True + ) + return sorted_type_polymorphic_records[0] + + # ============= EOF ============================================= From d672c725be55cac1e2a59ef03163fe7880f80cf0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 11 Nov 2025 17:00:45 -0700 Subject: [PATCH 17/24] fix: call erase and rebuild db from core/initializers.py --- tests/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index c67491ba2..678c60440 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -29,18 +29,22 @@ # time.tzset() -from transfers.transfer import erase_and_initalize from fastapi.testclient import TestClient from fastapi_pagination import add_pagination from starlette.middleware.cors import CORSMiddleware -from core.initializers import register_routes +from core.initializers import ( + init_lexicon, + init_parameter, + register_routes, + erase_and_rebuild_db, +) from db import Base, Parameter from db.engine import session_ctx from core.app import app with session_ctx() as session: - erase_and_initalize(session) + erase_and_rebuild_db(session) register_routes(app) From e1ff3a3e5efede5e37bc0908cb120d42b8598d7c Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 09:42:39 -0600 Subject: [PATCH 18/24] feat: Add Notes schema and polymorphic relationships to Thing and Location models - Created new Pydantic schema for the Notes model - Added polymorphic relationship `notes: List[NoteResponse] = []` to Thing and Location schemas for flexible data modeling. --- schemas/location.py | 3 +++ schemas/notes.py | 2 ++ schemas/thing.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/schemas/location.py b/schemas/location.py index 0bcd226f3..71dcf225d 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -87,6 +87,9 @@ class LocationResponse(BaseResponseModel): county: str | None quad_name: str | None + # The new relationship to the polymorphic Notes table + notes: List[NoteResponse] = [] + @field_validator("point", mode="before") def point_to_wkt(cls, value): if isinstance(value, WKBElement): diff --git a/schemas/notes.py b/schemas/notes.py index c96f6c736..e2d083626 100644 --- a/schemas/notes.py +++ b/schemas/notes.py @@ -2,6 +2,7 @@ Pydantic models for the Notes table. """ +from pydantic import BaseModel from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel # -------- BASE SCHEMA: ---------- @@ -32,6 +33,7 @@ class NoteResponse(BaseResponseModel, BaseNote): Response schema for Note details. """ + note_id: int notable_id: int notable_type: str diff --git a/schemas/thing.py b/schemas/thing.py index 4649096d4..32dd6c229 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -139,6 +139,9 @@ class BaseThingResponse(BaseResponseModel): # The new relationship to the polymorphic Notes table notes: List[NoteResponse] = [] + # The new relationship to the polymorphic Notes table + notes: List[NoteResponse] = [] + class WellResponse(BaseThingResponse): """ From a09348a856cc1c6af08263d39c0deab5a49c4908 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 22 Oct 2025 15:10:46 -0600 Subject: [PATCH 19/24] fix: Standardize location notes as List[NoteResponse] across schemas - Comment out string-type notes fields from CreateLocation and UpdateLocation schemas - Ensure LocationResponse consistently uses notes as List[NoteResponse] - Fix validation error where API was receiving string notes but expecting list type - Complete transition to polymorphic notes relationship for location entities - Update test data to create proper Note objects instead of using string values This resolves the ResponseValidationError where the API expected notes to be a list but received a string value ('these are some test notes'). --- schemas/location.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/schemas/location.py b/schemas/location.py index 71dcf225d..0bcd226f3 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -87,9 +87,6 @@ class LocationResponse(BaseResponseModel): county: str | None quad_name: str | None - # The new relationship to the polymorphic Notes table - notes: List[NoteResponse] = [] - @field_validator("point", mode="before") def point_to_wkt(cls, value): if isinstance(value, WKBElement): From 8ef490411dd6c7cbf33d212607f98f1a39d65598 Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 10 Nov 2025 16:47:10 -0700 Subject: [PATCH 20/24] feat: enhance note management by adding release status and updating note handling in CRUD operations --- schemas/notes.py | 2 -- services/crud_helper.py | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/schemas/notes.py b/schemas/notes.py index e2d083626..c96f6c736 100644 --- a/schemas/notes.py +++ b/schemas/notes.py @@ -2,7 +2,6 @@ Pydantic models for the Notes table. """ -from pydantic import BaseModel from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel # -------- BASE SCHEMA: ---------- @@ -33,7 +32,6 @@ class NoteResponse(BaseResponseModel, BaseNote): Response schema for Note details. """ - note_id: int notable_id: int notable_type: str diff --git a/services/crud_helper.py b/services/crud_helper.py index 810bcc7b3..bdf28e96f 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -53,6 +53,13 @@ def model_adder(session, table, model, user=None, **kwargs): session.commit() session.refresh(obj) + + if notes: + for n in notes: + note = obj.add_note(**n) + session.add(note) + obj.notes.append(note) + return obj From 8837553d73955379adfecd78880ecdc47acff6b1 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 11 Nov 2025 22:33:11 -0700 Subject: [PATCH 21/24] fix: streamline note handling by consolidating session commit and refresh logic --- core/initializers.py | 3 +++ services/crud_helper.py | 11 ++--------- tests/features/environment.py | 4 +--- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/core/initializers.py b/core/initializers.py index 1449e4463..945bb9c0d 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -102,6 +102,9 @@ def erase_and_rebuild_db(session: Session): Base.metadata.drop_all(session.bind) Base.metadata.create_all(session.bind) + init_lexicon() + init_parameter() + def init_lexicon(path: str = None) -> None: if path is None: diff --git a/services/crud_helper.py b/services/crud_helper.py index bdf28e96f..86c5f66ec 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -51,15 +51,8 @@ def model_adder(session, table, model, user=None, **kwargs): note = obj.add_note(**n) session.add(note) - session.commit() - session.refresh(obj) - - if notes: - for n in notes: - note = obj.add_note(**n) - session.add(note) - obj.notes.append(note) - + session.commit() + session.refresh(obj) return obj diff --git a/tests/features/environment.py b/tests/features/environment.py index ac97355c1..22cf97d77 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -16,7 +16,7 @@ import random from datetime import datetime, timedelta -from core.initializers import erase_and_rebuild_db, init_lexicon, init_parameter +from core.initializers import erase_and_rebuild_db from db import ( Location, Thing, @@ -194,8 +194,6 @@ def before_all(context): with session_ctx() as session: if session.query(LexiconTerm).count() == 0 or force: erase_and_rebuild_db(session) - init_lexicon() - init_parameter() loc_1 = add_location(context, session) loc_2 = add_location(context, session) From e5897c8309e64990735bed00e25603f5a44d6728 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 14 Nov 2025 11:14:22 -0700 Subject: [PATCH 22/24] refactor: rename notable_id and notable_type to target_id and target_table in Notes model; reintroduce NotesMixin for polymorphic relationships --- db/base.py | 52 +--------------------------------- db/location.py | 3 +- db/notes.py | 63 +++++++++++++++++++++++++++++++++++------ db/thing.py | 3 +- schemas/notes.py | 4 +-- services/crud_helper.py | 2 +- tests/test_location.py | 3 +- 7 files changed, 63 insertions(+), 67 deletions(-) diff --git a/db/base.py b/db/base.py index 279973dfd..6171f8273 100644 --- a/db/base.py +++ b/db/base.py @@ -59,7 +59,7 @@ from sqlalchemy_searchable import make_searchable if TYPE_CHECKING: - from db.notes import Notes + pass make_versioned() @@ -215,56 +215,6 @@ def permissions(self): ) -class NotesMixin: - """ - Mixin for models that can have multiple types or categories of notes. - It automatically creates a polymorphic One-to-Many relationship to the - Notes table. - """ - - @declared_attr - def notes(cls): - """ - The high-performance, declarative relationship for reading notes. - This provides a polymorphic one-to-many link to the Notes table. - - PERFORMANCE NOTE: Use with `selectinload` in queries to prevent the - N+1 query problem when accessing notes for multiple parent objects. - """ - # All parent tables use 'id' as their primary key. - pk_name = "id" - - return relationship( - "Notes", - primaryjoin=f"and_({cls.__name__}.{pk_name}==foreign(Notes.notable_id), " - f"Notes.notable_type=='{cls.__name__}')", - lazy="selectin", - viewonly=True, - ) - - def add_note( - self, - content: str, - note_type: str, - release_status: str = "draft", - created_by: str = None, - ) -> "Notes": - """ - A convenient factory method to create a new Note associated with this object. - This provides a clean, object-oriented API for writing. - """ - # This import is inside the method to avoid circular import issues at runtime. - from db.notes import Notes - - return Notes( - content=content, - note_type=note_type, - notable_id=self.id, - notable_type=self.__class__.__name__, - release_status=release_status, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/location.py b/db/location.py index f95e8b552..a01eb1356 100644 --- a/db/location.py +++ b/db/location.py @@ -30,8 +30,9 @@ from sqlalchemy.orm import relationship, Mapped, mapped_column from constants import SRID_WGS84 -from db.base import Base, AutoBaseMixin, ReleaseMixin, NotesMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term +from db.notes import NotesMixin if TYPE_CHECKING: from db.thing import Thing diff --git a/db/notes.py b/db/notes.py index 4bd39f742..461369d53 100644 --- a/db/notes.py +++ b/db/notes.py @@ -13,8 +13,8 @@ from typing import TYPE_CHECKING -from sqlalchemy import Integer, Text, Index -from sqlalchemy.orm import relationship, Mapped, mapped_column +from sqlalchemy import Integer, Text, Index, and_ +from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -30,12 +30,12 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): """ # --- Polymorphic Columns --- - notable_id: Mapped[int] = mapped_column( + target_id: Mapped[int] = mapped_column( Integer, nullable=False, comment="The ID of the parent record this note is about (e.g., a `thing_id`, `location_id`, etc).", ) - notable_type: Mapped[str] = mapped_column() + target_table: Mapped[str] = mapped_column() # notable_type: Mapped[str] = lexicon_term( # nullable=False, # comment="The type of the note associated with this record.", @@ -52,12 +52,12 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): # These are viewonly relationships used by the 'target' property below. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin="and_(foreign(Notes.notable_id) == Thing.id, Notes.notable_type == 'Thing')", + primaryjoin="and_(foreign(Notes.target_id) == Thing.id, Notes.target_table == 'thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin="and_(foreign(Notes.notable_id) == Location.id, Notes.notable_type == 'Location')", + primaryjoin="and_(foreign(Notes.target_id) == Location.id, Notes.target_table == 'location')", viewonly=True, ) @@ -70,9 +70,56 @@ def target(self): consistent way to access the parent of a polymorphic record without needing to check the 'notable_type' field manually. """ - return getattr(self, f"_{self.notable_type.lower()}_target") + return getattr(self, f"_{self.target_table.lower()}_target") # --- Table Arguments --- # A composite index to optimize retrieval of all note records for a specific parent object. - __table_args__ = (Index("ix_notes_polymorphic_link", "notable_id", "notable_type"),) + __table_args__ = (Index("ix_notes_polymorphic_link", "target_id", "target_table"),) + + +class NotesMixin: + """ + Mixin for models that can have multiple types or categories of notes. + It automatically creates a polymorphic One-to-Many relationship to the + Notes table. + """ + + @declared_attr + def notes(cls): + """ + The high-performance, declarative relationship for reading notes. + This provides a polymorphic one-to-many link to the Notes table. + + PERFORMANCE NOTE: Use with `selectinload` in queries to prevent the + N+1 query problem when accessing notes for multiple parent objects. + """ + return relationship( + "Notes", + primaryjoin=and_( + cls.id == foreign(Notes.target_id), + Notes.target_table == cls.__name__, + ), + cascade="all, delete-orphan", + lazy="selectin", + ) + + def add_note( + self, + content: str, + note_type: str, + release_status: str = "draft", + created_by: str = None, + ) -> Notes: + """ + A convenient factory method to create a new Note associated with this object. + This provides a clean, object-oriented API for writing. + """ + + return Notes( + content=content, + note_type=note_type, + target_id=self.id, + target_table=self.__class__.__name__, + release_status=release_status, + ) diff --git a/db/thing.py b/db/thing.py index 5b7515ebf..4fb1faa4e 100644 --- a/db/thing.py +++ b/db/thing.py @@ -20,7 +20,7 @@ from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType -from db import lexicon_term +from db import lexicon_term, NotesMixin from db.asset import Asset from db.base import ( AutoBaseMixin, @@ -28,7 +28,6 @@ ReleaseMixin, StatusHistoryMixin, PermissionMixin, - NotesMixin, ) if TYPE_CHECKING: diff --git a/schemas/notes.py b/schemas/notes.py index c96f6c736..85c47ed9b 100644 --- a/schemas/notes.py +++ b/schemas/notes.py @@ -32,8 +32,8 @@ class NoteResponse(BaseResponseModel, BaseNote): Response schema for Note details. """ - notable_id: int - notable_type: str + target_id: int + target_table: str # -------- UPDATE ---------- diff --git a/services/crud_helper.py b/services/crud_helper.py index 86c5f66ec..01eaeb254 100644 --- a/services/crud_helper.py +++ b/services/crud_helper.py @@ -18,7 +18,7 @@ from sqlalchemy.orm import Session, DeclarativeBase from starlette.status import HTTP_204_NO_CONTENT -from db import NotesMixin +from db.notes import NotesMixin from services.query_helper import simple_get_by_id diff --git a/tests/test_location.py b/tests/test_location.py index 31d9d7a38..cbf790674 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -26,7 +26,6 @@ client, override_authentication, cleanup_post_test, - cleanup_patch_test, ) @@ -130,7 +129,7 @@ def test_update_location(location): payload["state"] = location.state payload["county"] = location.county payload["quad_name"] = location.quad_name - cleanup_patch_test(Location, payload, location) + # cleanup_patch_test(Location, payload, location) def test_patch_location_404_not_found(location): From 8cb720f11d99769f0207d0c366f666cacb90930e Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 14 Nov 2025 12:59:07 -0700 Subject: [PATCH 23/24] fix: comment out test notes and polymorphic relationship mappings in environment and notes files --- db/notes.py | 24 ++++++++++++------------ tests/features/environment.py | 3 +-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/db/notes.py b/db/notes.py index 461369d53..afe52f155 100644 --- a/db/notes.py +++ b/db/notes.py @@ -19,8 +19,7 @@ from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term if TYPE_CHECKING: - from db.thing import Thing - from db.location import Location + pass class Notes(Base, AutoBaseMixin, ReleaseMixin): @@ -50,16 +49,16 @@ class Notes(Base, AutoBaseMixin, ReleaseMixin): # --- Polymorphic Parent Relationships (Internal) --- # These are viewonly relationships used by the 'target' property below. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Notes.target_id) == Thing.id, Notes.target_table == 'thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Notes.target_id) == Location.id, Notes.target_table == 'location')", - viewonly=True, - ) + # _thing_target: Mapped["Thing"] = relationship( + # "Thing", + # primaryjoin="and_(foreign(Notes.target_id) == Thing.id, Notes.target_table == 'thing')", + # viewonly=True, + # ) + # _location_target: Mapped["Location"] = relationship( + # "Location", + # primaryjoin="and_(foreign(Notes.target_id) == Location.id, Notes.target_table == 'location')", + # viewonly=True, + # ) @property def target(self): @@ -102,6 +101,7 @@ def notes(cls): ), cascade="all, delete-orphan", lazy="selectin", + overlaps="notes", ) def add_note( diff --git a/tests/features/environment.py b/tests/features/environment.py index a95fa76f8..664a795f9 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -29,7 +29,6 @@ Deployment, TransducerObservationBlock, ) - from db.engine import session_ctx @@ -49,7 +48,7 @@ def closure(context, *args, **kwargs): def add_location(context, session): loc = Location( # name="first location", - notes="these are some test notes", + # notes="these are some test notes", point="POINT(-107.949533 33.809665)", elevation=2464.9, release_status="draft", From 774a4a223e18f3b326d81d8bfd28d2ab426d04c9 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 17 Nov 2025 16:15:05 -0700 Subject: [PATCH 24/24] feat: enhance notes functionality in well and location models --- core/lexicon.json | 5 ++++- db/notes.py | 3 +++ db/thing.py | 19 ++++++++++++++++--- schemas/thing.py | 9 +++++---- services/thing_helper.py | 12 ++++++++++++ tests/features/environment.py | 21 ++++++++++++++++++--- 6 files changed, 58 insertions(+), 11 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 3a8fb0895..974cea9f9 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -679,6 +679,9 @@ {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, - {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."} + {"categories": ["note_type"], "term": "Other", "definition": "Other types of notes that do not fit into the predefined categories."}, + {"categories": ["note_type"], "term": "Water", "definition": "Water bearing zone information and other info from ose reports"}, + {"categories": ["note_type"], "term": "Measuring", "definition": "Notes about measuring/visiting the well, on Access form"} + ] } \ No newline at end of file diff --git a/db/notes.py b/db/notes.py index afe52f155..ab8384064 100644 --- a/db/notes.py +++ b/db/notes.py @@ -123,3 +123,6 @@ def add_note( target_table=self.__class__.__name__, release_status=release_status, ) + + def _get_notes(self, note_type: str) -> list[Notes]: + return [n for n in self.notes if n.note_type == note_type] diff --git a/db/thing.py b/db/thing.py index 85ad5dcc8..862bcf91c 100644 --- a/db/thing.py +++ b/db/thing.py @@ -54,9 +54,10 @@ class Thing( nullable=True, comment="To audit where the data came from in NM_Aquifer if it was transferred over", ) - notes = mapped_column(Text, nullable=True) - measuring_notes = mapped_column(Text, nullable=True) - water_notes = mapped_column(Text, nullable=True) + + # notes = mapped_column(Text, nullable=True) + # measuring_notes = mapped_column(Text, nullable=True) + # water_notes = mapped_column(Text, nullable=True) # TODO: should `name` be unique? name: Mapped[str] = mapped_column( @@ -279,6 +280,18 @@ def current_location(self): else None ) + @property + def water_notes(self): + return self._get_notes("Water") + + @property + def general_notes(self): + return self._get_notes("Other") + + @property + def measuring_notes(self): + return self._get_notes("Measuring") + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ diff --git a/schemas/thing.py b/schemas/thing.py index 2635a055d..d6392be64 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -20,7 +20,7 @@ from core.enums import WellPurpose, CasingMaterial, SpringType, ScreenType from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse -from schemas.notes import NoteResponse +from schemas.notes import NoteResponse, CreateNote # -------- VALIDATE ---------- @@ -99,6 +99,7 @@ class CreateWell(CreateBaseThing, ValidateWell): default=None, gt=0, description="Well casing depth in feet" ) well_casing_materials: list[CasingMaterial] | None = None + notes: list[CreateNote] | None = None class CreateSpring(CreateBaseThing): @@ -160,9 +161,9 @@ class WellResponse(BaseThingResponse): well_casing_materials: list[CasingMaterial] = [] well_construction_notes: str | None = None - water_notes: str | None = None - measuring_notes: str | None = None - notes: str | None = None + water_notes: list[NoteResponse] | None = None + measuring_notes: list[NoteResponse] | None = None + general_notes: list[NoteResponse] | None = None @field_validator("well_purposes", mode="before") def populate_well_purposes_with_strings(cls, well_purposes): diff --git a/services/thing_helper.py b/services/thing_helper.py index f9c661c1c..53ce54577 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -152,6 +152,10 @@ def add_thing( well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) data = data.model_dump(exclude=well_descriptor_table_list) + notes = None + if "notes" in data: + notes = data.pop("notes") + location_id = data.pop("location_id", None) group_id = data.pop("group_id", None) @@ -183,6 +187,14 @@ def add_thing( session.commit() session.refresh(thing) + + if notes: + for n in notes: + nn = thing.add_note(n["content"], n["note_type"]) + session.add(nn) + session.commit() + session.refresh(thing) + except Exception as e: session.rollback() raise e diff --git a/tests/features/environment.py b/tests/features/environment.py index 2fb881ee0..ada75806e 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -60,6 +60,10 @@ def add_location(context, session): session.add(loc) session.commit() session.refresh(loc) + n = loc.add_note("Test location", "Other") + session.add(n) + session.commit() + session.refresh(loc) context.objects["locations"].append(loc) return loc @@ -77,10 +81,11 @@ def add_well(context, session, location, name_num): well_construction_notes="Test well construction notes", well_casing_diameter=5.0, well_casing_depth=10.0, - notes="These are some test well notes", - measuring_notes="These are some measuring notes", - water_notes="This are some water notes", + # notes="These are some test well notes", + # measuring_notes="These are some measuring notes", + # water_notes="This are some water notes", ) + session.add(well) session.commit() @@ -88,7 +93,17 @@ def add_well(context, session, location, name_num): assoc.effective_start = "2025-02-01T00:00:00Z" session.add(assoc) session.commit() + session.refresh(well) + + for nt, c in ( + ("Other", "well notes"), + ("Water", "water notes"), + ("Measuring", "measuring notes"), + ): + n = well.add_note(c, nt) + session.add(n) + session.commit() session.refresh(well) context.objects["wells"].append(well)