diff --git a/.env.example b/.env.example index 227db2d9d..74fa58746 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,20 @@ DB_DRIVER=postgres POSTGRES_USER=admin POSTGRES_PASSWORD=password POSTGRES_DB= +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 + +# Connection pool configuration for parallel transfers +# pool_size: number of persistent connections to maintain +# max_overflow: additional connections allowed during peak usage +DB_POOL_SIZE=10 +DB_MAX_OVERFLOW=20 + +# Transfer configuration +# Enable parallel transfers (default: true) +TRANSFER_PARALLEL=1 +# Limit number of records per transfer type (for testing) +# TRANSFER_LIMIT=1000 # asset storage GCS_BUCKET_NAME= @@ -19,3 +33,6 @@ AUTHENTIK_URL= AUTHENTIK_CLIENT_ID= AUTHENTIK_AUTHORIZE_URL= AUTHENTIK_TOKEN_URL= + +# middleware +SESSION_SECRET_KEY=your_secret_key_here diff --git a/.github/workflows/CD_production.yml b/.github/workflows/CD_production.yml index 9804971d5..0908aa625 100644 --- a/.github/workflows/CD_production.yml +++ b/.github/workflows/CD_production.yml @@ -58,6 +58,7 @@ jobs: AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" + SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}" EOF - name: Deploy to Google Cloud diff --git a/.github/workflows/CD_staging.yml b/.github/workflows/CD_staging.yml index 6d1ac96cb..ae6ea0269 100644 --- a/.github/workflows/CD_staging.yml +++ b/.github/workflows/CD_staging.yml @@ -33,6 +33,7 @@ jobs: --output-file requirements.txt - name: Authenticate to Google Cloud + id: gcloud-auth uses: 'google-github-actions/auth@v2' with: credentials_json: ${{ secrets.CLOUD_SQL_SERVICE_ACCOUNT_KEY }} @@ -64,6 +65,16 @@ jobs: with: credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} + - name: Run Alembic migrations on staging database + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_PASSWORD: "${{ secrets.CLOUD_SQL_PASSWORD }}" + run: | + uv run alembic upgrade head + # Uses Google Cloud Secret Manager to store secret credentials - name: Create app.yaml run: | @@ -89,6 +100,7 @@ jobs: AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" + SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}" EOF - name: Deploy to Google Cloud diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5b3399c8..ab8641604 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,7 @@ jobs: POSTGRES_PASSWORD: postgres DB_DRIVER: postgres BASE_URL: http://localhost:8000 + SESSION_SECRET_KEY: supersecretkeyforunittests services: postgis: diff --git a/admin/__init__.py b/admin/__init__.py new file mode 100644 index 000000000..98a1ac316 --- /dev/null +++ b/admin/__init__.py @@ -0,0 +1,23 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Starlette Admin package for NMSampleLocations. + +Provides web-based administrative interface for managing database records. +""" +from admin.config import create_admin + +__all__ = ["create_admin"] diff --git a/admin/auth.py b/admin/auth.py new file mode 100644 index 000000000..5d08d450e --- /dev/null +++ b/admin/auth.py @@ -0,0 +1,264 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Admin authentication provider integrating with existing Authentik OIDC auth. + +This module provides a Starlette Admin AuthProvider that integrates with the +existing Authentik-based authentication system used by the NMSampleLocations API. +""" +import os +from typing import Optional + +from dataclasses import dataclass +from typing import List + +from starlette.requests import Request +from starlette.responses import RedirectResponse +from starlette_admin.auth import AdminUser, AuthProvider +from starlette_admin.exceptions import LoginFailed + +from core.permissions import _get_token_payload, verify_token + + +@dataclass +class AdminUserWithRoles(AdminUser): + """Extended AdminUser with roles for RBAC.""" + + roles: List[str] = None + + def __post_init__(self): + if self.roles is None: + self.roles = [] + + +class NMSampleLocationsAuthProvider(AuthProvider): + """ + Custom auth provider that integrates with existing Authentik OIDC authentication. + + Reuses the existing authentication infrastructure from core.permissions module. + + For MS Access users: This replaces Access file-level security with user-level + authentication. Each user logs in with their Authentik credentials and gets + assigned roles (Admin, Editor, Viewer) which control what they can do in the + admin interface. + """ + + async def is_authenticated(self, request: Request) -> bool: + """ + Check if user is authenticated by verifying their JWT token. + + This method is called on every admin page request to determine if the + user should be allowed access. + + Returns: + bool: True if user has a valid JWT token, False otherwise + """ + # Check if authentication is disabled (development mode only) + if int(os.environ.get("AUTHENTIK_DISABLE_AUTHENTICATION", 0)): + from core.settings import settings + + if settings.mode != "production": + # Allow unauthenticated access in development mode + request.state.user = AdminUserWithRoles( + username="dev_user", roles=["admin"] + ) + return True + + try: + # Try to get token from Authorization header + authorization = request.headers.get("Authorization") + if not authorization: + # Try to get token from session/cookie + token = request.session.get("token") + if not token: + return False + else: + # Extract token from "Bearer " format + token = ( + authorization.split(" ")[1] + if " " in authorization + else authorization + ) + + # Verify token using existing authentication system + is_valid = verify_token(token, scope=None, permissions=None) + + if is_valid: + # Store user in request state for later access + request.state.user = self._create_admin_user_from_token(token) + + return is_valid + except Exception: + return False + + def _create_admin_user_from_token(self, token: str) -> Optional[AdminUser]: + """ + Extract user information from JWT token and create AdminUser instance. + + Args: + token: JWT access token from Authentik + + Returns: + AdminUser instance with username and roles, or None if token invalid + """ + try: + # Decode JWT payload + payload = _get_token_payload(token) + + # Extract user information from JWT claims + username = ( + payload.get("preferred_username") + or payload.get("email") + or payload.get("sub") + ) + email = payload.get("email") + groups = payload.get("groups", []) + + # Map Authentik groups to admin roles + roles = [] + + # Standard roles + if "Admin" in groups: + roles.append("admin") + if "Editor" in groups: + roles.append("editor") + if "Viewer" in groups: + roles.append("viewer") + + # AMP-specific roles (for AMPAPI-related data) + if "AMPAdmin" in groups: + roles.append("amp_admin") + if "AMPEditor" in groups: + roles.append("amp_editor") + if "AMPViewer" in groups: + roles.append("amp_viewer") + + # Lexicon-specific roles + if "LexiconAdmin" in groups: + roles.append("lexicon_admin") + if "LexiconEditor" in groups: + roles.append("lexicon_editor") + + return AdminUserWithRoles( + username=username, + photo_url=None, # Could add user avatar URL from OIDC if available + roles=roles, + ) + except Exception: + return None + + def get_admin_user(self, request: Request) -> Optional[AdminUser]: + """ + Get the current admin user from the request. + + This method is called by Starlette Admin to get user information for + display in the UI and permission checks. + + Returns: + AdminUser instance with username and roles, or None if not authenticated + """ + # Check if user is already stored in request state + if hasattr(request.state, "user"): + return request.state.user + + try: + # Get token from request + authorization = request.headers.get("Authorization") + if not authorization: + token = request.session.get("token") + if not token: + return None + else: + token = ( + authorization.split(" ")[1] + if " " in authorization + else authorization + ) + + # Create AdminUser from token + admin_user = self._create_admin_user_from_token(token) + + # Store in request state for future calls + if admin_user: + request.state.user = admin_user + + return admin_user + except Exception: + return None + + async def login(self, *args, **kwargs) -> RedirectResponse: + """ + Redirect to Authentik OIDC login page. + + Note: Starlette Admin will show a login form, but we ignore the username/password + and redirect to Authentik OAuth flow instead. + + Args: + request: Starlette request object (extracted from args/kwargs) + *args/**kwargs: Ignored, kept for compatibility with different + Starlette Admin login call signatures + + Returns: + RedirectResponse to Authentik authorization endpoint + """ + # Starlette Admin has changed the AuthProvider.login signature across versions. + # Accept *args/**kwargs and extract the Request to stay compatible whether + # it calls login(request, data, ...) or login(username, password, remember_me, request). + request: Optional[Request] = kwargs.get("request") + if request is None: + for arg in args: + if isinstance(arg, Request): + request = arg + break + + if request is None: + raise LoginFailed("Unable to determine login request context.") + + # Redirect to Authentik OAuth authorization endpoint + authentik_authorize_url = os.environ.get("AUTHENTIK_AUTHORIZE_URL") + if not authentik_authorize_url: + raise LoginFailed( + "Authentik authentication is not configured. Please set AUTHENTIK_AUTHORIZE_URL environment variable." + ) + + # Store original URL to redirect back after login + original_url = str(request.url_for("admin:index")) + request.session["auth_redirect"] = original_url + + # Redirect to Authentik login + return RedirectResponse(url=authentik_authorize_url, status_code=302) + + async def logout(self, request: Request) -> RedirectResponse: + """ + Handle logout by clearing session and redirecting. + + Args: + request: Starlette request object + + Returns: + RedirectResponse to home page + """ + # Clear session tokens + request.session.pop("token", None) + request.session.pop("auth_redirect", None) + + # Clear user from request state + if hasattr(request.state, "user"): + delattr(request.state, "user") + + # Redirect to home page + # TODO: Consider redirecting to Authentik logout endpoint to fully log out + return RedirectResponse(url="/", status_code=302) diff --git a/admin/config.py b/admin/config.py new file mode 100644 index 000000000..e88dfdc37 --- /dev/null +++ b/admin/config.py @@ -0,0 +1,154 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Starlette Admin configuration and initialization. + +This module creates and configures the admin interface for NMSampleLocations. +""" +from starlette_admin.contrib.sqla import Admin + +from admin.auth import NMSampleLocationsAuthProvider +from admin.views import ( + LocationAdmin, + ThingAdmin, + ObservationAdmin, + ContactAdmin, + SensorAdmin, + DeploymentAdmin, + LexiconTermAdmin, + LexiconCategoryAdmin, + AssetAdmin, + AquiferTypeAdmin, + AquiferSystemAdmin, + GroupAdmin, + NotesAdmin, + SampleAdmin, + GeologicFormationAdmin, + DataProvenanceAdmin, + FieldEventAdmin, + FieldActivityAdmin, + FieldEventParticipantAdmin, + ParameterAdmin, +) +from db.engine import engine +from db.location import Location +from db.thing import Thing +from db.observation import Observation +from db.contact import Contact +from db.sensor import Sensor +from db.deployment import Deployment +from db.lexicon import ( + LexiconTerm, + LexiconCategory, +) +from db.asset import Asset +from db.aquifer_type import AquiferType +from db.aquifer_system import AquiferSystem +from db.group import Group +from db.notes import Notes +from db.sample import Sample +from db.geologic_formation import GeologicFormation +from db.data_provenance import DataProvenance +from db.field import FieldEvent, FieldActivity, FieldEventParticipant +from db.permission_history import PermissionHistory +from db.parameter import Parameter + + +def create_admin(app): + """ + Create and configure Starlette Admin instance. + + This function sets up the admin interface and mounts it to the FastAPI app + at the /admin route. + + For MS Access users: This replaces the Access database file with a web-based + admin interface. Instead of opening a .accdb file, staff will navigate to + https://your-domain.com/admin in their web browser. + + Args: + app: FastAPI application instance + + Returns: + Admin: Configured Starlette Admin instance + """ + # Create admin instance + admin = Admin( + engine=engine, + title="Ocotillod Admin", + base_url="/admin", + logo_url=None, # TODO: Add NMBGMR logo + auth_provider=NMSampleLocationsAuthProvider(), + middlewares=[], # Add custom middlewares here if needed + ) + + # Register model views + # Geography + admin.add_view(LocationAdmin(Location)) + + # Things (Wells, Springs, etc.) + admin.add_view(ThingAdmin(Thing)) + + # Observations (Water Levels) + admin.add_view(ObservationAdmin(Observation)) + + # Contacts (Owners) + admin.add_view(ContactAdmin(Contact)) + + # Equipment + admin.add_view(SensorAdmin(Sensor)) + admin.add_view(DeploymentAdmin(Deployment)) + + # Assets + admin.add_view(AssetAdmin(Asset)) + + # Aquifer + admin.add_view(AquiferSystemAdmin(AquiferSystem)) + admin.add_view(AquiferTypeAdmin(AquiferType)) + + # Groups + admin.add_view(GroupAdmin(Group)) + + # Notes + admin.add_view(NotesAdmin(Notes)) + + # Samples + admin.add_view(SampleAdmin(Sample)) + + # Field + admin.add_view(FieldEventAdmin(FieldEvent)) + admin.add_view(FieldActivityAdmin(FieldActivity)) + + # Parameters + admin.add_view(ParameterAdmin(Parameter)) + + # Geology + admin.add_view(GeologicFormationAdmin(GeologicFormation)) + + # Data provenance + admin.add_view(DataProvenanceAdmin(DataProvenance)) + + # Lexicon + admin.add_view(LexiconTermAdmin(LexiconTerm)) + admin.add_view(LexiconCategoryAdmin(LexiconCategory)) + + # Future: Add more views here as they are implemented + # admin.add_view(SampleAdmin) + # admin.add_view(GroupAdmin) + + # Mount admin to app + admin.mount_to(app) + + return admin diff --git a/admin/fields.py b/admin/fields.py new file mode 100644 index 000000000..cd71bbc17 --- /dev/null +++ b/admin/fields.py @@ -0,0 +1,140 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Custom fields for Starlette Admin. + +Provides field handlers for complex data types like PostGIS geometry. +""" +from typing import Any + +from geoalchemy2 import WKTElement +from geoalchemy2.shape import to_shape +from starlette.requests import Request +from starlette_admin import StringField + +from core.constants import SRID_WGS84 + + +class WKTField(StringField): + """ + Custom field for GeoAlchemy2 Geometry columns. + + This field converts between PostGIS geometry (WKBElement) and human-readable + WKT (Well-Known Text) format for display and editing in the admin interface. + + For MS Access users: Instead of entering Easting/Northing/UTM Zone in separate + fields, you'll enter coordinates in WKT format, for example: + POINT(-106.123 35.456) + + Note: Longitude comes first, then latitude (POINT(lon lat), not POINT(lat lon)) + """ + + async def serialize_value(self, request: Request, value: Any, action: str) -> str: + """ + Convert WKBElement (PostGIS geometry) to WKT string for display in form. + + This is called when rendering the edit/create form to show the current + value in a text input. + + Args: + request: Starlette request object + value: WKBElement from database (PostGIS geometry) + action: 'list', 'detail', 'edit', or 'create' + + Returns: + WKT string representation of geometry (e.g., "POINT(-106.123 35.456)") + """ + if value is None: + return "" + + try: + # Convert WKBElement to Shapely geometry, then to WKT + shape = to_shape(value) + return shape.wkt + except Exception: + # If conversion fails, return string representation + return str(value) + + async def parse_form_data( + self, request: Request, form_data: dict, action: str + ) -> Any: + """ + Convert WKT string from form input to WKTElement for database storage. + + This is called when saving the form to convert the user's input into + a format that can be stored in the PostGIS database. + + Args: + request: Starlette request object + form_data: Dictionary of form data + action: 'edit' or 'create' + + Returns: + WKTElement with SRID for PostGIS storage, or None if empty + + Raises: + ValueError: If WKT string is invalid + """ + wkt_string = form_data.get(self.name) + + if not wkt_string or wkt_string.strip() == "": + return None + + try: + # Parse and validate WKT string + from shapely.wkt import loads as wkt_loads + + shape = wkt_loads(wkt_string.strip()) + + # Convert to WKTElement with SRID (spatial reference identifier) + return WKTElement(shape.wkt, srid=SRID_WGS84) + except Exception as e: + raise ValueError( + f"Invalid WKT geometry: {e}. " + f"Expected format: POINT(longitude latitude), e.g., POINT(-106.123 35.456). " + f"Note: Longitude comes first, then latitude." + ) + + +class CoordinateHelpField(WKTField): + """ + Extended WKT field with detailed help text for coordinate entry. + + This version includes comprehensive help text for users transitioning + from MS Access UTM coordinate entry to WKT format. + """ + + def __init__(self, *args, **kwargs): + # Add detailed help text if not provided + if "help_text" not in kwargs: + kwargs["help_text"] = ( + "Enter coordinates in WKT (Well-Known Text) format.\n\n" + "Format: POINT(longitude latitude)\n" + "Example: POINT(-106.65082 35.08352)\n\n" + "Important:\n" + " * Longitude comes FIRST (negative for western hemisphere)\n" + " * Latitude comes SECOND\n" + " * No comma between values\n" + " * Use decimal degrees (not degrees-minutes-seconds)\n" + " * Coordinate system: WGS84 (SRID 4326)\n\n" + "If you have UTM coordinates:\n" + " 1. Use an online converter (e.g., https://www.latlong.net/utm-to-lat-long)\n" + " 2. Enter your Easting, Northing, and UTM Zone\n" + " 3. Convert to WGS84 lat/lon\n" + " 4. Enter here as POINT(lon lat)" + ) + + super().__init__(*args, **kwargs) diff --git a/admin/views/__init__.py b/admin/views/__init__.py new file mode 100644 index 000000000..74c2c141b --- /dev/null +++ b/admin/views/__init__.py @@ -0,0 +1,64 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Admin views package for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on database models. +""" +from admin.views.location import LocationAdmin +from admin.views.thing import ThingAdmin +from admin.views.observation import ObservationAdmin +from admin.views.contact import ContactAdmin +from admin.views.sensor import SensorAdmin +from admin.views.deployment import DeploymentAdmin +from admin.views.lexicon import LexiconTermAdmin, LexiconCategoryAdmin +from admin.views.asset import AssetAdmin +from admin.views.aquifer_type import AquiferTypeAdmin +from admin.views.aquifer_system import AquiferSystemAdmin +from admin.views.group import GroupAdmin +from admin.views.notes import NotesAdmin +from admin.views.sample import SampleAdmin +from admin.views.geologic_formation import GeologicFormationAdmin +from admin.views.data_provenance import DataProvenanceAdmin +from admin.views.field import ( + FieldEventAdmin, + FieldActivityAdmin, + FieldEventParticipantAdmin, +) +from admin.views.parameter import ParameterAdmin + +__all__ = [ + "LocationAdmin", + "ThingAdmin", + "ObservationAdmin", + "ContactAdmin", + "SensorAdmin", + "DeploymentAdmin", + "LexiconTermAdmin", + "LexiconCategoryAdmin", + "AssetAdmin", + "AquiferTypeAdmin", + "AquiferSystemAdmin", + "GroupAdmin", + "NotesAdmin", + "SampleAdmin", + "GeologicFormationAdmin", + "DataProvenanceAdmin", + "FieldEventAdmin", + "FieldActivityAdmin", + "FieldEventParticipantAdmin", + "ParameterAdmin", +] diff --git a/admin/views/aquifer_system.py b/admin/views/aquifer_system.py new file mode 100644 index 000000000..4b93e8dea --- /dev/null +++ b/admin/views/aquifer_system.py @@ -0,0 +1,121 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +AquiferSystemAdmin view for NMSampleLocations. +""" +from admin.fields import WKTField +from admin.views.base import OcotilloModelView + + +class AquiferSystemAdmin(OcotilloModelView): + """ + Admin view for AquiferSystem model. + """ + + # ========== Basic Configuration ========== + + name = "Aquifer Systems" + label = "Aquifer Systems" + icon = "fa fa-globe" + + # ========== List View ========== + + column_list = [ + "id", + "name", + "primary_aquifer_type", + "geographic_scale", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "name", + "primary_aquifer_type", + "geographic_scale", + "release_status", + "created_at", + ] + + column_default_sort = ("name", False) + + search_fields = [ + "name", + "description", + ] + + column_filters = [ + "primary_aquifer_type", + "geographic_scale", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "name", + "description", + "primary_aquifer_type", + "geographic_scale", + WKTField("boundary", label="Boundary (WKT)"), + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Aquifer System ID", + "name": "Name", + "description": "Description", + "primary_aquifer_type": "Primary Aquifer Type", + "geographic_scale": "Geographic Scale", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/aquifer_type.py b/admin/views/aquifer_type.py new file mode 100644 index 000000000..89ffd3c29 --- /dev/null +++ b/admin/views/aquifer_type.py @@ -0,0 +1,111 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +AquiferTypeAdmin view for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class AquiferTypeAdmin(OcotilloModelView): + """ + Admin view for AquiferType model. + """ + + # ========== Basic Configuration ========== + + name = "Aquifer Types" + label = "Aquifer Types" + icon = "fa fa-tint" + + # ========== List View ========== + + column_list = [ + "id", + "thing_aquifer_association_id", + "aquifer_type", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "thing_aquifer_association_id", + "aquifer_type", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + search_fields = [ + "aquifer_type", + ] + + column_filters = [ + "aquifer_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "thing_aquifer_association_id", + "aquifer_type", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Aquifer Type ID", + "thing_aquifer_association_id": "Thing-Aquifer Association", + "aquifer_type": "Aquifer Type", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/asset.py b/admin/views/asset.py new file mode 100644 index 000000000..6b8683fc2 --- /dev/null +++ b/admin/views/asset.py @@ -0,0 +1,134 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +AssetAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Asset model. +""" +from admin.views.base import OcotilloModelView + + +class AssetAdmin(OcotilloModelView): + """ + Admin view for Asset model. + """ + + # ========== Basic Configuration ========== + + name = "Assets" + label = "Assets" + icon = "fa fa-file" + + # ========== List View ========== + + column_list = [ + "id", + "name", + "label", + "mime_type", + "storage_service", + "size", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "name", + "mime_type", + "storage_service", + "size", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + search_fields = [ + "name", + "label", + "mime_type", + "storage_service", + "storage_path", + "uri", + ] + + column_filters = [ + "mime_type", + "storage_service", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "name", + "label", + "storage_service", + "storage_path", + "mime_type", + "size", + "uri", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Asset ID", + "name": "Name", + "label": "Label", + "storage_service": "Storage Service", + "storage_path": "Storage Path", + "mime_type": "MIME Type", + "size": "Size (bytes)", + "uri": "URI", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/base.py b/admin/views/base.py new file mode 100644 index 000000000..7d712f69f --- /dev/null +++ b/admin/views/base.py @@ -0,0 +1,138 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +from __future__ import annotations + +from typing import Any, Iterable + +from sqlalchemy import select, update +from starlette.requests import Request +from starlette.responses import Response +from starlette_admin import action +from starlette_admin.contrib.sqla import ModelView + +from db.engine import session_ctx + + +class OcotilloModelView(ModelView): + """ + Shared admin behaviors for Ocotillo data models. + + - RBAC: admin can create/edit/delete; editor can edit; any authenticated user can view. + - Data visibility: non-admin/editor users only see published rows when a release field exists. + - Publish/Unpublish actions: toggle release status when enabled and a release field is present. + """ + + release_field = "release_status" + draft_value = "draft" + published_value = "published" + enable_publish_actions: bool = True + + # ========= Permissions (RBAC) ========= + def _get_user(self, request: Request) -> Any: + return getattr(request.state, "user", None) + + def _roles(self, request: Request) -> list[str]: + user = self._get_user(request) + return getattr(user, "roles", []) if user else [] + + def _has_role(self, request: Request, roles: Iterable[str]) -> bool: + return bool(set(self._roles(request)) & set(roles)) + + def can_create(self, request: Request) -> bool: + return self._has_role(request, {"admin"}) + + def can_edit(self, request: Request) -> bool: + return self._has_role(request, {"admin", "editor"}) + + def can_delete(self, request: Request) -> bool: + return self._has_role(request, {"admin"}) + + def can_view_details(self, request: Request) -> bool: + return self._get_user(request) is not None + + # ========= Data Visibility ========= + def get_list_query(self, request: Request): + query = select(self.model) + user = self._get_user(request) + if user is None: + # Return an empty result set for anonymous users + return query.where(self.model.id == -1) + + if not hasattr(self.model, self.release_field): + return query + + if self._has_role(request, {"admin", "editor"}): + return query + return query.where( + getattr(self.model, self.release_field) == self.published_value + ) + + # ========= Actions (Publish / Unpublish) ========= + def _ensure_release_field(self) -> bool: + return self.enable_publish_actions and hasattr(self.model, self.release_field) + + @action( + name="publish_selected", + text="Publish Selected", + confirmation="Are you sure you want to publish the selected records?", + submit_btn_text="Yes, publish", + submit_btn_class="btn btn-success", + ) + async def publish_selected(self, request: Request, pks: list[int]) -> Response: + if not self._has_role(request, {"admin"}): + return Response("Only admins can publish", status_code=403) + if not self._ensure_release_field(): + return Response( + "Publish action not available for this model", status_code=400 + ) + + with session_ctx() as session: + result = session.execute( + update(self.model) + .where(self.model.id.in_(pks)) + .values({self.release_field: self.published_value}) + ) + session.commit() + updated_count = result.rowcount + return Response(f"Published {updated_count} record(s)", status_code=200) + + @action( + name="unpublish_selected", + text="Unpublish Selected (set to draft)", + confirmation="Are you sure you want to unpublish the selected records?", + submit_btn_text="Yes, unpublish", + submit_btn_class="btn btn-warning", + ) + async def unpublish_selected(self, request: Request, pks: list[int]) -> Response: + if not self._has_role(request, {"admin"}): + return Response("Only admins can unpublish", status_code=403) + if not self._ensure_release_field(): + return Response( + "Unpublish action not available for this model", status_code=400 + ) + + with session_ctx() as session: + result = session.execute( + update(self.model) + .where(self.model.id.in_(pks)) + .values({self.release_field: self.draft_value}) + ) + session.commit() + updated_count = result.rowcount + return Response(f"Unpublished {updated_count} record(s)", status_code=200) + + +# ============= EOF ============================================= diff --git a/admin/views/contact.py b/admin/views/contact.py new file mode 100644 index 000000000..dc9b4b740 --- /dev/null +++ b/admin/views/contact.py @@ -0,0 +1,169 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +ContactAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Contact (Owners) model. +""" +from admin.views.base import OcotilloModelView + + +class ContactAdmin(OcotilloModelView): + """ + Admin view for Contact model (Well Owners/Managers). + + Designed to replicate MS Access "Owners Data Entry Form" and "Owners Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all contacts + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published contacts (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Contacts" + label = "Contacts (Owners)" + icon = "fa fa-users" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "name", + "organization", + "role", + "contact_type", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "name", + "organization", + "role", + "contact_type", + "release_status", + "created_at", + ] + + column_default_sort = ("name", False) # Alphabetical by name + + search_fields = [ + "name", + "organization", + "role", + ] + + column_filters = [ + "organization", + "role", + "contact_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + # Contact Information + "name", + "organization", + "role", + "contact_type", + # Release Status + "release_status", + # Audit Fields + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + # Legacy Migration Fields + "nma_pk_owners", + "nma_pk_waterlevels", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_owners", + "nma_pk_waterlevels", + # Exclude complex relationships (manage separately) + "phones", + "emails", + "addresses", + "incomplete_nma_phones", + "permissions", + "author_associations", + "thing_associations", + "field_event_participants", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_pk_owners", + "nma_pk_waterlevels", + # Exclude complex relationships (manage separately) + "phones", + "emails", + "addresses", + "incomplete_nma_phones", + "permissions", + "author_associations", + "thing_associations", + "field_event_participants", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Contact ID", + "name": "Name", + "organization": "Organization", + "role": "Role", + "contact_type": "Contact Type", + "release_status": "Release Status", + "nma_pk_owners": "AMPAPI Owners ID (Legacy)", + "nma_pk_waterlevels": "AMPAPI WaterLevels ID (Legacy)", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "name": "Full name of the contact (First Last)", + "organization": "Organization or agency the contact is affiliated with", + "role": "Role of the contact (e.g., 'Owner', 'Measurer', 'Manager')", + "contact_type": "Type of contact (e.g., 'Primary', 'Secondary')", + "release_status": "'draft' (internal only) or 'published' (public)", + } diff --git a/admin/views/data_provenance.py b/admin/views/data_provenance.py new file mode 100644 index 000000000..caa92dd05 --- /dev/null +++ b/admin/views/data_provenance.py @@ -0,0 +1,131 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +DataProvenanceAdmin view for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class DataProvenanceAdmin(OcotilloModelView): + """ + Admin view for DataProvenance model. + """ + + name = "Data Provenance" + label = "Data Provenance" + icon = "fa fa-history" + + column_list = [ + "id", + "target_table", + "target_id", + "field_name", + "origin_type", + "origin_source", + "collection_method", + "accuracy_value", + "accuracy_unit", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "target_table", + "target_id", + "field_name", + "origin_type", + "collection_method", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + search_fields = [ + "target_table", + "field_name", + "origin_source", + ] + + column_filters = [ + "target_table", + "origin_type", + "collection_method", + "accuracy_unit", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "target_table", + "target_id", + "field_name", + "origin_type", + "origin_source", + "collection_method", + "accuracy_value", + "accuracy_unit", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Provenance ID", + "target_table": "Target Table", + "target_id": "Target ID", + "field_name": "Field Name", + "origin_type": "Origin Type", + "origin_source": "Origin Source", + "collection_method": "Collection Method", + "accuracy_value": "Accuracy Value", + "accuracy_unit": "Accuracy Unit", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/deployment.py b/admin/views/deployment.py new file mode 100644 index 000000000..1e25dfe68 --- /dev/null +++ b/admin/views/deployment.py @@ -0,0 +1,170 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +DeploymentAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Deployment model. +""" +from admin.views.base import OcotilloModelView + + +class DeploymentAdmin(OcotilloModelView): + """ + Admin view for Deployment model (Equipment Installation Log). + + Designed to replicate MS Access "Equipment Deployment Form" and "Deployment Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all deployments + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published deployments (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Deployments" + label = "Deployments (Equipment Installations)" + icon = "fa fa-plug" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "thing_id", + "sensor_id", + "installation_date", + "removal_date", + "recording_interval", + "recording_interval_units", + "release_status", + "created_at", + ] + + column_sortable_list = [ + "id", + "thing_id", + "sensor_id", + "installation_date", + "removal_date", + "recording_interval", + "release_status", + "created_at", + ] + + column_default_sort = ( + "installation_date", + True, + ) # True = descending (newest first) + + search_fields = [ + "hanging_point_description", + "notes", + ] + + column_filters = [ + "installation_date", + "removal_date", + "recording_interval_units", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + # Deployment Information + "thing_id", + "sensor_id", + "installation_date", + "removal_date", + "recording_interval", + "recording_interval_units", + "hanging_cable_length", + "hanging_point_height", + "hanging_point_description", + "notes", + # Release Status + "release_status", + # Audit Fields + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + # Exclude relationship objects (use IDs instead) + "thing", + "sensor", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + # Exclude relationship objects (use IDs instead) + "thing", + "sensor", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Deployment ID", + "thing_id": "Well/Thing", + "sensor_id": "Sensor/Equipment", + "installation_date": "Installation Date", + "removal_date": "Removal Date", + "recording_interval": "Recording Interval", + "recording_interval_units": "Interval Units", + "hanging_cable_length": "Hanging Cable Length (ft)", + "hanging_point_height": "Hanging Point Height (ft)", + "hanging_point_description": "Hanging Point Description", + "notes": "Notes", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "thing_id": "The well or thing where this equipment is deployed", + "sensor_id": "The sensor/equipment being deployed", + "installation_date": "Date the equipment was installed", + "removal_date": "Date the equipment was removed (leave blank if still installed)", + "recording_interval": "How often the sensor records data (numeric value)", + "recording_interval_units": "Units for recording interval (e.g., 'minutes', 'hours')", + "hanging_cable_length": "Length of cable from sensor to hanging point (feet)", + "hanging_point_height": "Height of hanging point above ground (feet)", + "hanging_point_description": "Description of the hanging point (e.g., 'Top of casing')", + "notes": "General notes about this deployment", + "release_status": "'draft' (internal only) or 'published' (public)", + } diff --git a/admin/views/field.py b/admin/views/field.py new file mode 100644 index 000000000..2e0b36f2a --- /dev/null +++ b/admin/views/field.py @@ -0,0 +1,276 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Field admin views for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class FieldEventAdmin(OcotilloModelView): + """ + Admin view for FieldEvent model. + """ + + name = "Field Events" + label = "Field Events" + icon = "fa fa-calendar" + + column_list = [ + "id", + "thing_id", + "event_date", + "notes", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "thing_id", + "event_date", + "release_status", + "created_at", + ] + + column_default_sort = ("event_date", True) + + search_fields = [ + "notes", + ] + + column_filters = [ + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "thing_id", + "event_date", + "notes", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Field Event ID", + "thing_id": "Thing", + "event_date": "Event Date", + "notes": "Notes", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +class FieldActivityAdmin(OcotilloModelView): + """ + Admin view for FieldActivity model. + """ + + name = "Field Activities" + label = "Field Activities" + icon = "fa fa-tasks" + + column_list = [ + "id", + "field_event_id", + "activity_type", + "notes", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "field_event_id", + "activity_type", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + search_fields = [ + "notes", + ] + + column_filters = [ + "activity_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "field_event_id", + "activity_type", + "notes", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Field Activity ID", + "field_event_id": "Field Event", + "activity_type": "Activity Type", + "notes": "Notes", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +class FieldEventParticipantAdmin(OcotilloModelView): + """ + Admin view for FieldEventParticipant model. + """ + + name = "Field Event Participants" + label = "Field Event Participants" + icon = "fa fa-users" + + column_list = [ + "id", + "field_event_id", + "contact_id", + "participant_role", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "field_event_id", + "contact_id", + "participant_role", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + column_filters = [ + "participant_role", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "field_event_id", + "contact_id", + "participant_role", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Participant ID", + "field_event_id": "Field Event", + "contact_id": "Contact", + "participant_role": "Participant Role", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/geologic_formation.py b/admin/views/geologic_formation.py new file mode 100644 index 000000000..f68123075 --- /dev/null +++ b/admin/views/geologic_formation.py @@ -0,0 +1,111 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +GeologicFormationAdmin view for NMSampleLocations. +""" +from admin.fields import WKTField +from admin.views.base import OcotilloModelView + + +class GeologicFormationAdmin(OcotilloModelView): + """ + Admin view for GeologicFormation model. + """ + + name = "Geologic Formations" + label = "Geologic Formations" + icon = "fa fa-layer-group" + + column_list = [ + "id", + "formation_code", + "lithology", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "formation_code", + "lithology", + "release_status", + "created_at", + ] + + column_default_sort = ("formation_code", False) + + search_fields = [ + "formation_code", + "description", + "lithology", + ] + + column_filters = [ + "lithology", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "formation_code", + "description", + "lithology", + WKTField("boundary", label="Boundary (WKT)"), + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Geologic Formation ID", + "formation_code": "Formation Code", + "description": "Description", + "lithology": "Lithology", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/group.py b/admin/views/group.py new file mode 100644 index 000000000..04d4e3718 --- /dev/null +++ b/admin/views/group.py @@ -0,0 +1,120 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +GroupAdmin view for NMSampleLocations. +""" +from admin.fields import WKTField +from admin.views.base import OcotilloModelView + + +class GroupAdmin(OcotilloModelView): + """ + Admin view for Group model. + """ + + # ========== Basic Configuration ========== + + name = "Groups" + label = "Groups" + icon = "fa fa-object-group" + + # ========== List View ========== + + column_list = [ + "id", + "name", + "group_type", + "parent_group_id", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "name", + "group_type", + "parent_group_id", + "release_status", + "created_at", + ] + + column_default_sort = ("name", False) + + search_fields = [ + "name", + "description", + ] + + column_filters = [ + "group_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "name", + "description", + "group_type", + "parent_group_id", + WKTField("project_area", label="Project Area (WKT)"), + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Group ID", + "name": "Name", + "description": "Description", + "group_type": "Group Type", + "parent_group_id": "Parent Group", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/lexicon.py b/admin/views/lexicon.py new file mode 100644 index 000000000..0640e9107 --- /dev/null +++ b/admin/views/lexicon.py @@ -0,0 +1,120 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Lexicon admin views for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class LexiconTermAdmin(OcotilloModelView): + """ + Admin view for LexiconTerm model. + """ + + name = "Lexicon Terms" + label = "Lexicon Terms" + icon = "fa fa-book" + enable_publish_actions = False + + column_list = [ + "id", + "term", + "definition", + ] + + column_sortable_list = [ + "id", + "term", + ] + + column_default_sort = ("term", False) + + search_fields = [ + "term", + "definition", + ] + + fields = [ + "id", + "term", + "definition", + ] + + exclude_fields_from_create = [ + "id", + ] + + exclude_fields_from_edit = [ + "id", + ] + + labels = { + "id": "Term ID", + "term": "Term", + "definition": "Definition", + } + + +class LexiconCategoryAdmin(OcotilloModelView): + """ + Admin view for LexiconCategory model. + """ + + name = "Lexicon Categories" + label = "Lexicon Categories" + icon = "fa fa-tags" + enable_publish_actions = False + + column_list = [ + "id", + "name", + "description", + ] + + column_sortable_list = [ + "id", + "name", + ] + + column_default_sort = ("name", False) + + search_fields = [ + "name", + "description", + ] + + fields = [ + "id", + "name", + "description", + ] + + exclude_fields_from_create = [ + "id", + ] + + exclude_fields_from_edit = [ + "id", + ] + + labels = { + "id": "Category ID", + "name": "Name", + "description": "Description", + } + + +# ============= EOF ============================================= diff --git a/admin/views/location.py b/admin/views/location.py new file mode 100644 index 000000000..dbeeba06d --- /dev/null +++ b/admin/views/location.py @@ -0,0 +1,173 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +LocationAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Location model. +""" +from admin.fields import CoordinateHelpField +from admin.views.base import OcotilloModelView +from db.location import Location + + +class LocationAdmin(OcotilloModelView): + """ + Admin view for Location model. + + Designed to replicate MS Access "Location Entry Form" and "Location Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all locations + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published locations (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Locations" + label = "Locations" + icon = "fa fa-map-marker" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "description", + "county", + "state", + "elevation", + "quad_name", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "description", + "elevation", + "county", + "state", + "quad_name", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) # True = descending + + search_fields = [ + "description", + "county", + "state", + "quad_name", + ] + + column_filters = [ + "county", + "state", + "release_status", + "elevation", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + "description", + CoordinateHelpField( + "point", + label="Coordinates (WKT)", + required=True, + ), + "elevation", + "county", + "state", + "quad_name", + "nma_notes_location", + "nma_coordinate_notes", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_location", + "nma_date_created", + "nma_site_date", + ] + + fields_default_sort = ["description", "point", "elevation", "county", "state"] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_location", + "nma_date_created", + "nma_site_date", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_pk_location", + "nma_date_created", + "nma_site_date", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Location ID", + "description": "Description", + "point": "Coordinates (WKT)", + "elevation": "Elevation (meters)", + "county": "County", + "state": "State", + "quad_name": "USGS Quad Name", + "release_status": "Release Status", + "nma_notes_location": "Location Notes", + "nma_coordinate_notes": "Coordinate Notes", + "nma_pk_location": "AMPAPI Location ID (Legacy)", + "nma_date_created": "AMPAPI Date Created (Legacy)", + "nma_site_date": "AMPAPI Site Date (Legacy)", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "description": "Brief description of this location (e.g., 'Well near Albuquerque')", + "elevation": "Elevation in meters. Vertical datum: NAVD88. Will be displayed in feet in reports.", + "release_status": "Data release status: 'draft' (internal only) or 'published' (public)", + "nma_notes_location": "General notes about this location", + "nma_coordinate_notes": "Notes about coordinate accuracy, source, or collection method", + "county": "New Mexico county name", + "state": "State (usually 'New Mexico')", + "quad_name": "USGS 7.5-minute quadrangle map name", + } diff --git a/admin/views/notes.py b/admin/views/notes.py new file mode 100644 index 000000000..1b2e81434 --- /dev/null +++ b/admin/views/notes.py @@ -0,0 +1,121 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +NotesAdmin view for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class NotesAdmin(OcotilloModelView): + """ + Admin view for Notes model. + """ + + # ========== Basic Configuration ========== + + name = "Notes" + label = "Notes" + icon = "fa fa-sticky-note" + + # ========== List View ========== + + column_list = [ + "id", + "target_table", + "target_id", + "note_type", + "content", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "target_table", + "target_id", + "note_type", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) + + search_fields = [ + "target_table", + "note_type", + "content", + ] + + column_filters = [ + "target_table", + "note_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "target_table", + "target_id", + "note_type", + "content", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Note ID", + "target_table": "Target Table", + "target_id": "Target ID", + "note_type": "Note Type", + "content": "Content", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/observation.py b/admin/views/observation.py new file mode 100644 index 000000000..8724a87dd --- /dev/null +++ b/admin/views/observation.py @@ -0,0 +1,177 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +ObservationAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Observation (Water Levels) model. +""" +from admin.views.base import OcotilloModelView + + +class ObservationAdmin(OcotilloModelView): + """ + Admin view for Observation model (Water Levels). + + Designed to replicate MS Access "Water Level Entry Form" and "Water Level Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all observations + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published observations (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Observations" + label = "Observations (Water Levels)" + icon = "fa fa-line-chart" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "observation_datetime", + "value", + "unit", + "measuring_point_height", + "groundwater_level_reason", + "release_status", + "created_at", + ] + + column_sortable_list = [ + "id", + "observation_datetime", + "value", + "unit", + "measuring_point_height", + "release_status", + "created_at", + ] + + column_default_sort = ( + "observation_datetime", + True, + ) # True = descending (newest first) + + search_fields = [ + "groundwater_level_reason", + "notes", + ] + + column_filters = [ + "observation_datetime", + "unit", + "groundwater_level_reason", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200, 500] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + # Core measurement data + "observation_datetime", + "value", + "unit", + "measuring_point_height", + "groundwater_level_reason", + "notes", + # Relationships (display as selects) + "sample_id", + "sensor_id", + "parameter_id", + "analysis_method_id", + # Release Status + "release_status", + # Audit Fields + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + # Legacy Migration Fields + "nma_pk_waterlevels", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_waterlevels", + # Exclude relationship objects (use IDs instead) + "sample", + "sensor", + "parameter", + "analysis_method", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_pk_waterlevels", + # Exclude relationship objects (use IDs instead) + "sample", + "sensor", + "parameter", + "analysis_method", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Observation ID", + "observation_datetime": "Date/Time Measured", + "value": "Depth to Water (ft)", + "unit": "Unit", + "measuring_point_height": "MP Height (ft)", + "groundwater_level_reason": "Level Status/Reason", + "notes": "Notes", + "sample_id": "Sample", + "sensor_id": "Sensor/Equipment", + "parameter_id": "Parameter", + "analysis_method_id": "Analysis Method", + "release_status": "Release Status", + "nma_pk_waterlevels": "AMPAPI WaterLevels ID (Legacy)", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "observation_datetime": "Date and time of the water level measurement (UTC)", + "value": "Depth to water from measuring point (feet)", + "unit": "Unit of measurement (typically 'ft' for feet)", + "measuring_point_height": "Height of measuring point above ground surface (feet)", + "groundwater_level_reason": "Reason/status: obstruction, dry well, equipment failure, etc. Leave blank if normal measurement.", + "notes": "Additional notes about this observation", + "sample_id": "Associated sample record", + "sensor_id": "Equipment used to take measurement (if automated)", + "parameter_id": "The parameter being measured (e.g., 'Depth to Water')", + "release_status": "'draft' (internal only) or 'published' (public)", + } diff --git a/admin/views/parameter.py b/admin/views/parameter.py new file mode 100644 index 000000000..a132c0aae --- /dev/null +++ b/admin/views/parameter.py @@ -0,0 +1,120 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +ParameterAdmin view for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class ParameterAdmin(OcotilloModelView): + """ + Admin view for Parameter model. + """ + + name = "Parameters" + label = "Parameters" + icon = "fa fa-flask" + + column_list = [ + "id", + "parameter_name", + "matrix", + "parameter_type", + "cas_number", + "default_unit", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "parameter_name", + "matrix", + "parameter_type", + "cas_number", + "default_unit", + "release_status", + "created_at", + ] + + column_default_sort = ("parameter_name", False) + + search_fields = [ + "parameter_name", + "cas_number", + ] + + column_filters = [ + "matrix", + "parameter_type", + "default_unit", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + fields = [ + "id", + "parameter_name", + "matrix", + "parameter_type", + "cas_number", + "default_unit", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Parameter ID", + "parameter_name": "Parameter Name", + "matrix": "Matrix", + "parameter_type": "Parameter Type", + "cas_number": "CAS Number", + "default_unit": "Default Unit", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/sample.py b/admin/views/sample.py new file mode 100644 index 000000000..872797a11 --- /dev/null +++ b/admin/views/sample.py @@ -0,0 +1,139 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +SampleAdmin view for NMSampleLocations. +""" +from admin.views.base import OcotilloModelView + + +class SampleAdmin(OcotilloModelView): + """ + Admin view for Sample model. + """ + + # ========== Basic Configuration ========== + + name = "Samples" + label = "Samples" + icon = "fa fa-flask" + + # ========== List View ========== + + column_list = [ + "id", + "sample_name", + "sample_date", + "sample_matrix", + "sample_method", + "qc_type", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "sample_name", + "sample_date", + "sample_matrix", + "sample_method", + "qc_type", + "release_status", + "created_at", + ] + + column_default_sort = ("sample_date", True) + + search_fields = [ + "sample_name", + "notes", + "nma_pk_waterlevels", + ] + + column_filters = [ + "sample_matrix", + "sample_method", + "qc_type", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "field_activity_id", + "field_event_participant_id", + "sample_date", + "sample_name", + "sample_matrix", + "sample_method", + "qc_type", + "depth_top", + "depth_bottom", + "notes", + "nma_pk_waterlevels", + "release_status", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + ] + + labels = { + "id": "Sample ID", + "field_activity_id": "Field Activity", + "field_event_participant_id": "Field Event Participant", + "sample_date": "Sample Date", + "sample_name": "Sample Name", + "sample_matrix": "Sample Matrix", + "sample_method": "Sample Method", + "qc_type": "QC Type", + "depth_top": "Depth Top (ft)", + "depth_bottom": "Depth Bottom (ft)", + "notes": "Notes", + "nma_pk_waterlevels": "AMPAPI WaterLevels ID (Legacy)", + "release_status": "Release Status", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/admin/views/sensor.py b/admin/views/sensor.py new file mode 100644 index 000000000..a0dedda5a --- /dev/null +++ b/admin/views/sensor.py @@ -0,0 +1,170 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +SensorAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Sensor (Equipment) model. +""" +from admin.views.base import OcotilloModelView + + +class SensorAdmin(OcotilloModelView): + """ + Admin view for Sensor model (Equipment). + + Designed to replicate MS Access "Equipment Entry Form" and "Equipment Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all sensors + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published sensors (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Sensors" + label = "Sensors (Equipment)" + icon = "fa fa-microchip" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "name", + "sensor_type", + "model", + "serial_no", + "pcn_number", + "owner_agency", + "sensor_status", + "release_status", + "created_at", + ] + + column_sortable_list = [ + "id", + "name", + "sensor_type", + "model", + "serial_no", + "owner_agency", + "sensor_status", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) # True = descending + + search_fields = [ + "name", + "serial_no", + "model", + "pcn_number", + ] + + column_filters = [ + "sensor_type", + "owner_agency", + "sensor_status", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + # Equipment Information + "name", + "sensor_type", + "model", + "serial_no", + "pcn_number", + "owner_agency", + "sensor_status", + "notes", + # Release Status + "release_status", + # Audit Fields + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + # Legacy Migration Fields + "nma_pk_equipment", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_equipment", + # Exclude complex relationships + "observations", + "deployments", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_pk_equipment", + # Exclude complex relationships + "observations", + "deployments", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Sensor ID", + "name": "Name", + "sensor_type": "Type", + "model": "Model", + "serial_no": "Serial Number", + "pcn_number": "PCN Number", + "owner_agency": "Owner Agency", + "sensor_status": "Status", + "notes": "Notes", + "release_status": "Release Status", + "nma_pk_equipment": "AMPAPI Equipment ID (Legacy)", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "name": "Name or identifier for this sensor/equipment", + "sensor_type": "Type of equipment: 'Pressure Transducer', 'Acoustic Sounder', 'Data Logger', etc.", + "model": "Manufacturer model number", + "serial_no": "Equipment serial number (must be unique)", + "pcn_number": "Property Control Number for NMBGMR-owned equipment", + "owner_agency": "Agency/organization that owns this equipment", + "sensor_status": "Current status: 'In Service', 'In Repair', 'Retired', 'Lost', etc.", + "notes": "General notes about this equipment", + "release_status": "'draft' (internal only) or 'published' (public)", + } diff --git a/admin/views/thing.py b/admin/views/thing.py new file mode 100644 index 000000000..875b2d104 --- /dev/null +++ b/admin/views/thing.py @@ -0,0 +1,223 @@ +# =============================================================================== +# Copyright 2025 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +ThingAdmin view for NMSampleLocations. + +Provides MS Access-like interface for CRUD operations on Thing (Wells/Springs) model. +""" +from admin.views.base import OcotilloModelView +from db.thing import Thing + + +class ThingAdmin(OcotilloModelView): + """ + Admin view for Thing model (Wells, Springs, etc.). + + Designed to replicate MS Access "Well Data Entry Form" and "Well Datasheet View". + + Permission Model: + - Admin: Can create, edit, delete all things + - Editor: Can create and edit, cannot delete + - Viewer: Can only view published things (read-only) + """ + + # ========== Basic Configuration ========== + + name = "Things" + label = "Things (Wells/Springs)" + icon = "fa fa-tint" + + # ========== List View (MS Access Datasheet View Equivalent) ========== + + column_list = [ + "id", + "name", + "thing_type", + "well_depth", + "hole_depth", + "first_visit_date", + "release_status", + "created_at", + "updated_by_name", + ] + + column_sortable_list = [ + "id", + "name", + "thing_type", + "well_depth", + "hole_depth", + "first_visit_date", + "release_status", + "created_at", + ] + + column_default_sort = ("created_at", True) # True = descending + + search_fields = [ + "name", + "thing_type", + "well_driller_name", + ] + + column_filters = [ + "thing_type", + "well_depth", + "first_visit_date", + "release_status", + "created_at", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View (MS Access Form View Equivalent) ========== + + fields = [ + "id", + # Basic Information + "name", + "thing_type", + "first_visit_date", + # Well Construction + "well_depth", + "hole_depth", + "well_casing_diameter", + "well_casing_depth", + "well_completion_date", + "well_driller_name", + "well_construction_method", + "well_pump_type", + "well_pump_depth", + "formation_completion_code", + "is_suitable_for_datalogger", + # Spring-specific + "spring_type", + # Release Status + "release_status", + # Audit Fields + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + # Legacy Migration Fields + "nma_pk_welldata", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_pk_welldata", + # Exclude complex relationships from create form + "location_associations", + "contact_associations", + "asset_associations", + "field_events", + "deployments", + "group_associations", + "screens", + "well_purposes", + "well_casing_materials", + "links", + "measuring_points", + "monitoring_frequencies", + "aquifer_associations", + "formation_associations", + "status_history", + "permission_history", + "data_provenance", + "notes", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_pk_welldata", + # Exclude complex relationships from edit form (manage separately) + "location_associations", + "contact_associations", + "asset_associations", + "field_events", + "deployments", + "group_associations", + "screens", + "well_purposes", + "well_casing_materials", + "links", + "measuring_points", + "monitoring_frequencies", + "aquifer_associations", + "formation_associations", + "status_history", + "permission_history", + "data_provenance", + "notes", + ] + + # ========== Field Labels and Help Text ========== + + labels = { + "id": "Thing ID", + "name": "PointID", + "thing_type": "Type", + "first_visit_date": "First Visit Date", + "well_depth": "Well Depth (ft)", + "hole_depth": "Hole Depth (ft)", + "well_casing_diameter": "Casing Diameter (in)", + "well_casing_depth": "Casing Depth (ft)", + "well_completion_date": "Completion Date", + "well_driller_name": "Driller Name", + "well_construction_method": "Construction Method", + "well_pump_type": "Pump Type", + "well_pump_depth": "Pump Depth (ft)", + "formation_completion_code": "Formation Code", + "is_suitable_for_datalogger": "Datalogger OK?", + "spring_type": "Spring Type", + "release_status": "Release Status", + "nma_pk_welldata": "AMPAPI WellData ID (Legacy)", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + help_texts = { + "name": "Unique identifier for this well/spring (PointID from legacy system)", + "thing_type": "Type of infrastructure: 'water well', 'spring', etc.", + "first_visit_date": "Date of NMBGMR's first recorded interaction", + "well_depth": "Total depth of well from ground surface to bottom (feet)", + "hole_depth": "Depth of drilled hole from ground surface (feet)", + "well_casing_diameter": "Diameter of well casing (inches)", + "well_casing_depth": "Depth of casing from ground surface (feet)", + "well_completion_date": "Date the well was completed", + "well_driller_name": "Name of the well driller", + "well_construction_method": "Method used to construct the well", + "well_pump_type": "Type of pump installed", + "well_pump_depth": "Depth of pump intake from ground surface (feet)", + "formation_completion_code": "Geologic formation where well was completed", + "is_suitable_for_datalogger": "Can a datalogger be installed at this well?", + "spring_type": "Type of spring (for springs only)", + "release_status": "'draft' (internal only) or 'published' (public)", + } diff --git a/alembic/versions/2101e0b029dc_make_location_description_nullable.py b/alembic/versions/2101e0b029dc_make_location_description_nullable.py index efe5b413f..f190a426d 100644 --- a/alembic/versions/2101e0b029dc_make_location_description_nullable.py +++ b/alembic/versions/2101e0b029dc_make_location_description_nullable.py @@ -14,12 +14,6 @@ import sqlalchemy_utils -def _column_exists(bind, table: str, column: str) -> bool: - inspector = sa.inspect(bind) - cols = [c["name"] for c in inspector.get_columns(table)] - return column in cols - - # revision identifiers, used by Alembic. revision: str = "2101e0b029dc" down_revision: Union[str, Sequence[str], None] = "66ac1af4ba69" @@ -33,20 +27,11 @@ def upgrade() -> None: Makes the location.description column nullable to accommodate legacy data from MS Access that may not have descriptions. """ - bind = op.get_bind() - if _column_exists(bind, "location", "description"): - op.alter_column( - "location", "description", existing_type=sa.String(), nullable=True - ) - else: - # If the column is absent (non-standard schema), skip the alteration. - pass + op.alter_column("location", "description", existing_type=sa.String(), nullable=True) def downgrade() -> None: """Downgrade schema.""" - bind = op.get_bind() - if _column_exists(bind, "location", "description"): - op.alter_column( - "location", "description", existing_type=sa.String(), nullable=False - ) + op.alter_column( + "location", "description", existing_type=sa.String(), nullable=False + ) diff --git a/core/app.py b/core/app.py index b0e0184fe..a0acc2539 100644 --- a/core/app.py +++ b/core/app.py @@ -24,6 +24,7 @@ ) from fastapi.openapi.utils import get_openapi +from transfers.seed import seed_all from .initializers import ( register_routes, erase_and_rebuild_db, @@ -38,6 +39,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """ if settings.get_enum("MODE") == "development": erase_and_rebuild_db() + seed_all(10) register_routes(app) yield diff --git a/core/lexicon.json b/core/lexicon.json index 90ead61b9..987024724 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -750,6 +750,10 @@ {"categories": ["sensor_type"], "term": "Camera", "definition": "Camera"}, {"categories": ["sensor_type"], "term": "Soil Moisture Sensor", "definition": "Soil Moisture Sensor"}, {"categories": ["sensor_type"], "term": "Tipping Bucket", "definition": "Tipping Bucket"}, + {"categories": ["sensor_type"], "term": "Weather Station", "definition": "Weather Station"}, + {"categories": ["sensor_type"], "term": "Weir", "definition": "Weir for stream flow measurement"}, + {"categories": ["sensor_type"], "term": "Snow Lysimeter", "definition": "Snow Lysimeter for snowmelt measurement"}, + {"categories": ["sensor_type"], "term": "Lysimeter", "definition": "Lysimeter for soil water measurement"}, {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, diff --git a/db/engine.py b/db/engine.py index bc177eb8e..ce31aec5f 100644 --- a/db/engine.py +++ b/db/engine.py @@ -90,10 +90,17 @@ def getconn(): ) return conn + # Configure connection pool for parallel transfers + pool_size = int(os.environ.get("DB_POOL_SIZE", "10")) + max_overflow = int(os.environ.get("DB_MAX_OVERFLOW", "20")) + engine = create_engine( "postgresql+pg8000://", creator=getconn, echo=False, + pool_size=pool_size, + max_overflow=max_overflow, + pool_pre_ping=True, ) return engine @@ -120,10 +127,19 @@ def getconn(): # else: # url = "sqlite:///./development.db" + # Configure connection pool for parallel transfers + # pool_size: number of persistent connections + # max_overflow: additional connections during peak usage + pool_size = int(os.environ.get("DB_POOL_SIZE", "10")) + max_overflow = int(os.environ.get("DB_MAX_OVERFLOW", "20")) + engine = create_engine( url, # echo=True, plugins=["geoalchemy2"], + pool_size=pool_size, + max_overflow=max_overflow, + pool_pre_ping=True, # Verify connections before use ) async_engine = create_async_engine( diff --git a/db/location.py b/db/location.py index 2fbeecc82..f748beb7f 100644 --- a/db/location.py +++ b/db/location.py @@ -43,7 +43,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin, NotesMixin, DataProvenanceMixi __versioned__ = {} nma_pk_location: Mapped[UUID] = mapped_column(String(36), nullable=True) - description: Mapped[str] = mapped_column + description: Mapped[str] = mapped_column(nullable=True) # name: Mapped[str] = mapped_column(String(255), nullable=True) point: Mapped[WKBElement] = mapped_column( Geometry(geometry_type="POINT", srid=SRID_WGS84, spatial_index=True) diff --git a/db/transducer.py b/db/transducer.py index 00fb1e3a1..25928c9ca 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -33,13 +33,19 @@ if TYPE_CHECKING: from db.parameter import Parameter from db.contact import Contact + from db.thing import Thing class TransducerObservationBlock(Base, AutoBaseMixin, ReleaseMixin): """ Represents a contiguous block of transducer observations that share a QC status. + Each block is associated with a specific Thing (well) to ensure uniqueness. """ + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False, index=True + ) + parameter_id: Mapped[int] = mapped_column( ForeignKey("parameter.id", ondelete="CASCADE"), nullable=False, index=True ) @@ -60,16 +66,18 @@ class TransducerObservationBlock(Base, AutoBaseMixin, ReleaseMixin): comment="Foreign key to the Contact table", ) + thing: Mapped["Thing"] = relationship("Thing") parameter: Mapped["Parameter"] = relationship("Parameter") reviewer: Mapped["Contact"] = relationship("Contact") __table_args__ = ( UniqueConstraint( + "thing_id", "review_status", "parameter_id", "start_datetime", "end_datetime", - name="uq_transducer_block_status_parameter_time", + name="uq_transducer_block_thing_status_parameter_time", ), CheckConstraint( "end_datetime > start_datetime", name="check_transuder_block_time_order" diff --git a/features/admin/README.md b/features/admin/README.md new file mode 100644 index 000000000..39d02cceb --- /dev/null +++ b/features/admin/README.md @@ -0,0 +1,274 @@ +# Admin Interface BDD Feature Tests + +This directory contains Behavior-Driven Development (BDD) feature files for the Starlette Admin interface. + +## Features + +### `authentication.feature` +Documents authentication and authorization business rules: +- Authentik OIDC integration +- Role-based access control (Admin/Editor/Viewer) +- JWT token verification +- Session management +- Security constraints + +**Coverage:** +- 11 scenarios +- Authentication flows +- RBAC permissions +- Development mode behavior + +### `location_admin.feature` +Documents Location admin CRUD operations and business rules: +- List view functionality (search, filter, sort, pagination) +- Create/edit forms with WKT coordinate input +- Bulk operations (publish/unpublish) +- Data visibility by release status +- Audit trail (created_by, updated_by) +- MS Access equivalent operations + +**Coverage:** +- 24 scenarios +- Full CRUD lifecycle +- Validation rules +- Permission checks +- Data visibility rules + +## Running the Tests + +### Prerequisites + +1. **Test Database**: PostgreSQL + PostGIS test database + ```bash + # Create test database + createdb nmsamplelocations_test + psql nmsamplelocations_test -c "CREATE EXTENSION postgis;" + + # Run migrations + alembic upgrade head + ``` + +2. **Environment Variables**: Create `.env.test` file: + ```bash + DATABASE_URL=postgresql://user:password@localhost/nmsamplelocations_test + AUTHENTIK_URL=https://auth.example.com + AUTHENTIK_CLIENT_ID=test_client_id + AUTHENTIK_DISABLE_AUTHENTICATION=1 # For testing + MODE=development + ``` + +3. **Python Dependencies**: + ```bash + uv add --dev behave playwright pytest-playwright + uv sync + ``` + +### Run Tests + +```bash +# From NMSampleLocations directory +cd /path/to/NMSampleLocations + +# Run all admin feature tests +behave features/admin/ + +# Run specific feature +behave features/admin/authentication.feature +behave features/admin/location_admin.feature + +# Run with tags +behave features/admin/ --tags=@smoke +behave features/admin/ --tags=@rbac +behave features/admin/ --tags=@bulk-actions + +# Verbose output +behave features/admin/ -v + +# Progress format (dots) +behave features/admin/ --format progress +``` + +## Test Data Cleanup + +The `environment.py` file handles automatic cleanup: +- **before_scenario**: Creates new database session, sets up test user +- **after_scenario**: Deletes all test data created during scenario +- Test data is tracked in `context.scenario_data_ids` +- Ensures test isolation between scenarios + +## Step Definitions + +### Browser-Based Tests (Playwright) + +**File**: `steps/admin_ui_steps.py` + +Uses Playwright for browser automation: +- Tests actual admin UI in a real browser +- Clicks buttons, fills forms, verifies page content +- Tests JavaScript interactions +- Slow but comprehensive + +**Example**: +```python +@when('I navigate to "/admin/location"') +def step_navigate_to_admin_location(context): + context.page.goto(f"{context.base_url}/admin/location") + +@then('I should see the "Create" button') +def step_see_create_button(context): + create_button = context.page.locator('button:has-text("Create")') + expect(create_button).to_be_visible() +``` + +### API-Based Tests (FastAPI TestClient) + +**File**: `steps/admin_api_steps.py` + +Uses FastAPI TestClient for direct API testing: +- Tests admin backend without browser +- Fast execution +- Tests data operations directly +- Good for validation and permission tests + +**Example**: +```python +@given('the following locations exist') +def step_create_test_locations(context): + for row in context.table: + location = Location( + description=row['description'], + point=WKTElement(f"POINT(-106.0 35.0)", srid=4326), + elevation=1500.0, + **row + ) + context.session.add(location) + context.session.commit() +``` + +## Test Tags + +| Tag | Description | +|-----|-------------| +| `@admin` | All admin interface tests | +| `@authentication` | Authentication/authorization tests | +| `@location` | Location-specific tests | +| `@smoke` | Critical smoke tests (run first) | +| `@rbac` | Role-based access control tests | +| `@list-view` | List view functionality | +| `@create` | Create operation tests | +| `@update` | Update operation tests | +| `@delete` | Delete operation tests | +| `@bulk-actions` | Bulk action tests | +| `@data-visibility` | Release status filtering tests | +| `@validation` | Form validation tests | +| `@permissions` | Permission check tests | +| `@security` | Security-related tests | +| `@ms-access-migration` | MS Access equivalent features | + +## Test Organization + +### Feature File Structure + +```gherkin +@tag +Feature: Feature name + As a + I need to + So that + + Background: + Given + + @scenario-tags + Scenario: Scenario description + Given + When + Then +``` + +### Writing New Scenarios + +1. **Start with business language**: Use terms staff understand (not technical jargon) +2. **Use MS Access equivalents**: Reference familiar concepts (e.g., "Datasheet View") +3. **Tag appropriately**: Use `@smoke` for critical paths, specific tags for features +4. **Keep scenarios focused**: One scenario tests one thing +5. **Use scenario outlines**: For testing multiple similar cases with different data + +## Example Test Run + +```bash +$ behave features/admin/ --tags=@smoke + +Feature: Admin Authentication and Authorization + + Scenario: Unauthenticated user is redirected to login + Given I am not authenticated + When I navigate to "/admin" + Then I should be redirected to the Authentik login page + ... + ✓ Passed + + Scenario: Authenticated admin user can access admin interface + Given I am authenticated as user "admin@nmbgmr.nmt.edu" + ... + ✓ Passed + +Feature: Location Admin CRUD Operations + + Scenario: View location list with default columns + When I navigate to "/admin/location" + ... + ✓ Passed + + Scenario: Create a new location with valid data + When I navigate to "/admin/location" + And I click the "Create" button + ... + ✓ Passed + +4 scenarios (4 passed) +28 steps (28 passed) +0m15.234s +``` + +## Troubleshooting + +### "Playwright not found" error +```bash +# Install Playwright browsers +playwright install chromium +``` + +### "Database connection refused" +```bash +# Check PostgreSQL is running +pg_isready + +# Check DATABASE_URL in .env.test +``` + +### "Authentik authentication errors" +```bash +# Set to development mode in tests +export AUTHENTIK_DISABLE_AUTHENTICATION=1 +export MODE=development +``` + +### Tests fail but manual testing works +- Check test data cleanup (orphaned data) +- Verify test isolation (scenarios affecting each other) +- Check for race conditions (async operations) + +## Next Steps + +1. **Implement step definitions**: Create `steps/admin_ui_steps.py` and `steps/admin_api_steps.py` +2. **Set up environment.py**: Configure Behave hooks for setup/teardown +3. **Add CI integration**: Run tests in GitHub Actions +4. **Expand coverage**: Add features for Thing, Sample, Observation admin views + +## Related Documentation + +- [Behave Documentation](https://behave.readthedocs.io/) +- [Playwright for Python](https://playwright.dev/python/) +- [Starlette Admin Docs](https://jowilf.github.io/starlette-admin/) +- [AMPAPI BDD Tests](../../AMPAPI/features/README.md) - Similar pattern diff --git a/features/admin/authentication.feature b/features/admin/authentication.feature new file mode 100644 index 000000000..7a9cf5463 --- /dev/null +++ b/features/admin/authentication.feature @@ -0,0 +1,96 @@ +@admin @authentication +Feature: Admin Authentication and Authorization + As a data manager transitioning from MS Access + I need to authenticate and be authorized to use the admin interface + So that I can manage location data securely via web browser + + Background: + Given the admin interface is mounted at "/admin" + And Authentik OIDC authentication is configured + + @smoke + Scenario: Unauthenticated user is redirected to login + Given I am not authenticated + When I navigate to "/admin" + Then I should be redirected to the Authentik login page + And the login page should display "Sign in with Authentik" + + @smoke + Scenario: Authenticated admin user can access admin interface + Given I am authenticated as user "admin@nmbgmr.nmt.edu" + And my user has the "Admin" group + When I navigate to "/admin" + Then I should see the admin dashboard + And I should see "Locations" in the sidebar menu + And I should see my username "admin@nmbgmr.nmt.edu" in the header + + Scenario: Editor user has limited permissions + Given I am authenticated as user "editor@nmbgmr.nmt.edu" + And my user has the "Editor" group + When I navigate to "/admin/location" + Then I should see the "Create" button + But I should not see the "Delete" button on any row + + Scenario: Viewer user has read-only access + Given I am authenticated as user "viewer@nmbgmr.nmt.edu" + And my user has the "Viewer" group + When I navigate to "/admin/location" + Then I should not see the "Create" button + And I should not see the "Edit" button on any row + And I should not see the "Delete" button on any row + And I should only see locations with release_status "published" + + @rbac + Scenario Outline: Role-based access to location operations + Given I am authenticated as user "" + And my user has the "" group + When I navigate to "/admin/location" + Then I see the "Create" button + And I see the "Edit" button on rows + And I see the "Delete" button on rows + And I see draft locations + + Examples: + | role | email | can_create | can_edit | can_delete | visibility | + | Admin | admin@nmbgmr.nmt.edu | should | should | should | should | + | Editor | editor@nmbgmr.nmt.edu | should | should | should not | should | + | Viewer | viewer@nmbgmr.nmt.edu | should not | should not | should not | should not | + + Scenario: Logout clears session and redirects + Given I am authenticated as user "admin@nmbgmr.nmt.edu" + And I am on the admin dashboard at "/admin" + When I click the "Logout" button + Then my session token should be cleared + And I should be redirected to "/" + And I should not be able to access "/admin" without re-authenticating + + @security + Scenario: Invalid JWT token is rejected + Given I have an invalid JWT token + When I attempt to access "/admin/location" + Then I should receive a 401 Unauthorized response + And I should be redirected to the login page + + @security + Scenario: Expired JWT token requires re-authentication + Given I am authenticated as user "admin@nmbgmr.nmt.edu" + And my JWT token has expired + When I navigate to "/admin/location" + Then I should be redirected to the Authentik login page + And I should see a message "Session expired, please log in again" + + @development-mode + Scenario: Development mode bypasses authentication + Given the environment variable "AUTHENTIK_DISABLE_AUTHENTICATION" is set to "1" + And the application mode is "development" + When I navigate to "/admin/location" + Then I should see the admin interface without logging in + And I should have "admin" role by default + + @security + Scenario: Authentication bypass is blocked in production + Given the environment variable "AUTHENTIK_DISABLE_AUTHENTICATION" is set to "1" + And the application mode is "production" + When I attempt to access "/admin/location" + Then I should receive a 424 Failed Dependency response + And I should see an error "Authentication is disabled in production mode" diff --git a/features/admin/location_admin.feature b/features/admin/location_admin.feature new file mode 100644 index 000000000..29a53dcd2 --- /dev/null +++ b/features/admin/location_admin.feature @@ -0,0 +1,329 @@ +@admin @location +Feature: Location Admin CRUD Operations + As a data manager who previously used MS Access + I need to create, read, update, and delete location records via the web admin interface + So that I can manage location data without opening an Access database file + + Background: + Given I am authenticated as user "admin@nmbgmr.nmt.edu" with "Admin" role + And the admin interface is available at "/admin" + + # ========== List View (MS Access Datasheet Equivalent) ========== + + @smoke @list-view + Scenario: View location list with default columns + When I navigate to "/admin/location" + Then I should see the location list page + And I should see the following columns: + | Column Name | + | Location ID | + | Description | + | County | + | State | + | Elevation | + | Quad Name | + | Release Status | + | Created At | + | Updated By | + And the list should be sorted by "Created At" descending by default + + @list-view @search + Scenario: Search locations by description + Given the following locations exist: + | description | county | state | + | Well near Albuquerque | Bernalillo | New Mexico | + | Spring in Santa Fe | Santa Fe | New Mexico | + | Test well in Rio Rancho | Sandoval | New Mexico | + When I navigate to "/admin/location" + And I enter "Albuquerque" in the search box + Then I should see 1 location in the results + And I should see "Well near Albuquerque" in the results + But I should not see "Spring in Santa Fe" in the results + + @list-view @filter + Scenario: Filter locations by county + Given the following locations exist: + | description | county | release_status | + | Well A | Bernalillo | published | + | Well B | Bernalillo | draft | + | Well C | Santa Fe | published | + When I navigate to "/admin/location" + And I select "Bernalillo" from the "County" filter + Then I should see 2 locations in the results + And I should see "Well A" in the results + And I should see "Well B" in the results + But I should not see "Well C" in the results + + @list-view @filter + Scenario: Filter locations by release status + Given the following locations exist: + | description | release_status | + | Published 1 | published | + | Published 2 | published | + | Draft 1 | draft | + When I navigate to "/admin/location" + And I select "published" from the "Release Status" filter + Then I should see 2 locations in the results + And I should see "Published 1" in the results + And I should see "Published 2" in the results + But I should not see "Draft 1" in the results + + @list-view @sorting + Scenario: Sort locations by column + Given the following locations exist: + | description | elevation | + | High Point | 2500.0 | + | Low Point | 1200.0 | + | Mid Point | 1800.0 | + When I navigate to "/admin/location" + And I click the "Elevation" column header + Then the locations should be sorted by elevation ascending + And I should see "Low Point" as the first result + And I should see "High Point" as the last result + + @list-view @pagination + Scenario: Paginate through location list + Given 75 locations exist in the database + When I navigate to "/admin/location" + Then I should see 50 locations on page 1 + And I should see a "Next" pagination button + When I click the "Next" button + Then I should see 25 locations on page 2 + And I should see a "Previous" pagination button + + @list-view @export + Scenario: Export locations to CSV + Given the following locations exist: + | description | county | elevation | + | Well 1 | Bernalillo | 1500.0 | + | Well 2 | Santa Fe | 2000.0 | + When I navigate to "/admin/location" + And I click the "Export" button + And I select "CSV" as the export format + Then a CSV file should be downloaded + And the CSV file should contain 2 data rows plus 1 header row + And the CSV should include columns: "id,description,county,state,elevation,release_status" + + # ========== Create Operation (MS Access Form Equivalent) ========== + + @smoke @create + Scenario: Create a new location with valid data + When I navigate to "/admin/location" + And I click the "Create" button + Then I should see the location creation form + When I fill in the form: + | Field | Value | + | Description | Test well near Albuquerque | + | Coordinates (WKT) | POINT(-106.65082 35.08352) | + | Elevation | 1500.5 | + | County | Bernalillo | + | State | New Mexico | + | Quad Name | Albuquerque West | + | Release Status | draft | + And I click the "Save" button + Then I should see a success message "Location created successfully" + And I should be redirected to the location list + And I should see "Test well near Albuquerque" in the location list + + @create @validation + Scenario: Create location fails with invalid WKT format + When I navigate to "/admin/location/create" + And I fill in the form: + | Field | Value | + | Description | Invalid coordinates test | + | Coordinates (WKT) | INVALID WKT STRING | + | Elevation | 1500.0 | + And I click the "Save" button + Then I should see a validation error "Invalid WKT geometry" + And the error should explain "Expected format: POINT(longitude latitude)" + And the location should not be created + + @create @validation + Scenario: Create location fails with longitude/latitude reversed + When I navigate to "/admin/location/create" + And I fill in the form: + | Field | Value | + | Description | Reversed coordinates test | + | Coordinates (WKT) | POINT(35.08352 -106.65082) | + | Elevation | 1500.0 | + And I click the "Save" button + Then I should see a warning "Are you sure these coordinates are correct? Longitude should be negative for New Mexico." + # Note: This is a soft validation - staff can override if intentional + + @create @ms-access-migration + Scenario: WKT field provides help for UTM conversion + When I navigate to "/admin/location/create" + And I hover over the "Coordinates (WKT)" field + Then I should see help text explaining: + """ + Enter coordinates in WKT (Well-Known Text) format. + + Format: POINT(longitude latitude) + Example: POINT(-106.65082 35.08352) + + Important: + " Longitude comes FIRST (negative for western hemisphere) + " Latitude comes SECOND + " No comma between values + + If you have UTM coordinates, use an online converter first. + """ + + @create @required-fields + Scenario: Create location fails with missing required fields + When I navigate to "/admin/location/create" + And I click the "Save" button without filling any fields + Then I should see validation errors: + | Field | Error | + | Description | This field is required | + | Coordinates (WKT) | This field is required | + | Elevation | This field is required | + + # ========== Read/Detail View ========== + + @read @detail-view + Scenario: View location details + Given a location exists with: + | Field | Value | + | description | Test Well ABC | + | point | POINT(-106.65082 35.08352) | + | elevation | 1500.5 | + | county | Bernalillo | + | state | New Mexico | + | release_status | draft | + | nma_pk_location | 550e8400-e29b-41d4-a716-446655440000 | + When I navigate to the location detail page + Then I should see all location fields displayed + And I should see "POINT(-106.65082 35.08352)" for coordinates + And I should see "1500.5" for elevation + And I should see read-only legacy fields: + | Field | Value | + | AMPAPI Location ID (Legacy) | 550e8400-e29b-41d4-a716-446655440000 | + | AMPAPI Date Created (Legacy) | (if populated during migration) | + + # ========== Update Operation ========== + + @update + Scenario: Edit an existing location + Given a location exists with description "Old description" + When I navigate to the location edit page + And I update the form: + | Field | Value | + | Description | New description | + And I click the "Save" button + Then I should see a success message "Location updated successfully" + And the location should have description "New description" + + @update @audit + Scenario: Editing a location updates audit fields + Given a location exists with description "Test Location" + And the location was created by "admin@nmbgmr.nmt.edu" + When I am authenticated as "editor@nmbgmr.nmt.edu" with "Editor" role + And I navigate to the location edit page + And I update the description to "Updated Location" + And I click the "Save" button + Then the location's "updated_by_name" should be "editor@nmbgmr.nmt.edu" + And the location's "created_by_name" should still be "admin@nmbgmr.nmt.edu" + + @update @read-only-fields + Scenario: Legacy migration fields cannot be edited + Given a location exists with nma_pk_location "550e8400-e29b-41d4-a716-446655440000" + When I navigate to the location edit page + Then the "AMPAPI Location ID" field should be read-only + And the "AMPAPI Date Created" field should be read-only + And the "AMPAPI Site Date" field should be read-only + + # ========== Delete Operation ========== + + @delete @permissions + Scenario: Admin can delete a location + Given a location exists with description "Location to delete" + And I am authenticated as "admin@nmbgmr.nmt.edu" with "Admin" role + When I navigate to the location detail page + And I click the "Delete" button + Then I should see a confirmation dialog "Are you sure you want to delete this location?" + When I confirm the deletion + Then I should see a success message "Location deleted successfully" + And the location should no longer exist in the database + + @delete @permissions + Scenario: Editor cannot delete a location + Given a location exists with description "Location to delete" + And I am authenticated as "editor@nmbgmr.nmt.edu" with "Editor" role + When I navigate to the location detail page + Then I should not see a "Delete" button + + # ========== Bulk Actions (MS Access Update Query Equivalent) ========== + + @bulk-actions @publish + Scenario: Bulk publish selected locations + Given the following locations exist: + | description | release_status | + | Location 1 | draft | + | Location 2 | draft | + | Location 3 | published | + And I am authenticated as "admin@nmbgmr.nmt.edu" with "Admin" role + When I navigate to "/admin/location" + And I select locations "Location 1" and "Location 2" using checkboxes + And I select "Publish Selected" from the actions dropdown + And I click "Execute" button + Then I should see a confirmation "Are you sure you want to publish the selected locations?" + When I confirm the action + Then I should see "Successfully published 2 location(s)" + And "Location 1" should have release_status "published" + And "Location 2" should have release_status "published" + + @bulk-actions @unpublish + Scenario: Bulk unpublish selected locations + Given the following locations exist: + | description | release_status | + | Location 1 | published | + | Location 2 | published | + And I am authenticated as "admin@nmbgmr.nmt.edu" with "Admin" role + When I navigate to "/admin/location" + And I select locations "Location 1" and "Location 2" using checkboxes + And I select "Unpublish Selected" from the actions dropdown + And I click "Execute" button + Then I should see "Successfully unpublished 2 location(s)" + And "Location 1" should have release_status "draft" + And "Location 2" should have release_status "draft" + + @bulk-actions @permissions + Scenario: Editor cannot perform bulk publish action + Given the following locations exist: + | description | release_status | + | Location 1 | draft | + And I am authenticated as "editor@nmbgmr.nmt.edu" with "Editor" role + When I navigate to "/admin/location" + And I select location "Location 1" using checkbox + And I attempt to execute "Publish Selected" action + Then I should see an error "Only admins can publish locations" + And "Location 1" should still have release_status "draft" + + # ========== Data Visibility (Release Status Filtering) ========== + + @data-visibility @viewer + Scenario: Viewer only sees published locations + Given the following locations exist: + | description | release_status | + | Public Well | published | + | Draft Well | draft | + | Internal Well | draft | + And I am authenticated as "viewer@nmbgmr.nmt.edu" with "Viewer" role + When I navigate to "/admin/location" + Then I should see 1 location in the results + And I should see "Public Well" in the results + But I should not see "Draft Well" in the results + And I should not see "Internal Well" in the results + + @data-visibility @editor + Scenario: Editor sees all locations (draft and published) + Given the following locations exist: + | description | release_status | + | Public Well | published | + | Draft Well | draft | + And I am authenticated as "editor@nmbgmr.nmt.edu" with "Editor" role + When I navigate to "/admin/location" + Then I should see 2 locations in the results + And I should see "Public Well" in the results + And I should see "Draft Well" in the results diff --git a/main.py b/main.py index 1effe425e..f56a99e60 100644 --- a/main.py +++ b/main.py @@ -26,11 +26,27 @@ from starlette.middleware.cors import CORSMiddleware +from starlette.middleware.sessions import SessionMiddleware from core.app import app register_routes(app) +# Session middleware is required for the admin auth flow (request.session access). +SESSION_SECRET_KEY = os.environ.get("SESSION_SECRET_KEY") +if not SESSION_SECRET_KEY: + raise ValueError("SESSION_SECRET_KEY environment variable is not set.") + +app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET_KEY) + +# ========== Starlette Admin Interface ========== +# Mount admin interface at /admin +# This provides a web-based UI for managing database records (replaces MS Access) +from admin import create_admin + +create_admin(app) +# ============================================== + app.add_middleware( CORSMiddleware, allow_origins=["*"], # Allows all origins, adjust as needed for security diff --git a/pyproject.toml b/pyproject.toml index 5fc75bb74..14f7e9690 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ "numpy==2.3.3", "packaging==25.0", "pandas==2.3.2", - "pandas-stubs==2.3.0.250703", + "pandas-stubs~=2.3.2", "pg8000==1.31.5", "phonenumbers==9.0.13", "pillow==11.3.0", @@ -92,6 +92,7 @@ dependencies = [ "sqlalchemy-searchable==2.1.0", "sqlalchemy-utils==0.42.0", "starlette==0.49.1", + "starlette-admin[i18n]>=0.16.0", "typing-extensions==4.15.0", "typing-inspection==0.4.1", "tzdata==2025.2", diff --git a/run_backfill.sh b/run_backfill.sh new file mode 100755 index 000000000..42e6e7620 --- /dev/null +++ b/run_backfill.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Load environment variables from .env and run the staging backfill. +# Usage: ./run_backfill.sh [--batch-size N] + +set -euo pipefail + +ENV_FILE=".env" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE; aborting." >&2 + exit 1 +fi + +# Export variables from .env +set -a +source "$ENV_FILE" +set +a + +# Forward any args (e.g., --batch-size 500) +python -m transfers.backfill.staging "$@" diff --git a/tests/features/environment.py b/tests/features/environment.py index 530ca453a..c8ddcb13b 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -325,8 +325,9 @@ def add_deployment(context, session, tid, sid): @add_context_object_container("blocks") -def add_block(context, session, parameter): +def add_block(context, session, parameter, thing): block = TransducerObservationBlock( + thing_id=thing.id, parameter_id=parameter.id, start_datetime=datetime.now() - timedelta(hours=1), end_datetime=datetime.now() + timedelta(hours=1), @@ -673,7 +674,7 @@ def before_all(context): # parameter ID can be hardcoded because init_parameter always creates the same one parameter = session.get(Parameter, 1) - block = add_block(context, session, parameter) + block = add_block(context, session, parameter, well_1) for i in range(1, 10): add_transducer_observation( context, session, block, deployment.id, random.random() diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py index a0ba1f02e..8d374b599 100644 --- a/transfers/aquifer_system_transfer.py +++ b/transfers/aquifer_system_transfer.py @@ -42,7 +42,7 @@ def transfer_aquifer_systems(session: Session, limit: int = None) -> tuple: # 4. Process each row for i, row in enumerate(cleaned_df.itertuples()): # check if limit is reached - if limit and i >= limit: + if limit is not None and limit > 0 and i >= limit: logger.info(f"Reached limit of {limit} rows. Stopping migration.") break diff --git a/transfers/sensor_transfer.py b/transfers/sensor_transfer.py index 2f4ce7cf3..3a39a1a03 100644 --- a/transfers/sensor_transfer.py +++ b/transfers/sensor_transfer.py @@ -30,11 +30,21 @@ ) EQUIPMENT_TO_SENSOR_TYPE_MAP = { + # Hydrological sensors "Pressure transducer": "Pressure Transducer", "Acoustic sounder": "Acoustic Sounder", "Barometer": "Barometer", "DiverLink": "DiverLink", "Diver Cable": "Diver Cable", + # Meteorological/environmental sensors + "Precip Collector": "Precip Collector", + "Soil Moisture": "Soil Moisture Sensor", + "Weather Station": "Weather Station", + "Camera": "Camera", + "Weir": "Weir", + "Snow Lysimeter": "Snow Lysimeter", + "Tipping Bucket": "Tipping Bucket", + "Lysimeter": "Lysimeter", } diff --git a/transfers/stratigraphy_transfer.py b/transfers/stratigraphy_transfer.py index 9f97a4904..f75f43abf 100644 --- a/transfers/stratigraphy_transfer.py +++ b/transfers/stratigraphy_transfer.py @@ -78,7 +78,7 @@ def transfer_stratigraphy(session: Session, limit: int = None) -> tuple: for well_index, (pointid, strat_group) in enumerate(well_groups): # Check limit (on number of wells, not records) - if limit and well_index >= limit: + if limit is not None and limit > 0 and well_index >= limit: logger.info(f"Reached limit of {limit} wells. Stopping.") break diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 6cfcea4c8..5b3c49d50 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -46,7 +46,7 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - logger.critical(f"PointID {pointid} has duplicate records. Skipping.") continue - if limit and i >= limit: + if limit is not None and limit > 0 and i >= limit: logger.warning(f"Reached limit of {limit} rows. Stopping migration.") break diff --git a/transfers/transfer.py b/transfers/transfer.py index 83b99b94a..d1b359bae 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -14,6 +14,8 @@ # limitations under the License. # =============================================================================== import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed from dotenv import load_dotenv @@ -56,6 +58,60 @@ def message(msg, pad=10, new_line_at_top=True): logger.info(f"{pad} {msg} {pad}") +def _execute_transfer(klass, flags: dict = None): + """Execute a single transfer class. Thread-safe since each creates its own session.""" + pointids = None + if os.getenv("TRANSFER_TEST_POINTIDS"): + pointids = os.getenv("TRANSFER_TEST_POINTIDS").split(",") + + transferer = klass(flags=flags, pointids=pointids) + transferer.transfer() + return transferer.input_df, transferer.cleaned_df, transferer.errors + + +def _execute_transfer_with_timing(name: str, klass, flags: dict = None): + """Execute transfer and return timing info.""" + start = time.time() + logger.info(f"Starting parallel transfer: {name}") + result = _execute_transfer(klass, flags) + elapsed = time.time() - start + logger.info(f"Completed parallel transfer: {name} in {elapsed:.2f}s") + return name, result, elapsed + + +def _execute_session_transfer_with_timing(name: str, transfer_func, limit: int): + """Execute a session-based transfer function and return timing info.""" + start = time.time() + logger.info(f"Starting parallel transfer: {name}") + with session_ctx() as session: + result = transfer_func(session, limit=limit) + elapsed = time.time() - start + logger.info(f"Completed parallel transfer: {name} in {elapsed:.2f}s") + return name, result, elapsed + + +def _execute_permissions_with_timing(name: str): + """Execute permissions transfer and return timing info.""" + start = time.time() + logger.info(f"Starting parallel transfer: {name}") + with session_ctx() as session: + transfer_permissions(session) + elapsed = time.time() - start + logger.info(f"Completed parallel transfer: {name} in {elapsed:.2f}s") + return name, None, elapsed + + +def _execute_foundational_transfer_with_timing(name: str, transfer_func, limit: int): + """Execute a foundational transfer (aquifer systems, formations) with its own session.""" + start = time.time() + logger.info(f"Starting parallel foundational transfer: {name}") + with session_ctx() as session: + result = transfer_func(session, limit=limit) + elapsed = time.time() - start + logger.info(f"Completed parallel foundational transfer: {name} in {elapsed:.2f}s") + return name, result, elapsed + + @timeit def transfer_all(metrics, limit=100): message("STARTING TRANSFER", new_line_at_top=False) @@ -63,16 +119,48 @@ def transfer_all(metrics, limit=100): logger.info("Erase and rebuilding database") erase_and_rebuild_db() - flags = {"TRANSFER_ALL_WELLS": True, "LIMIT": limit} # not currently used - - with session_ctx() as session: - transfer_aquifer_systems(session, limit=limit) - transfer_geologic_formations(session, limit=limit) + flags = {"TRANSFER_ALL_WELLS": True, "LIMIT": limit} + + # ========================================================================= + # PHASE 1: Foundation (Parallel - these are independent of each other) + # ========================================================================= + message("PHASE 1: FOUNDATIONAL TRANSFERS (PARALLEL)") + foundational_tasks = [ + ("AquiferSystems", transfer_aquifer_systems), + ("GeologicFormations", transfer_geologic_formations), + ] + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = { + executor.submit( + _execute_foundational_transfer_with_timing, name, func, limit + ): name + for name, func in foundational_tasks + } + + for future in as_completed(futures): + name = futures[future] + try: + result_name, result, elapsed = future.result() + logger.info( + f"Foundational transfer {result_name} completed in {elapsed:.2f}s" + ) + except Exception as e: + logger.critical(f"Foundational transfer {name} failed: {e}") + raise # Fail fast - foundational transfers must succeed message("TRANSFERRING WELLS") - results = _execute_transfer(WellTransferer, flags=flags) + use_parallel_wells = get_bool_env("TRANSFER_PARALLEL_WELLS", False) + if use_parallel_wells: + logger.info("Using PARALLEL wells transfer") + transferer = WellTransferer(flags=flags) + transferer.transfer_parallel() + results = (transferer.input_df, transferer.cleaned_df, transferer.errors) + else: + results = _execute_transfer(WellTransferer, flags=flags) metrics.well_metrics(*results) + # Get transfer flags transfer_screens = get_bool_env("TRANSFER_WELL_SCREENS", True) transfer_sensors = get_bool_env("TRANSFER_SENSORS", True) transfer_contacts = get_bool_env("TRANSFER_CONTACTS", True) @@ -81,8 +169,197 @@ def transfer_all(metrics, limit=100): transfer_acoustic = get_bool_env("TRANSFER_WATERLEVELS_ACOUSTIC", True) transfer_link_ids = get_bool_env("TRANSFER_LINK_IDS", True) transfer_groups = get_bool_env("TRANSFER_GROUPS", True) - transfer_assets = get_bool_env("TRANSFER_ASSETS", True) + transfer_assets = get_bool_env("TRANSFER_ASSETS", False) + use_parallel = get_bool_env("TRANSFER_PARALLEL", True) + + if use_parallel: + _transfer_parallel( + metrics, + flags, + limit, + transfer_screens, + transfer_sensors, + transfer_contacts, + transfer_waterlevels, + transfer_pressure, + transfer_acoustic, + transfer_link_ids, + transfer_groups, + transfer_assets, + ) + else: + _transfer_sequential( + metrics, + flags, + limit, + transfer_screens, + transfer_sensors, + transfer_contacts, + transfer_waterlevels, + transfer_pressure, + transfer_acoustic, + transfer_link_ids, + transfer_groups, + transfer_assets, + ) + +def _transfer_parallel( + metrics, + flags, + limit, + transfer_screens, + transfer_sensors, + transfer_contacts, + transfer_waterlevels, + transfer_pressure, + transfer_acoustic, + transfer_link_ids, + transfer_groups, + transfer_assets, +): + """Execute transfers in parallel where possible.""" + message("PARALLEL TRANSFER GROUP 1") + + # ========================================================================= + # PHASE 2: Parallel Group 1 (Independent transfers after wells) + # ========================================================================= + parallel_tasks_1 = [] + + if transfer_screens: + parallel_tasks_1.append(("WellScreens", WellScreenTransferer, flags)) + if transfer_contacts: + parallel_tasks_1.append(("Contacts", ContactTransfer, flags)) + if transfer_waterlevels: + parallel_tasks_1.append(("WaterLevels", WaterLevelTransferer, flags)) + if transfer_link_ids: + parallel_tasks_1.append(("LinkIdsWellData", LinkIdsWellDataTransferer, flags)) + parallel_tasks_1.append( + ("LinkIdsLocation", LinkIdsLocationDataTransferer, flags) + ) + if transfer_groups: + parallel_tasks_1.append(("Groups", ProjectGroupTransferer, flags)) + if transfer_assets: + parallel_tasks_1.append(("Assets", AssetTransferer, flags)) + + # Track results for metrics + results_map = {} + + # Execute parallel group 1 + with ThreadPoolExecutor(max_workers=min(8, len(parallel_tasks_1) + 2)) as executor: + futures = {} + + # Submit class-based transfers + for name, klass, task_flags in parallel_tasks_1: + future = executor.submit( + _execute_transfer_with_timing, name, klass, task_flags + ) + futures[future] = name + + # Submit session-based transfers + future = executor.submit( + _execute_session_transfer_with_timing, + "Stratigraphy", + transfer_stratigraphy, + limit, + ) + futures[future] = "Stratigraphy" + + future = executor.submit(_execute_permissions_with_timing, "Permissions") + futures[future] = "Permissions" + + # Collect results + for future in as_completed(futures): + name = futures[future] + try: + result_name, result, elapsed = future.result() + results_map[result_name] = result + logger.info(f"Parallel task {result_name} completed in {elapsed:.2f}s") + except Exception as e: + logger.critical(f"Parallel task {name} failed: {e}") + + # Record metrics for parallel group 1 + if "WellScreens" in results_map and results_map["WellScreens"]: + metrics.well_screen_metrics(*results_map["WellScreens"]) + if "Contacts" in results_map and results_map["Contacts"]: + metrics.contact_metrics(*results_map["Contacts"]) + if "Stratigraphy" in results_map and results_map["Stratigraphy"]: + metrics.stratigraphy_metrics(*results_map["Stratigraphy"]) + if "WaterLevels" in results_map and results_map["WaterLevels"]: + metrics.water_level_metrics(*results_map["WaterLevels"]) + if "LinkIdsWellData" in results_map and results_map["LinkIdsWellData"]: + metrics.welldata_link_ids_metrics(*results_map["LinkIdsWellData"]) + if "LinkIdsLocation" in results_map and results_map["LinkIdsLocation"]: + metrics.location_link_ids_metrics(*results_map["LinkIdsLocation"]) + if "Groups" in results_map and results_map["Groups"]: + metrics.group_metrics(*results_map["Groups"]) + if "Assets" in results_map and results_map["Assets"]: + metrics.asset_metrics(*results_map["Assets"]) + + # ========================================================================= + # PHASE 3: Sensors (Sequential - required before continuous water levels) + # ========================================================================= + if transfer_sensors: + message("TRANSFERRING SENSORS") + results = _execute_transfer(SensorTransferer, flags=flags) + metrics.sensor_metrics(*results) + + # ========================================================================= + # PHASE 4: Parallel Group 2 (Continuous water levels - after sensors) + # ========================================================================= + if transfer_pressure or transfer_acoustic: + message("PARALLEL TRANSFER GROUP 2 (Continuous Water Levels)") + + parallel_tasks_2 = [] + if transfer_pressure: + parallel_tasks_2.append( + ("Pressure", WaterLevelsContinuousPressureTransferer, flags) + ) + if transfer_acoustic: + parallel_tasks_2.append( + ("Acoustic", WaterLevelsContinuousAcousticTransferer, flags) + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = {} + for name, klass, task_flags in parallel_tasks_2: + future = executor.submit( + _execute_transfer_with_timing, name, klass, task_flags + ) + futures[future] = name + + for future in as_completed(futures): + name = futures[future] + try: + result_name, result, elapsed = future.result() + results_map[result_name] = result + logger.info( + f"Parallel task {result_name} completed in {elapsed:.2f}s" + ) + except Exception as e: + logger.critical(f"Parallel task {name} failed: {e}") + + if "Pressure" in results_map and results_map["Pressure"]: + metrics.pressure_metrics(*results_map["Pressure"]) + if "Acoustic" in results_map and results_map["Acoustic"]: + metrics.acoustic_metrics(*results_map["Acoustic"]) + + +def _transfer_sequential( + metrics, + flags, + limit, + transfer_screens, + transfer_sensors, + transfer_contacts, + transfer_waterlevels, + transfer_pressure, + transfer_acoustic, + transfer_link_ids, + transfer_groups, + transfer_assets, +): + """Original sequential transfer logic.""" if transfer_screens: message("TRANSFERRING WELL SCREENS") results = _execute_transfer(WellScreenTransferer, flags=flags) @@ -93,19 +370,6 @@ def transfer_all(metrics, limit=100): results = _execute_transfer(SensorTransferer, flags=flags) metrics.sensor_metrics(*results) - # Developer's notes all the metadata for these Things are not defined in the models/schemas yet' - # message("TRANSFERRING SPRINGS") - # timeit_direct(transfer_springs, sess, limit=limit) - # - # message("TRANSFERRING PERENNIAL STREAMS") - # timeit_direct(transfer_perennial_stream, sess, limit=limit) - # - # message("TRANSFERRING EPHEMERAL STREAMS") - # timeit_direct(transfer_ephemeral_stream, sess, limit=limit) - # - # message("TRANSFERRING METEOROLOGICAL") - # timeit_direct(transfer_met, sess, limit) - if transfer_contacts: message("TRANSFERRING CONTACTS") results = _execute_transfer(ContactTransfer, flags=flags) @@ -115,7 +379,7 @@ def transfer_all(metrics, limit=100): with session_ctx() as session: transfer_permissions(session) - message("TRANSFERRING STRATIGRAPY") + message("TRANSFERRING STRATIGRAPHY") with session_ctx() as session: results = transfer_stratigraphy(session, limit=limit) metrics.stratigraphy_metrics(*results) @@ -157,17 +421,6 @@ def transfer_all(metrics, limit=100): metrics.asset_metrics(*results) -def _execute_transfer(klass, flags: dict = None): - - pointids = None - if os.getenv("TRANSFER_TEST_POINTIDS"): - pointids = os.getenv("TRANSFER_TEST_POINTIDS").split(",") - - transferer = klass(flags=flags, pointids=pointids) - transferer.transfer() - return transferer.input_df, transferer.cleaned_df, transferer.errors - - def main(): message("START--------------------------------------") limit = int(os.getenv("TRANSFER_LIMIT", 1000)) @@ -177,7 +430,6 @@ def main(): metrics.close() metrics.save_to_storage_bucket() - # todo: move the log file to a storage bucket save_log_to_bucket() message("END--------------------------------------") diff --git a/transfers/transferer.py b/transfers/transferer.py index 7b8076fd4..be814e2bd 100644 --- a/transfers/transferer.py +++ b/transfers/transferer.py @@ -88,7 +88,7 @@ def _limit_iterator(self, session: Session, limit: int, step: int = 100): start_time = time.time() logger.info(f"Starting transfer of {n} [limit={limit}] rows") for i, row in enumerate(df.itertuples()): - if limit and i >= limit: + if limit > 0 and i >= limit: logger.info(f"Reached limit of {limit} rows. Stopping migration.") break diff --git a/transfers/util.py b/transfers/util.py index ce3d66ada..523001310 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -472,6 +472,7 @@ def make_location(row: pd.Series, elevations: dict) -> tuple: location = Location( nma_pk_location=row.LocationId, + description=row.PointID, # Use PointID as location description point=transformed_point.wkt, elevation=z, release_status="public" if row.PublicRelease else "private", diff --git a/transfers/waterlevels_transducer_transfer.py b/transfers/waterlevels_transducer_transfer.py index 70400daa2..338e71027 100644 --- a/transfers/waterlevels_transducer_transfer.py +++ b/transfers/waterlevels_transducer_transfer.py @@ -83,11 +83,26 @@ def _transfer_hook(self, session: Session) -> None: qced = group[field == 1] notqced = group[~(field == 1)] + # Check for deployments first to get thing_id + if not deployments: + logger.critical( + f"Thing with PointID={pointid} has no deployments. Skipping all water levels" + ) + self._capture_error(pointid, "no deployments", "DateMeasured") + continue + + # Get thing_id from the first deployment + thing_id = deployments[0].thing_id + qced_block = TransducerObservationBlock( - parameter_id=self.groundwater_parameter_id, review_status="approved" + thing_id=thing_id, + parameter_id=self.groundwater_parameter_id, + review_status="approved", ) notqced_block = TransducerObservationBlock( - parameter_id=self.groundwater_parameter_id, review_status="not reviewed" + thing_id=thing_id, + parameter_id=self.groundwater_parameter_id, + review_status="not reviewed", ) for block, rows, release_status in ( @@ -97,13 +112,6 @@ def _transfer_hook(self, session: Session) -> None: block.start_datetime = rows.DateMeasured.min() block.end_datetime = rows.DateMeasured.max() - if not deployments: - logger.critical( - f"Thing with PointID={pointid} has no deployments. Skipping water levels {release_status} block" - ) - self._capture_error(pointid, "no deployments", "DateMeasured") - continue - if rows.empty: logger.info(f"no {release_status} records for pointid {pointid}") continue diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 02d6b1c69..2331dadc3 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -14,8 +14,11 @@ # limitations under the License. # =============================================================================== import json +import os import re import time +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, UTC from zoneinfo import ZoneInfo @@ -52,6 +55,7 @@ get_county_from_point, get_quad_name_from_point, ) +from db.engine import session_ctx from transfers.transferer import ChunkTransferer, Transferer from transfers.util import ( make_location, @@ -837,6 +841,744 @@ def _after_hook_chunk(self, well, formations): ) return objs + def transfer_parallel(self, num_workers: int = None) -> None: + """ + Transfer wells using parallel processing for improved performance. + + Each worker processes a batch of wells with its own database session. + The after_hook runs sequentially after all workers complete. + """ + if num_workers is None: + num_workers = int(os.environ.get("TRANSFER_WORKERS", "4")) + + # Load dataframes + self.input_df, self.cleaned_df = self._get_dfs() + df = self.cleaned_df + n = len(df) + + if n == 0: + logger.info("No wells to transfer") + return + + # Calculate batch size + batch_size = max(100, n // num_workers) + batches = [df.iloc[i : i + batch_size] for i in range(0, n, batch_size)] + + logger.info( + f"Starting parallel transfer of {n} wells with {num_workers} workers, " + f"{len(batches)} batches of ~{batch_size} wells each" + ) + + # Pre-load aquifers and formations to avoid race conditions + with session_ctx() as session: + self._aquifers = session.query(AquiferSystem).all() + session.expunge_all() + + # Thread-safe collections for results + all_errors = [] + errors_lock = threading.Lock() + aquifers_lock = threading.Lock() + + def process_batch(batch_idx: int, batch_df: pd.DataFrame) -> dict: + """Process a batch of wells in a separate thread with its own session.""" + batch_errors = [] + batch_start = time.time() + + try: + with session_ctx() as session: + # Load aquifers and formations for this session + local_aquifers = session.query(AquiferSystem).all() + local_formations = { + f.formation_code: f + for f in session.query(GeologicFormation).all() + } + + for i, row in enumerate(batch_df.itertuples()): + try: + # Process single well with all dependent objects + self._step_parallel_complete( + session, + batch_df, + i, + row, + local_aquifers, + local_formations, + batch_errors, + aquifers_lock, + ) + except Exception as e: + batch_errors.append( + { + "pointid": getattr(row, "PointID", "Unknown"), + "error": str(e), + "table": "WellData", + "field": "Unknown", + } + ) + + # Commit periodically + if i > 0 and i % 100 == 0: + try: + session.commit() + session.expunge_all() + # Re-query after expunge + local_aquifers = session.query(AquiferSystem).all() + local_formations = { + f.formation_code: f + for f in session.query(GeologicFormation).all() + } + except Exception as e: + logger.critical( + f"Batch {batch_idx}: Error committing: {e}" + ) + session.rollback() + + # Final commit for this batch + session.commit() + + except Exception as e: + logger.critical(f"Batch {batch_idx} failed: {e}") + batch_errors.append( + { + "pointid": "Batch", + "error": str(e), + "table": "WellData", + "field": "BatchProcessing", + } + ) + + elapsed = time.time() - batch_start + logger.info( + f"Batch {batch_idx}/{len(batches)} completed: {len(batch_df)} wells " + f"in {elapsed:.2f}s ({len(batch_df)/elapsed:.1f} wells/sec)" + ) + + return {"errors": batch_errors} + + # Execute batches in parallel + with ThreadPoolExecutor(max_workers=num_workers) as executor: + futures = { + executor.submit(process_batch, idx, batch): idx + for idx, batch in enumerate(batches) + } + + for future in as_completed(futures): + batch_idx = futures[future] + try: + result = future.result() + with errors_lock: + all_errors.extend(result["errors"]) + except Exception as e: + logger.critical(f"Batch {batch_idx} raised exception: {e}") + with errors_lock: + all_errors.append( + { + "pointid": f"Batch-{batch_idx}", + "error": str(e), + "table": "WellData", + "field": "ThreadException", + } + ) + + # Store merged results + self.errors = all_errors + + logger.info(f"Parallel transfer complete: {n} wells, {len(all_errors)} errors") + + # Dump cached elevations (minimal after-processing) + dump_cached_elevations(self._cached_elevations) + + def _step_parallel( + self, + session: Session, + df: pd.DataFrame, + i: int, + row, + local_aquifers: list, + batch_locations: dict, + batch_errors: list, + aquifers_lock: threading.Lock, + ): + """ + Process a single well row in parallel mode. + Similar to _step but uses thread-local state. + """ + try: + first_visit_date = _get_first_visit_date(row) + well_purposes = ( + [] if isna(row.CurrentUse) else self._extract_well_purposes(row) + ) + well_casing_materials = ( + [] if isna(row.CasingDescription) else _extract_casing_materials(row) + ) + well_pump_type = _extract_well_pump_type(row) + + wcm = None + if notna(row.ConstructionMethod): + wcm = self._get_lexicon_value_safe( + row, + f"LU_ConstructionMethod:{row.ConstructionMethod}", + "Unknown", + batch_errors, + ) + + is_suitable_for_datalogger = False + if notna(row.OpenWellLoggerOK): + is_suitable_for_datalogger = bool(row.OpenWellLoggerOK) + + mpheight = row.MPHeight + mpheight_description = row.MeasuringPoint + if mpheight is None: + mphs = self._measuring_point_estimator.estimate_measuring_point_height( + row + ) + if mphs: + try: + mpheight = mphs[0][0] + mpheight_description = mphs[1][0] + except IndexError: + pass + + data = CreateWell( + location_id=0, + name=row.PointID, + first_visit_date=first_visit_date, + hole_depth=row.HoleDepth, + well_depth=row.WellDepth, + well_casing_diameter=( + row.CasingDiameter * 12 if row.CasingDiameter else None + ), + well_casing_depth=row.CasingDepth, + release_status="public" if row.PublicRelease else "private", + measuring_point_height=mpheight, + measuring_point_description=mpheight_description, + notes=( + [{"content": row.Notes, "note_type": "General"}] + if row.Notes + else [] + ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=wcm, + well_pump_type=well_pump_type, + is_suitable_for_datalogger=is_suitable_for_datalogger, + ) + + CreateWell.model_validate(data) + except ValidationError as e: + batch_errors.append( + { + "pointid": row.PointID, + "error": f"Validation Error: {e.errors()}", + "table": "WellData", + "field": "UnknownField", + } + ) + return + + well = None + try: + well_data = data.model_dump( + exclude=[ + "location_id", + "group_id", + "well_purposes", + "well_casing_materials", + "measuring_point_height", + "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", + ] + ) + well_data["thing_type"] = "water well" + well_data["nma_pk_welldata"] = row.WellID + + well_data.pop("notes") + well = Thing(**well_data) + session.add(well) + + if well_purposes: + for wp in well_purposes: + if wp in WellPurposeEnum: + wp_obj = WellPurpose(thing=well, purpose=wp) + session.add(wp_obj) + + if well_casing_materials: + for wcm in well_casing_materials: + if wcm in WellCasingMaterialEnum: + wcm_obj = WellCasingMaterial(thing=well, material=wcm) + session.add(wcm_obj) + except Exception as e: + if well is not None: + session.expunge(well) + batch_errors.append( + { + "pointid": row.PointID, + "error": str(e), + "table": "WellData", + "field": "UnknownField", + } + ) + return + + try: + location, elevation_method, notes = make_location( + row, self._cached_elevations + ) + session.add(location) + batch_locations[row.PointID] = (elevation_method, notes) + except Exception as e: + batch_errors.append( + { + "pointid": row.PointID, + "error": str(e), + "table": "WellData", + "field": "Location", + } + ) + return + + assoc = LocationThingAssociation( + effective_start=datetime.now(tz=ZoneInfo("UTC")) + ) + assoc.location = location + assoc.thing = well + session.add(assoc) + + if not isna(row.AquiferType): + try: + self._add_aquifers_parallel( + session, row, well, local_aquifers, aquifers_lock + ) + except Exception as e: + logger.warning(f"Error adding aquifer for {well.name}: {e}") + + def _step_parallel_complete( + self, + session: Session, + df: pd.DataFrame, + i: int, + row, + local_aquifers: list, + local_formations: dict, + batch_errors: list, + aquifers_lock: threading.Lock, + ): + """ + Process a single well with ALL dependent objects in one pass. + Combines _step_parallel and _after_hook_chunk for maximum parallelization. + """ + try: + first_visit_date = _get_first_visit_date(row) + well_purposes = ( + [] if isna(row.CurrentUse) else self._extract_well_purposes(row) + ) + well_casing_materials = ( + [] if isna(row.CasingDescription) else _extract_casing_materials(row) + ) + well_pump_type = _extract_well_pump_type(row) + + wcm = None + if notna(row.ConstructionMethod): + wcm = self._get_lexicon_value_safe( + row, + f"LU_ConstructionMethod:{row.ConstructionMethod}", + "Unknown", + batch_errors, + ) + + is_suitable_for_datalogger = False + if notna(row.OpenWellLoggerOK): + is_suitable_for_datalogger = bool(row.OpenWellLoggerOK) + + mpheight = row.MPHeight + mpheight_description = row.MeasuringPoint + if mpheight is None: + mphs = self._measuring_point_estimator.estimate_measuring_point_height( + row + ) + if mphs: + try: + mpheight = mphs[0][0] + mpheight_description = mphs[1][0] + except IndexError: + pass + + data = CreateWell( + location_id=0, + name=row.PointID, + first_visit_date=first_visit_date, + hole_depth=row.HoleDepth, + well_depth=row.WellDepth, + well_casing_diameter=( + row.CasingDiameter * 12 if row.CasingDiameter else None + ), + well_casing_depth=row.CasingDepth, + release_status="public" if row.PublicRelease else "private", + measuring_point_height=mpheight, + measuring_point_description=mpheight_description, + notes=( + [{"content": row.Notes, "note_type": "General"}] + if row.Notes + else [] + ), + well_completion_date=row.CompletionDate, + well_driller_name=row.DrillerName, + well_construction_method=wcm, + well_pump_type=well_pump_type, + is_suitable_for_datalogger=is_suitable_for_datalogger, + ) + + CreateWell.model_validate(data) + except ValidationError as e: + batch_errors.append( + { + "pointid": row.PointID, + "error": f"Validation Error: {e.errors()}", + "table": "WellData", + "field": "UnknownField", + } + ) + return + + well = None + try: + well_data = data.model_dump( + exclude=[ + "location_id", + "group_id", + "well_purposes", + "well_casing_materials", + "measuring_point_height", + "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", + ] + ) + well_data["thing_type"] = "water well" + well_data["nma_pk_welldata"] = row.WellID + + well_data.pop("notes") + well = Thing(**well_data) + session.add(well) + + if well_purposes: + for wp in well_purposes: + if wp in WellPurposeEnum: + wp_obj = WellPurpose(thing=well, purpose=wp) + session.add(wp_obj) + + if well_casing_materials: + for wcm in well_casing_materials: + if wcm in WellCasingMaterialEnum: + wcm_obj = WellCasingMaterial(thing=well, material=wcm) + session.add(wcm_obj) + except Exception as e: + if well is not None: + session.expunge(well) + batch_errors.append( + { + "pointid": row.PointID, + "error": str(e), + "table": "WellData", + "field": "UnknownField", + } + ) + return + + try: + location, elevation_method, location_notes = make_location( + row, self._cached_elevations + ) + session.add(location) + except Exception as e: + batch_errors.append( + { + "pointid": row.PointID, + "error": str(e), + "table": "WellData", + "field": "Location", + } + ) + return + + assoc = LocationThingAssociation( + effective_start=datetime.now(tz=ZoneInfo("UTC")) + ) + assoc.location = location + assoc.thing = well + session.add(assoc) + + # Flush to get IDs for dependent objects + session.flush() + + # === Now add all dependent objects that need well.id and location.id === + + # Aquifers + if not isna(row.AquiferType): + try: + self._add_aquifers_parallel( + session, row, well, local_aquifers, aquifers_lock + ) + except Exception as e: + logger.warning(f"Error adding aquifer for {well.name}: {e}") + + # Formation zone + formation_code = row.FormationZone if hasattr(row, "FormationZone") else None + if formation_code: + formation_code = formation_code.strip() if formation_code else None + if formation_code: + if formation_code in local_formations: + well.formation_completion_code = local_formations[ + formation_code + ].formation_code + else: + batch_errors.append( + { + "pointid": row.PointID, + "error": f"Unknown formation: {formation_code}", + "table": "WellData", + "field": "FormationZone", + } + ) + + # Well notes + if notna(row.Notes): + note = well.add_note(row.Notes, "General") + session.add(note) + if row.ConstructionNotes: + note = well.add_note(row.ConstructionNotes, "Construction") + session.add(note) + if row.WaterNotes: + note = well.add_note(row.WaterNotes, "Water") + session.add(note) + + # Location notes + for note_type, note_content in location_notes.items(): + if notna(note_content): + location_note = location.add_note(note_content, note_type) + session.add(location_note) + + # Data provenances + data_provenances = make_location_data_provenance( + row, location, elevation_method + ) + for dp in data_provenances: + session.add(dp) + + # Well data provenances + cs = ( + "CompletionSource", + { + "field_name": "well_completion_date", + "origin_type": f"LU_Depth_CompletionSource:{row.CompletionSource}", + }, + ) + ds = ( + "DataSource", + {"field_name": "well_construction_method", "origin_source": row.DataSource}, + ) + des = ( + "DepthSource", + { + "field_name": "well_depth", + "origin_type": f"LU_Depth_CompletionSource:{row.DepthSource}", + }, + ) + + for row_field, kw in (cs, ds, des): + if notna(row[row_field]): + if "origin_type" in kw: + try: + ot = lexicon_mapper.map_value(kw["origin_type"]) + kw["origin_type"] = ot + except KeyError: + continue + dp = DataProvenance(target_id=well.id, target_table="thing", **kw) + session.add(dp) + + # Measuring point history + mphs = self._measuring_point_estimator.estimate_measuring_point_height(row) + for mph, mph_desc, start_date, end_date in zip(*mphs): + measuring_point_history = MeasuringPointHistory( + thing_id=well.id, + measuring_point_height=mph, + measuring_point_description=mph_desc, + start_date=start_date, + end_date=end_date, + ) + session.add(measuring_point_history) + + # Status history + target_id = well.id + target_table = "thing" + if notna(row.MonitoringStatus): + if ( + "X" in row.MonitoringStatus + or "I" in row.MonitoringStatus + or "C" in row.MonitoringStatus + ): + status_value = "Not currently monitored" + else: + status_value = "Currently monitored" + + status_history = StatusHistory( + status_type="Monitoring Status", + status_value=status_value, + reason=row.MonitorStatusReason, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + + for code in NMA_MONITORING_FREQUENCY.keys(): + if code in row.MonitoringStatus: + monitoring_frequency = NMA_MONITORING_FREQUENCY[code] + monitoring_frequency_history = MonitoringFrequencyHistory( + thing_id=well.id, + monitoring_frequency=monitoring_frequency, + start_date=datetime.now(tz=UTC), + end_date=None, + ) + session.add(monitoring_frequency_history) + + if notna(row.Status): + try: + status_value = lexicon_mapper.map_value(f"LU_Status:{row.Status}") + status_history = StatusHistory( + status_type="Well Status", + status_value=status_value, + reason=row.StatusUserNotes, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + except KeyError: + batch_errors.append( + { + "pointid": row.PointID, + "error": f"Unknown lexicon value: LU_Status:{row.Status}", + "table": "WellData", + "field": "Status", + } + ) + + def _get_lexicon_value_safe(self, row, value, default, errors_list): + """Thread-safe version of _get_lexicon_value.""" + try: + return lexicon_mapper.map_value(value) + except KeyError: + errors_list.append( + { + "pointid": row.PointID, + "error": f"Unknown lexicon value: {value}", + "table": "WellData", + "field": "Unknown", + } + ) + return default + + def _add_aquifers_parallel(self, session, row, well, local_aquifers, aquifers_lock): + """Thread-safe version of _add_aquifers.""" + aquifer_codes = _extract_aquifer_type_codes(row.AquiferType) + if not aquifer_codes: + return + + if isna(row.AqClass): + try: + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + except KeyError: + return + else: + try: + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferClass:{row.AqClass}" + ) + except KeyError: + try: + aquifer_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + except KeyError: + return + + if aquifer_name is None: + return + + try: + primary_type = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_codes[0]}" + ) + except KeyError: + primary_type = "Unknown" + + # Step 1: Check local cache under lock (fast check only) + aquifer = None + need_db_lookup = False + + with aquifers_lock: + aquifer = next((a for a in local_aquifers if a.name == aquifer_name), None) + if not aquifer: + need_db_lookup = True + + # Step 2: Database operations OUTSIDE the lock to avoid deadlock + if need_db_lookup: + # Check if it exists in DB + aquifer = ( + session.query(AquiferSystem) + .filter(AquiferSystem.name == aquifer_name) + .first() + ) + + if not aquifer: + try: + aquifer = AquiferSystem( + name=aquifer_name, + primary_aquifer_type=primary_type, + geographic_scale=None, + ) + session.add(aquifer) + session.flush() + logger.info(f"Created aquifer: {aquifer_name}") + + # Update local cache under lock + with aquifers_lock: + # Check again to avoid duplicates + existing = next( + (a for a in local_aquifers if a.name == aquifer_name), None + ) + if not existing: + local_aquifers.append(aquifer) + except Exception as e: + # Race condition - another thread created it + session.rollback() + aquifer = ( + session.query(AquiferSystem) + .filter(AquiferSystem.name == aquifer_name) + .first() + ) + + if aquifer: + aquifer_assoc = ThingAquiferAssociation(thing=well, aquifer_system=aquifer) + session.add(aquifer_assoc) + + for aquifer_code in aquifer_codes: + try: + type_name = lexicon_mapper.map_value( + f"LU_AquiferType:{aquifer_code}" + ) + aquifer_type = AquiferType( + thing_aquifer_association=aquifer_assoc, + aquifer_type=type_name, + ) + session.add(aquifer_type) + except KeyError: + pass + class WellChunkTransferer(ChunkTransferer): source_table: str = None diff --git a/uv.lock b/uv.lock index 2edcb7570..67ea6ae0d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.13" [[package]] @@ -178,6 +178,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/aa/91355b5f539caf1b94f0e66ff1e4ee39373b757fce08204981f7829ede51/authlib-1.6.4-py2.py3-none-any.whl", hash = "sha256:39313d2a2caac3ecf6d8f95fbebdfd30ae6ea6ae6a6db794d976405fdd9aa796", size = 243076, upload-time = "2025-09-17T09:59:22.259Z" }, ] +[[package]] +name = "babel" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, +] + [[package]] name = "bcrypt" version = "4.3.0" @@ -1091,6 +1100,7 @@ dependencies = [ { name = "sqlalchemy-searchable" }, { name = "sqlalchemy-utils" }, { name = "starlette" }, + { name = "starlette-admin", extra = ["i18n"] }, { name = "typing-extensions" }, { name = "typing-inspection" }, { name = "tzdata" }, @@ -1161,7 +1171,7 @@ requires-dist = [ { name = "numpy", specifier = "==2.3.3" }, { name = "packaging", specifier = "==25.0" }, { name = "pandas", specifier = "==2.3.2" }, - { name = "pandas-stubs", specifier = "==2.3.0.250703" }, + { name = "pandas-stubs", specifier = "~=2.3.2" }, { name = "pg8000", specifier = "==1.31.5" }, { name = "phonenumbers", specifier = "==9.0.13" }, { name = "pillow", specifier = "==11.3.0" }, @@ -1198,6 +1208,7 @@ requires-dist = [ { name = "sqlalchemy-searchable", specifier = "==2.1.0" }, { name = "sqlalchemy-utils", specifier = "==0.42.0" }, { name = "starlette", specifier = "==0.49.1" }, + { name = "starlette-admin", extras = ["i18n"], specifier = ">=0.16.0" }, { name = "typing-extensions", specifier = "==4.15.0" }, { name = "typing-inspection", specifier = "==0.4.1" }, { name = "tzdata", specifier = "==2025.2" }, @@ -1254,15 +1265,15 @@ wheels = [ [[package]] name = "pandas-stubs" -version = "2.3.0.250703" +version = "2.3.3.251219" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "types-pytz" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/df/c1c51c5cec087b8f4d04669308b700e9648745a77cdd0c8c5e16520703ca/pandas_stubs-2.3.0.250703.tar.gz", hash = "sha256:fb6a8478327b16ed65c46b1541de74f5c5947f3601850caf3e885e0140584717", size = 103910, upload-time = "2025-07-02T17:49:11.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/ee/5407e9e63d22a47774f9246ca80b24f82c36f26efd39f9e3c5b584b915aa/pandas_stubs-2.3.3.251219.tar.gz", hash = "sha256:dc2883e6daff49d380d1b5a2e864983ab9be8cd9a661fa861e3dea37559a5af4", size = 106899, upload-time = "2025-12-19T15:49:53.766Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/cb/09d5f9bf7c8659af134ae0ffc1a349038a5d0ff93e45aedc225bde2872a3/pandas_stubs-2.3.0.250703-py3-none-any.whl", hash = "sha256:a9265fc69909f0f7a9cabc5f596d86c9d531499fed86b7838fd3278285d76b81", size = 154719, upload-time = "2025-07-02T17:49:10.697Z" }, + { url = "https://files.pythonhosted.org/packages/64/20/69f2a39792a653fd64d916cd563ed79ec6e5dcfa6408c4674021d810afcf/pandas_stubs-2.3.3.251219-py3-none-any.whl", hash = "sha256:ccc6337febb51d6d8a08e4c96b479478a0da0ef704b5e08bd212423fe1cb549c", size = 163667, upload-time = "2025-12-19T15:49:52.072Z" }, ] [[package]] @@ -1913,6 +1924,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/da/545b75d420bb23b5d494b0517757b351963e974e79933f01e05c929f20a6/starlette-0.49.1-py3-none-any.whl", hash = "sha256:d92ce9f07e4a3caa3ac13a79523bd18e3bc0042bb8ff2d759a8e7dd0e1859875", size = 74175, upload-time = "2025-10-28T17:34:09.13Z" }, ] +[[package]] +name = "starlette-admin" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "python-multipart" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/5e/2487c92a3239d01cf1cb7db5d8455454b231c49a7576c06f3241050e8813/starlette_admin-0.16.0.tar.gz", hash = "sha256:e706a1582a22a69202d3165d8c626d5868822c229353a81e1d189666d8418f64", size = 2104304, upload-time = "2025-12-13T21:49:40.087Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/85/f3950e95771e92079ad2c6f8d52ce4c04a0e2e47cedc989ae465d9cfbd9c/starlette_admin-0.16.0-py3-none-any.whl", hash = "sha256:9b7ee51cc275684ba75dda5eafc650e0c8afa1d2b7e99e4d1c83fe7d1e83de9e", size = 2175927, upload-time = "2025-12-13T21:49:42.272Z" }, +] + +[package.optional-dependencies] +i18n = [ + { name = "babel" }, +] + [[package]] name = "types-pytz" version = "2025.2.0.20250809"