From 563b33f3def63a50407b2c738577de30aaf8a9df Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 7 Jan 2026 17:24:12 -0700 Subject: [PATCH 1/3] feat: add NMA WaterLevelsContinuous_Pressure fields to transducer_observation model --- ...essure_fields_to_transducer_observation.py | 193 ++++++++++++++++++ db/transducer.py | 44 ++++ transfers/waterlevels_transducer_transfer.py | 62 +++++- 3 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py diff --git a/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py b/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py new file mode 100644 index 000000000..e2e19c03b --- /dev/null +++ b/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py @@ -0,0 +1,193 @@ +"""Add NMA WaterLevelsContinuous_Pressure fields to transducer_observation. + +Revision ID: c9f1d2e3a4b5 +Revises: b7d4c6a1b2c3 +Create Date: 2026-02-10 03:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "c9f1d2e3a4b5" +down_revision: Union[str, Sequence[str], None] = "b7d4c6a1b2c3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add legacy NMA WaterLevelsContinuous_Pressure columns.""" + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_checked_by", + sa.String(length=4), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_created", + sa.DateTime(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_data_source", + sa.String(length=5), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_global_id", + sa.String(length=40), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_measurement_method", + sa.String(length=2), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_measuring_agency", + sa.String(length=50), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_notes", + sa.String(length=100), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_processed_by", + sa.String(length=4), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_qced", + sa.Boolean(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_temperature_water", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_updated", + sa.DateTime(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_water_head", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + sa.Float(), + nullable=True, + ), + ) + + +def downgrade() -> None: + """Drop legacy NMA WaterLevelsContinuous_Pressure columns.""" + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_water_head", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_updated", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_temperature_water", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_qced", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_processed_by", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_notes", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_measuring_agency", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_measurement_method", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_global_id", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_data_source", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_created", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_checked_by", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + ) diff --git a/db/transducer.py b/db/transducer.py index 25928c9ca..608260ce5 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -18,9 +18,11 @@ from typing import TYPE_CHECKING from sqlalchemy import ( + Boolean, ForeignKey, Float, DateTime, + String, Text, CheckConstraint, Index, @@ -118,6 +120,48 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): DateTime(timezone=True), nullable=False, index=True ) value: Mapped[float] = mapped_column(Float, nullable=False) + nma_waterlevelscontinuous_pressure_conddl_ms_cm: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_checked_by: Mapped[str] = mapped_column( + String(4), nullable=True + ) + nma_waterlevelscontinuous_pressure_created: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + nma_waterlevelscontinuous_pressure_data_source: Mapped[str] = mapped_column( + String(5), nullable=True + ) + nma_waterlevelscontinuous_pressure_global_id: Mapped[str] = mapped_column( + String(40), nullable=True + ) + nma_waterlevelscontinuous_pressure_measurement_method: Mapped[str] = mapped_column( + String(2), nullable=True + ) + nma_waterlevelscontinuous_pressure_measuring_agency: Mapped[str] = mapped_column( + String(50), nullable=True + ) + nma_waterlevelscontinuous_pressure_notes: Mapped[str] = mapped_column( + String(100), nullable=True + ) + nma_waterlevelscontinuous_pressure_processed_by: Mapped[str] = mapped_column( + String(4), nullable=True + ) + nma_waterlevelscontinuous_pressure_qced: Mapped[bool] = mapped_column( + Boolean, nullable=True + ) + nma_waterlevelscontinuous_pressure_temperature_water: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_updated: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + nma_waterlevelscontinuous_pressure_water_head: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_water_head_adjusted: Mapped[float] = ( + mapped_column(Float, nullable=True) + ) # qc_block_id: Mapped[Optional[int]] = mapped_column( # ForeignKey("transducer_observation_block.id", ondelete="SET NULL"), index=True diff --git a/transfers/waterlevels_transducer_transfer.py b/transfers/waterlevels_transducer_transfer.py index 338e71027..ffd1ad8d4 100644 --- a/transfers/waterlevels_transducer_transfer.py +++ b/transfers/waterlevels_transducer_transfer.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== +import keyword import pandas as pd from pandas import Timestamp from pydantic import ValidationError @@ -39,6 +40,8 @@ class WaterLevelsContinuousTransferer(Transferer): def __init__(self, *args, **kw): super().__init__(*args, **kw) self.groundwater_parameter_id = get_groundwater_parameter_id() + self._itertuples_field_map = {} + self._df_columns = set() if self._sensor_types is None: raise ValueError("_sensor_types must be set") if self._partition_field is None: @@ -55,6 +58,11 @@ def _get_dfs(self): # remove duplicate rows cleaned_df = cleaned_df.drop_duplicates(subset=["PointID", "DateMeasured"]) + self._df_columns = set(cleaned_df.columns) + self._itertuples_field_map = self._build_itertuples_field_map( + cleaned_df.columns + ) + return input_df, cleaned_df def _transfer_hook(self, session: Session) -> None: @@ -188,12 +196,64 @@ def _make_observation( obspayload = CreateTransducerObservation.model_validate( payload ).model_dump() - return TransducerObservation(**obspayload) + legacy_payload = self._legacy_payload(row) + return TransducerObservation(**obspayload, **legacy_payload) except ValidationError as e: logger.critical(f"Observation validation error: {e.errors()}") self._capture_error(pointid, str(e), "DepthToWaterBGS") + def _legacy_payload(self, row: pd.Series) -> dict: + def val(key: str): + if key not in self._df_columns: + return None + field = self._itertuples_field_map.get(key, key) + v = getattr(row, field, None) + if pd.isna(v): + return None + return v + + return { + "nma_waterlevelscontinuous_pressure_conddl_ms_cm": val("CONDDL (mS/cm)"), + "nma_waterlevelscontinuous_pressure_checked_by": val("CheckedBy"), + "nma_waterlevelscontinuous_pressure_created": val("Created"), + "nma_waterlevelscontinuous_pressure_data_source": val("DataSource"), + "nma_waterlevelscontinuous_pressure_global_id": val("GlobalID"), + "nma_waterlevelscontinuous_pressure_measurement_method": val( + "MeasurementMethod" + ), + "nma_waterlevelscontinuous_pressure_measuring_agency": val( + "MeasuringAgency" + ), + "nma_waterlevelscontinuous_pressure_notes": val("Notes"), + "nma_waterlevelscontinuous_pressure_processed_by": val("ProcessedBy"), + "nma_waterlevelscontinuous_pressure_qced": val("QCed"), + "nma_waterlevelscontinuous_pressure_temperature_water": val( + "TemperatureWater" + ), + "nma_waterlevelscontinuous_pressure_updated": val("Updated"), + "nma_waterlevelscontinuous_pressure_water_head": val("WaterHead"), + "nma_waterlevelscontinuous_pressure_water_head_adjusted": val( + "WaterHeadAdjusted" + ), + } + + @staticmethod + def _build_itertuples_field_map(columns: list[str]) -> dict[str, str]: + """ + Map original column names to itertuples field names for invalid identifiers. + """ + invalid_count = 0 + mapping: dict[str, str] = {} + for col in columns: + if not isinstance(col, str): + continue + if col.isidentifier() and not keyword.iskeyword(col): + continue + invalid_count += 1 + mapping[col] = f"_{invalid_count}" + return mapping + class WaterLevelsContinuousPressureTransferer(WaterLevelsContinuousTransferer): source_table = "WaterLevelsContinuous_Pressure" From 77b1887d2083e040fbc4692ae8f4427c96fa3c37 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 7 Jan 2026 17:36:37 -0700 Subject: [PATCH 2/3] feat: add TransducerObservationAdmin view for managing transducer observations --- admin/config.py | 9 +- admin/views/__init__.py | 2 + admin/views/transducer_observation.py | 186 ++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 admin/views/transducer_observation.py diff --git a/admin/config.py b/admin/config.py index e88dfdc37..f6f874d4c 100644 --- a/admin/config.py +++ b/admin/config.py @@ -38,9 +38,9 @@ SampleAdmin, GeologicFormationAdmin, DataProvenanceAdmin, + TransducerObservationAdmin, FieldEventAdmin, FieldActivityAdmin, - FieldEventParticipantAdmin, ParameterAdmin, ) from db.engine import engine @@ -62,8 +62,8 @@ from db.sample import Sample from db.geologic_formation import GeologicFormation from db.data_provenance import DataProvenance -from db.field import FieldEvent, FieldActivity, FieldEventParticipant -from db.permission_history import PermissionHistory +from db.transducer import TransducerObservation +from db.field import FieldEvent, FieldActivity from db.parameter import Parameter @@ -140,6 +140,9 @@ def create_admin(app): # Data provenance admin.add_view(DataProvenanceAdmin(DataProvenance)) + # Transducer observations + admin.add_view(TransducerObservationAdmin(TransducerObservation)) + # Lexicon admin.add_view(LexiconTermAdmin(LexiconTerm)) admin.add_view(LexiconCategoryAdmin(LexiconCategory)) diff --git a/admin/views/__init__.py b/admin/views/__init__.py index 74c2c141b..e9fa62617 100644 --- a/admin/views/__init__.py +++ b/admin/views/__init__.py @@ -33,6 +33,7 @@ from admin.views.sample import SampleAdmin from admin.views.geologic_formation import GeologicFormationAdmin from admin.views.data_provenance import DataProvenanceAdmin +from admin.views.transducer_observation import TransducerObservationAdmin from admin.views.field import ( FieldEventAdmin, FieldActivityAdmin, @@ -57,6 +58,7 @@ "SampleAdmin", "GeologicFormationAdmin", "DataProvenanceAdmin", + "TransducerObservationAdmin", "FieldEventAdmin", "FieldActivityAdmin", "FieldEventParticipantAdmin", diff --git a/admin/views/transducer_observation.py b/admin/views/transducer_observation.py new file mode 100644 index 000000000..f0aeca787 --- /dev/null +++ b/admin/views/transducer_observation.py @@ -0,0 +1,186 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +TransducerObservationAdmin view for transducer observations. +""" +from admin.views.base import OcotilloModelView + + +class TransducerObservationAdmin(OcotilloModelView): + """ + Admin view for TransducerObservation model. + """ + + # ========== Basic Configuration ========== + + name = "Transducer Observations" + label = "Transducer Observations" + icon = "fa fa-tachometer-alt" + + # ========== List View ========== + + column_list = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + ] + + column_sortable_list = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + ] + + column_default_sort = ("observation_datetime", True) + + column_filters = [ + "observation_datetime", + "parameter_id", + "deployment_id", + "release_status", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + readonly_fields = [ + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + labels = { + "id": "Observation ID", + "observation_datetime": "Observation Date/Time", + "value": "Value", + "parameter_id": "Parameter", + "deployment_id": "Deployment", + "release_status": "Release Status", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm": "CONDDL (mS/cm)", + "nma_waterlevelscontinuous_pressure_checked_by": "Checked By", + "nma_waterlevelscontinuous_pressure_created": "Created", + "nma_waterlevelscontinuous_pressure_data_source": "Data Source", + "nma_waterlevelscontinuous_pressure_global_id": "Global ID", + "nma_waterlevelscontinuous_pressure_measurement_method": "Measurement Method", + "nma_waterlevelscontinuous_pressure_measuring_agency": "Measuring Agency", + "nma_waterlevelscontinuous_pressure_notes": "Notes", + "nma_waterlevelscontinuous_pressure_processed_by": "Processed By", + "nma_waterlevelscontinuous_pressure_qced": "QCed", + "nma_waterlevelscontinuous_pressure_temperature_water": "Temperature Water", + "nma_waterlevelscontinuous_pressure_updated": "Updated", + "nma_waterlevelscontinuous_pressure_water_head": "Water Head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted": "Water Head Adjusted", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= From 45f1a64969ebc7776271e2082c93e770d2855998 Mon Sep 17 00:00:00 2001 From: "jake.ross" Date: Wed, 7 Jan 2026 20:33:13 -0700 Subject: [PATCH 3/3] feat: refactor _build_itertuples_field_map to use DataFrame for column mapping --- transfers/waterlevels_transducer_transfer.py | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/transfers/waterlevels_transducer_transfer.py b/transfers/waterlevels_transducer_transfer.py index ffd1ad8d4..11048c793 100644 --- a/transfers/waterlevels_transducer_transfer.py +++ b/transfers/waterlevels_transducer_transfer.py @@ -14,7 +14,6 @@ # limitations under the License. # =============================================================================== -import keyword import pandas as pd from pandas import Timestamp from pydantic import ValidationError @@ -59,9 +58,7 @@ def _get_dfs(self): cleaned_df = cleaned_df.drop_duplicates(subset=["PointID", "DateMeasured"]) self._df_columns = set(cleaned_df.columns) - self._itertuples_field_map = self._build_itertuples_field_map( - cleaned_df.columns - ) + self._itertuples_field_map = self._build_itertuples_field_map(cleaned_df) return input_df, cleaned_df @@ -239,19 +236,21 @@ def val(key: str): } @staticmethod - def _build_itertuples_field_map(columns: list[str]) -> dict[str, str]: + def _build_itertuples_field_map(df: pd.DataFrame) -> dict[str, str]: """ - Map original column names to itertuples field names for invalid identifiers. + Map original column names to itertuples field names using pandas' rename logic. """ - invalid_count = 0 mapping: dict[str, str] = {} - for col in columns: - if not isinstance(col, str): - continue - if col.isidentifier() and not keyword.iskeyword(col): - continue - invalid_count += 1 - mapping[col] = f"_{invalid_count}" + iterator = df.itertuples() + first_row = next(iterator, None) + if first_row is None: + return mapping + + fields = first_row._fields + for idx, col in enumerate(df.columns): + field = fields[idx + 1] + if field != col: + mapping[col] = field return mapping