diff --git a/admin/config.py b/admin/config.py index 1c43f15e6..2ba35b4f8 100644 --- a/admin/config.py +++ b/admin/config.py @@ -39,6 +39,7 @@ ChemistrySampleInfoAdmin, GeologicFormationAdmin, DataProvenanceAdmin, + TransducerObservationAdmin, FieldEventAdmin, FieldActivityAdmin, ParameterAdmin, @@ -64,6 +65,7 @@ from db.nma_legacy import ChemistrySampleInfo from db.geologic_formation import GeologicFormation from db.data_provenance import DataProvenance +from db.transducer import TransducerObservation from db.field import FieldEvent, FieldActivity from db.parameter import Parameter @@ -142,6 +144,9 @@ def create_admin(app): # Data provenance admin.add_view(DataProvenanceAdmin(DataProvenance)) + # Transducer observations + admin.add_view(TransducerObservationAdmin(TransducerObservation)) + # Lexicon admin.add_view(LexiconTermAdmin(LexiconTerm)) admin.add_view(LexiconCategoryAdmin(LexiconCategory)) diff --git a/admin/views/__init__.py b/admin/views/__init__.py index 6b1de4c09..ffb8a25fe 100644 --- a/admin/views/__init__.py +++ b/admin/views/__init__.py @@ -34,6 +34,7 @@ from admin.views.chemistry_sampleinfo import ChemistrySampleInfoAdmin from admin.views.geologic_formation import GeologicFormationAdmin from admin.views.data_provenance import DataProvenanceAdmin +from admin.views.transducer_observation import TransducerObservationAdmin from admin.views.field import ( FieldEventAdmin, FieldActivityAdmin, @@ -59,6 +60,7 @@ "ChemistrySampleInfoAdmin", "GeologicFormationAdmin", "DataProvenanceAdmin", + "TransducerObservationAdmin", "FieldEventAdmin", "FieldActivityAdmin", "FieldEventParticipantAdmin", diff --git a/admin/views/transducer_observation.py b/admin/views/transducer_observation.py new file mode 100644 index 000000000..f0aeca787 --- /dev/null +++ b/admin/views/transducer_observation.py @@ -0,0 +1,186 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +TransducerObservationAdmin view for transducer observations. +""" +from admin.views.base import OcotilloModelView + + +class TransducerObservationAdmin(OcotilloModelView): + """ + Admin view for TransducerObservation model. + """ + + # ========== Basic Configuration ========== + + name = "Transducer Observations" + label = "Transducer Observations" + icon = "fa fa-tachometer-alt" + + # ========== List View ========== + + column_list = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + ] + + column_sortable_list = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + ] + + column_default_sort = ("observation_datetime", True) + + column_filters = [ + "observation_datetime", + "parameter_id", + "deployment_id", + "release_status", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "id", + "observation_datetime", + "value", + "parameter_id", + "deployment_id", + "release_status", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + ] + + exclude_fields_from_create = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "updated_by_id", + "updated_by_name", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + exclude_fields_from_edit = [ + "id", + "created_at", + "created_by_id", + "created_by_name", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + readonly_fields = [ + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + "nma_waterlevelscontinuous_pressure_checked_by", + "nma_waterlevelscontinuous_pressure_created", + "nma_waterlevelscontinuous_pressure_data_source", + "nma_waterlevelscontinuous_pressure_global_id", + "nma_waterlevelscontinuous_pressure_measurement_method", + "nma_waterlevelscontinuous_pressure_measuring_agency", + "nma_waterlevelscontinuous_pressure_notes", + "nma_waterlevelscontinuous_pressure_processed_by", + "nma_waterlevelscontinuous_pressure_qced", + "nma_waterlevelscontinuous_pressure_temperature_water", + "nma_waterlevelscontinuous_pressure_updated", + "nma_waterlevelscontinuous_pressure_water_head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ] + + labels = { + "id": "Observation ID", + "observation_datetime": "Observation Date/Time", + "value": "Value", + "parameter_id": "Parameter", + "deployment_id": "Deployment", + "release_status": "Release Status", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm": "CONDDL (mS/cm)", + "nma_waterlevelscontinuous_pressure_checked_by": "Checked By", + "nma_waterlevelscontinuous_pressure_created": "Created", + "nma_waterlevelscontinuous_pressure_data_source": "Data Source", + "nma_waterlevelscontinuous_pressure_global_id": "Global ID", + "nma_waterlevelscontinuous_pressure_measurement_method": "Measurement Method", + "nma_waterlevelscontinuous_pressure_measuring_agency": "Measuring Agency", + "nma_waterlevelscontinuous_pressure_notes": "Notes", + "nma_waterlevelscontinuous_pressure_processed_by": "Processed By", + "nma_waterlevelscontinuous_pressure_qced": "QCed", + "nma_waterlevelscontinuous_pressure_temperature_water": "Temperature Water", + "nma_waterlevelscontinuous_pressure_updated": "Updated", + "nma_waterlevelscontinuous_pressure_water_head": "Water Head", + "nma_waterlevelscontinuous_pressure_water_head_adjusted": "Water Head Adjusted", + "created_at": "Created At", + "created_by_name": "Created By", + "updated_by_name": "Updated By", + } + + +# ============= EOF ============================================= diff --git a/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py b/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py new file mode 100644 index 000000000..e2e19c03b --- /dev/null +++ b/alembic/versions/c9f1d2e3a4b5_add_nma_waterlevelscontinuous_pressure_fields_to_transducer_observation.py @@ -0,0 +1,193 @@ +"""Add NMA WaterLevelsContinuous_Pressure fields to transducer_observation. + +Revision ID: c9f1d2e3a4b5 +Revises: b7d4c6a1b2c3 +Create Date: 2026-02-10 03:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "c9f1d2e3a4b5" +down_revision: Union[str, Sequence[str], None] = "b7d4c6a1b2c3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add legacy NMA WaterLevelsContinuous_Pressure columns.""" + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_checked_by", + sa.String(length=4), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_created", + sa.DateTime(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_data_source", + sa.String(length=5), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_global_id", + sa.String(length=40), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_measurement_method", + sa.String(length=2), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_measuring_agency", + sa.String(length=50), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_notes", + sa.String(length=100), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_processed_by", + sa.String(length=4), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_qced", + sa.Boolean(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_temperature_water", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_updated", + sa.DateTime(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_water_head", + sa.Float(), + nullable=True, + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + sa.Float(), + nullable=True, + ), + ) + + +def downgrade() -> None: + """Drop legacy NMA WaterLevelsContinuous_Pressure columns.""" + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_water_head_adjusted", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_water_head", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_updated", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_temperature_water", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_qced", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_processed_by", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_notes", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_measuring_agency", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_measurement_method", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_global_id", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_data_source", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_created", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_checked_by", + ) + op.drop_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_conddl_ms_cm", + ) diff --git a/db/transducer.py b/db/transducer.py index 25928c9ca..608260ce5 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -18,9 +18,11 @@ from typing import TYPE_CHECKING from sqlalchemy import ( + Boolean, ForeignKey, Float, DateTime, + String, Text, CheckConstraint, Index, @@ -118,6 +120,48 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): DateTime(timezone=True), nullable=False, index=True ) value: Mapped[float] = mapped_column(Float, nullable=False) + nma_waterlevelscontinuous_pressure_conddl_ms_cm: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_checked_by: Mapped[str] = mapped_column( + String(4), nullable=True + ) + nma_waterlevelscontinuous_pressure_created: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + nma_waterlevelscontinuous_pressure_data_source: Mapped[str] = mapped_column( + String(5), nullable=True + ) + nma_waterlevelscontinuous_pressure_global_id: Mapped[str] = mapped_column( + String(40), nullable=True + ) + nma_waterlevelscontinuous_pressure_measurement_method: Mapped[str] = mapped_column( + String(2), nullable=True + ) + nma_waterlevelscontinuous_pressure_measuring_agency: Mapped[str] = mapped_column( + String(50), nullable=True + ) + nma_waterlevelscontinuous_pressure_notes: Mapped[str] = mapped_column( + String(100), nullable=True + ) + nma_waterlevelscontinuous_pressure_processed_by: Mapped[str] = mapped_column( + String(4), nullable=True + ) + nma_waterlevelscontinuous_pressure_qced: Mapped[bool] = mapped_column( + Boolean, nullable=True + ) + nma_waterlevelscontinuous_pressure_temperature_water: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_updated: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + nma_waterlevelscontinuous_pressure_water_head: Mapped[float] = mapped_column( + Float, nullable=True + ) + nma_waterlevelscontinuous_pressure_water_head_adjusted: Mapped[float] = ( + mapped_column(Float, nullable=True) + ) # qc_block_id: Mapped[Optional[int]] = mapped_column( # ForeignKey("transducer_observation_block.id", ondelete="SET NULL"), index=True diff --git a/transfers/waterlevels_transducer_transfer.py b/transfers/waterlevels_transducer_transfer.py index 338e71027..11048c793 100644 --- a/transfers/waterlevels_transducer_transfer.py +++ b/transfers/waterlevels_transducer_transfer.py @@ -39,6 +39,8 @@ class WaterLevelsContinuousTransferer(Transferer): def __init__(self, *args, **kw): super().__init__(*args, **kw) self.groundwater_parameter_id = get_groundwater_parameter_id() + self._itertuples_field_map = {} + self._df_columns = set() if self._sensor_types is None: raise ValueError("_sensor_types must be set") if self._partition_field is None: @@ -55,6 +57,9 @@ def _get_dfs(self): # remove duplicate rows cleaned_df = cleaned_df.drop_duplicates(subset=["PointID", "DateMeasured"]) + self._df_columns = set(cleaned_df.columns) + self._itertuples_field_map = self._build_itertuples_field_map(cleaned_df) + return input_df, cleaned_df def _transfer_hook(self, session: Session) -> None: @@ -188,12 +193,66 @@ def _make_observation( obspayload = CreateTransducerObservation.model_validate( payload ).model_dump() - return TransducerObservation(**obspayload) + legacy_payload = self._legacy_payload(row) + return TransducerObservation(**obspayload, **legacy_payload) except ValidationError as e: logger.critical(f"Observation validation error: {e.errors()}") self._capture_error(pointid, str(e), "DepthToWaterBGS") + def _legacy_payload(self, row: pd.Series) -> dict: + def val(key: str): + if key not in self._df_columns: + return None + field = self._itertuples_field_map.get(key, key) + v = getattr(row, field, None) + if pd.isna(v): + return None + return v + + return { + "nma_waterlevelscontinuous_pressure_conddl_ms_cm": val("CONDDL (mS/cm)"), + "nma_waterlevelscontinuous_pressure_checked_by": val("CheckedBy"), + "nma_waterlevelscontinuous_pressure_created": val("Created"), + "nma_waterlevelscontinuous_pressure_data_source": val("DataSource"), + "nma_waterlevelscontinuous_pressure_global_id": val("GlobalID"), + "nma_waterlevelscontinuous_pressure_measurement_method": val( + "MeasurementMethod" + ), + "nma_waterlevelscontinuous_pressure_measuring_agency": val( + "MeasuringAgency" + ), + "nma_waterlevelscontinuous_pressure_notes": val("Notes"), + "nma_waterlevelscontinuous_pressure_processed_by": val("ProcessedBy"), + "nma_waterlevelscontinuous_pressure_qced": val("QCed"), + "nma_waterlevelscontinuous_pressure_temperature_water": val( + "TemperatureWater" + ), + "nma_waterlevelscontinuous_pressure_updated": val("Updated"), + "nma_waterlevelscontinuous_pressure_water_head": val("WaterHead"), + "nma_waterlevelscontinuous_pressure_water_head_adjusted": val( + "WaterHeadAdjusted" + ), + } + + @staticmethod + def _build_itertuples_field_map(df: pd.DataFrame) -> dict[str, str]: + """ + Map original column names to itertuples field names using pandas' rename logic. + """ + mapping: dict[str, str] = {} + iterator = df.itertuples() + first_row = next(iterator, None) + if first_row is None: + return mapping + + fields = first_row._fields + for idx, col in enumerate(df.columns): + field = fields[idx + 1] + if field != col: + mapping[col] = field + return mapping + class WaterLevelsContinuousPressureTransferer(WaterLevelsContinuousTransferer): source_table = "WaterLevelsContinuous_Pressure"