diff --git a/admin/config.py b/admin/config.py index e88dfdc37..adeab0c70 100644 --- a/admin/config.py +++ b/admin/config.py @@ -36,11 +36,11 @@ GroupAdmin, NotesAdmin, SampleAdmin, + HydraulicsDataAdmin, GeologicFormationAdmin, DataProvenanceAdmin, FieldEventAdmin, FieldActivityAdmin, - FieldEventParticipantAdmin, ParameterAdmin, ) from db.engine import engine @@ -60,10 +60,10 @@ from db.group import Group from db.notes import Notes from db.sample import Sample +from db.nma_legacy import NMAHydraulicsData from db.geologic_formation import GeologicFormation from db.data_provenance import DataProvenance -from db.field import FieldEvent, FieldActivity, FieldEventParticipant -from db.permission_history import PermissionHistory +from db.field import FieldEvent, FieldActivity from db.parameter import Parameter @@ -127,6 +127,9 @@ def create_admin(app): # Samples admin.add_view(SampleAdmin(Sample)) + # Hydraulics + admin.add_view(HydraulicsDataAdmin(NMAHydraulicsData)) + # Field admin.add_view(FieldEventAdmin(FieldEvent)) admin.add_view(FieldActivityAdmin(FieldActivity)) diff --git a/admin/views/__init__.py b/admin/views/__init__.py index 74c2c141b..55b7bbd20 100644 --- a/admin/views/__init__.py +++ b/admin/views/__init__.py @@ -31,6 +31,7 @@ from admin.views.group import GroupAdmin from admin.views.notes import NotesAdmin from admin.views.sample import SampleAdmin +from admin.views.hydraulicsdata import HydraulicsDataAdmin from admin.views.geologic_formation import GeologicFormationAdmin from admin.views.data_provenance import DataProvenanceAdmin from admin.views.field import ( @@ -55,6 +56,7 @@ "GroupAdmin", "NotesAdmin", "SampleAdmin", + "HydraulicsDataAdmin", "GeologicFormationAdmin", "DataProvenanceAdmin", "FieldEventAdmin", diff --git a/admin/views/hydraulicsdata.py b/admin/views/hydraulicsdata.py new file mode 100644 index 000000000..6082ce50f --- /dev/null +++ b/admin/views/hydraulicsdata.py @@ -0,0 +1,129 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +HydraulicsDataAdmin view for legacy NMA_HydraulicsData. +""" +from admin.views.base import OcotilloModelView + + +class HydraulicsDataAdmin(OcotilloModelView): + """ + Admin view for NMAHydraulicsData model. + """ + + # ========== Basic Configuration ========== + + name = "Hydraulics Data" + label = "Hydraulics Data" + icon = "fa fa-tint" + + can_create = False + can_edit = False + can_delete = False + + # ========== List View ========== + + column_list = [ + "global_id", + "point_id", + "hydraulic_unit", + "hydraulic_unit_type", + "test_top", + "test_bottom", + "t_ft2_d", + "k_darcy", + "data_source", + ] + + column_sortable_list = [ + "global_id", + "point_id", + "hydraulic_unit", + "hydraulic_unit_type", + "test_top", + "test_bottom", + "t_ft2_d", + "k_darcy", + "data_source", + ] + + search_fields = [ + "global_id", + "point_id", + "hydraulic_unit", + "hydraulic_remarks", + ] + + column_filters = [ + "hydraulic_unit", + "hydraulic_unit_type", + "data_source", + ] + + can_export = True + export_types = ["csv", "excel"] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + + fields = [ + "global_id", + "point_id", + "hydraulic_unit", + "hydraulic_unit_type", + "hydraulic_remarks", + "test_top", + "test_bottom", + "t_ft2_d", + "s_dimensionless", + "ss_ft_1", + "sy_decimalfractn", + "kh_ft_d", + "kv_ft_d", + "hl_day_1", + "hd_ft2_d", + "cs_gal_d_ft", + "p_decimal_fraction", + "k_darcy", + "data_source", + ] + + labels = { + "global_id": "GlobalID", + "point_id": "PointID", + "hydraulic_unit": "HydraulicUnit", + "hydraulic_unit_type": "HydraulicUnitType", + "hydraulic_remarks": "Hydraulic Remarks", + "test_top": "TestTop", + "test_bottom": "TestBottom", + "t_ft2_d": "T (ft2/d)", + "s_dimensionless": "S (dimensionless)", + "ss_ft_1": "Ss (ft-1)", + "sy_decimalfractn": "Sy (decimalfractn)", + "kh_ft_d": "KH (ft/d)", + "kv_ft_d": "KV (ft/d)", + "hl_day_1": "HL (day-1)", + "hd_ft2_d": "HD (ft2/d)", + "cs_gal_d_ft": "Cs (gal/d/ft)", + "p_decimal_fraction": "P (decimal fraction)", + "k_darcy": "k (darcy)", + "data_source": "Data Source", + } + + +# ============= EOF ============================================= diff --git a/alembic/versions/d1a2b3c4e5f6_create_nma_hydraulicsdata.py b/alembic/versions/d1a2b3c4e5f6_create_nma_hydraulicsdata.py new file mode 100644 index 000000000..59368b502 --- /dev/null +++ b/alembic/versions/d1a2b3c4e5f6_create_nma_hydraulicsdata.py @@ -0,0 +1,55 @@ +"""Create legacy NMA_HydraulicsData table. + +Revision ID: d1a2b3c4e5f6 +Revises: c9f1d2e3a4b5 +Create Date: 2026-02-10 04:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect + +# revision identifiers, used by Alembic. +revision: str = "d1a2b3c4e5f6" +down_revision: Union[str, Sequence[str], None] = "c9f1d2e3a4b5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the legacy hydraulics data table used for backfill.""" + bind = op.get_bind() + inspector = inspect(bind) + if not inspector.has_table("NMA_HydraulicsData"): + op.create_table( + "NMA_HydraulicsData", + sa.Column("GlobalID", sa.String(length=40), primary_key=True), + sa.Column("PointID", sa.String(length=50), nullable=True), + sa.Column("HydraulicUnit", sa.String(length=18), nullable=True), + sa.Column("TestTop", sa.SmallInteger(), nullable=False), + sa.Column("TestBottom", sa.SmallInteger(), nullable=False), + sa.Column("HydraulicUnitType", sa.String(length=2), nullable=True), + sa.Column("Hydraulic Remarks", sa.String(length=200), nullable=True), + sa.Column("T (ft2/d)", sa.Float(), nullable=True), + sa.Column("S (dimensionless)", sa.Float(), nullable=True), + sa.Column("Ss (ft-1)", sa.Float(), nullable=True), + sa.Column("Sy (decimalfractn)", sa.Float(), nullable=True), + sa.Column("KH (ft/d)", sa.Float(), nullable=True), + sa.Column("KV (ft/d)", sa.Float(), nullable=True), + sa.Column("HL (day-1)", sa.Float(), nullable=True), + sa.Column("HD (ft2/d)", sa.Float(), nullable=True), + sa.Column("Cs (gal/d/ft)", sa.Float(), nullable=True), + sa.Column("P (decimal fraction)", sa.Float(), nullable=True), + sa.Column("k (darcy)", sa.Float(), nullable=True), + sa.Column("Data Source", sa.String(length=255), nullable=True), + ) + + +def downgrade() -> None: + """Drop the legacy hydraulics data table.""" + bind = op.get_bind() + inspector = inspect(bind) + if inspector.has_table("NMA_HydraulicsData"): + op.drop_table("NMA_HydraulicsData") diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 8033dcc47..cc7e02d43 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -19,14 +19,7 @@ from datetime import date, datetime from typing import Optional -from sqlalchemy import ( - Boolean, - Date, - DateTime, - Float, - Integer, - String, -) +from sqlalchemy import Boolean, Date, DateTime, Float, Integer, SmallInteger, String from sqlalchemy.orm import Mapped, mapped_column from db.base import Base @@ -148,4 +141,42 @@ class ViewNGWMNLithology(Base): ) +class NMAHydraulicsData(Base): + """ + Legacy HydraulicsData table from AMPAPI. + """ + + __tablename__ = "NMA_HydraulicsData" + + global_id: Mapped[str] = mapped_column("GlobalID", String(40), primary_key=True) + point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) + data_source: Mapped[Optional[str]] = mapped_column("Data Source", String(255)) + + cs_gal_d_ft: Mapped[Optional[float]] = mapped_column("Cs (gal/d/ft)", Float) + hd_ft2_d: Mapped[Optional[float]] = mapped_column("HD (ft2/d)", Float) + hl_day_1: Mapped[Optional[float]] = mapped_column("HL (day-1)", Float) + kh_ft_d: Mapped[Optional[float]] = mapped_column("KH (ft/d)", Float) + kv_ft_d: Mapped[Optional[float]] = mapped_column("KV (ft/d)", Float) + p_decimal_fraction: Mapped[Optional[float]] = mapped_column( + "P (decimal fraction)", Float + ) + s_dimensionless: Mapped[Optional[float]] = mapped_column("S (dimensionless)", Float) + ss_ft_1: Mapped[Optional[float]] = mapped_column("Ss (ft-1)", Float) + sy_decimalfractn: Mapped[Optional[float]] = mapped_column( + "Sy (decimalfractn)", Float + ) + t_ft2_d: Mapped[Optional[float]] = mapped_column("T (ft2/d)", Float) + k_darcy: Mapped[Optional[float]] = mapped_column("k (darcy)", Float) + + test_bottom: Mapped[int] = mapped_column("TestBottom", SmallInteger, nullable=False) + test_top: Mapped[int] = mapped_column("TestTop", SmallInteger, nullable=False) + hydraulic_unit: Mapped[Optional[str]] = mapped_column("HydraulicUnit", String(18)) + hydraulic_unit_type: Mapped[Optional[str]] = mapped_column( + "HydraulicUnitType", String(2) + ) + hydraulic_remarks: Mapped[Optional[str]] = mapped_column( + "Hydraulic Remarks", String(200) + ) + + # ============= EOF ============================================= diff --git a/transfers/backfill/hydraulicsdata.py b/transfers/backfill/hydraulicsdata.py new file mode 100644 index 000000000..8972eec72 --- /dev/null +++ b/transfers/backfill/hydraulicsdata.py @@ -0,0 +1,151 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== + +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session + +from db import NMAHydraulicsData +from transfers.logger import logger +from transfers.transferer import Transferer +from transfers.util import read_csv + + +class NMAHydraulicsDataBackfill(Transferer): + """ + Backfill for the legacy NMA_HydraulicsData table. + """ + + source_table = "HydraulicsData" + + def __init__(self, *args, batch_size: int = 1000, **kwargs): + super().__init__(*args, **kwargs) + self.batch_size = batch_size + + def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: + input_df = read_csv(self.source_table) + return input_df, input_df + + def _transfer_hook(self, session: Session) -> None: + rows = self._dedupe_rows( + [self._row_dict(row) for row in self.cleaned_df.to_dict("records")], + key="GlobalID", + ) + + insert_stmt = insert(NMAHydraulicsData) + excluded = insert_stmt.excluded + + for i in range(0, len(rows), self.batch_size): + chunk = rows[i : i + self.batch_size] + logger.info( + f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows) into NMA_HydraulicsData" + ) + stmt = insert_stmt.values(chunk).on_conflict_do_update( + index_elements=["GlobalID"], + set_={ + "PointID": excluded["PointID"], + "HydraulicUnit": excluded["HydraulicUnit"], + "TestTop": excluded["TestTop"], + "TestBottom": excluded["TestBottom"], + "HydraulicUnitType": excluded["HydraulicUnitType"], + "Hydraulic Remarks": excluded["Hydraulic Remarks"], + "T (ft2/d)": excluded["T (ft2/d)"], + "S (dimensionless)": excluded["S (dimensionless)"], + "Ss (ft-1)": excluded["Ss (ft-1)"], + "Sy (decimalfractn)": excluded["Sy (decimalfractn)"], + "KH (ft/d)": excluded["KH (ft/d)"], + "KV (ft/d)": excluded["KV (ft/d)"], + "HL (day-1)": excluded["HL (day-1)"], + "HD (ft2/d)": excluded["HD (ft2/d)"], + "Cs (gal/d/ft)": excluded["Cs (gal/d/ft)"], + "P (decimal fraction)": excluded["P (decimal fraction)"], + "k (darcy)": excluded["k (darcy)"], + "Data Source": excluded["Data Source"], + }, + ) + session.execute(stmt) + session.commit() + session.expunge_all() + + def _row_dict(self, row: dict[str, Any]) -> dict[str, Any]: + def val(key: str) -> Optional[Any]: + v = row.get(key) + if pd.isna(v): + return None + return v + + def as_int(key: str) -> Optional[int]: + v = val(key) + if v is None: + return None + try: + return int(v) + except (TypeError, ValueError): + return None + + return { + "GlobalID": val("GlobalID"), + "PointID": val("PointID"), + "HydraulicUnit": val("HydraulicUnit"), + "TestTop": as_int("TestTop"), + "TestBottom": as_int("TestBottom"), + "HydraulicUnitType": val("HydraulicUnitType"), + "Hydraulic Remarks": val("Hydraulic Remarks"), + "T (ft2/d)": val("T (ft2/d)"), + "S (dimensionless)": val("S (dimensionless)"), + "Ss (ft-1)": val("Ss (ft-1)"), + "Sy (decimalfractn)": val("Sy (decimalfractn)"), + "KH (ft/d)": val("KH (ft/d)"), + "KV (ft/d)": val("KV (ft/d)"), + "HL (day-1)": val("HL (day-1)"), + "HD (ft2/d)": val("HD (ft2/d)"), + "Cs (gal/d/ft)": val("Cs (gal/d/ft)"), + "P (decimal fraction)": val("P (decimal fraction)"), + "k (darcy)": val("k (darcy)"), + "Data Source": val("Data Source"), + } + + def _dedupe_rows( + self, rows: list[dict[str, Any]], key: str + ) -> list[dict[str, Any]]: + """ + Deduplicate rows within a batch by the given key to avoid ON CONFLICT loops. + Later rows win. + """ + deduped = {} + for row in rows: + gid = row.get(key) + if gid is None: + continue + deduped[gid] = row + return list(deduped.values()) + + +def run(batch_size: int = 1000) -> None: + """Entrypoint to execute the backfill.""" + transferer = NMAHydraulicsDataBackfill(batch_size=batch_size) + transferer.transfer() + + +if __name__ == "__main__": + # Allow running via `python -m transfers.backfill.hydraulicsdata` + run() + +# ============= EOF ============================================= diff --git a/transfers/backfill/staging.py b/transfers/backfill/staging.py index 172b67371..679db5c4c 100644 --- a/transfers/backfill/staging.py +++ b/transfers/backfill/staging.py @@ -32,6 +32,8 @@ from transfers.backfill.waterlevelscontinuous_pressure_daily import ( run as run_pressure_daily, ) +from transfers.backfill.chemistry_sampleinfo import run as run_chemistry_sampleinfo +from transfers.backfill.hydraulicsdata import run as run_hydraulicsdata from transfers.logger import logger @@ -41,6 +43,8 @@ def run(batch_size: int = 1000) -> None: """ steps = ( ("WaterLevelsContinuous_Pressure_Daily", run_pressure_daily), + ("Chemistry_SampleInfo", run_chemistry_sampleinfo), + ("HydraulicsData", run_hydraulicsdata), ("NGWMN views", run_ngwmn_views), )