From bf596bdd15b5856f07291437faddfca4990b74bd Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 16:20:32 -0600 Subject: [PATCH 01/41] feat(transfers): NM_Wells 1:1 staging mirror + ref-table lexicon loader Phase 1 of the NM_Wells -> Ocotillo migration: faithful column-for-column staging mirror of the legacy NM_Wells SQL Server DB, plus loaders. The transform into the Ocotillo model (Phase 2) is documented inline but not built. - db/nmw_legacy.py: 17 NMW_* mirror models (5 Main, 7 Geothermal, 5 DST), source column names preserved, per-column Phase-2 transform-target notes. Main columns from the planning workbook field map; Geothermal/DST columns, lengths and PKs taken directly from the SQL-dump DDL. - alembic: two migrations (Main; Geothermal+DST) chained off current head, bodies generated from model metadata. Single head. - transfers/nmw_mirror_transfer.py: data-driven CSV -> NMW_* loader with type coercion (NaN/NaT -> None, rowversion dropped), chunked ON CONFLICT upsert. Gated by TRANSFER_NMW_MIRROR (default off; separate source DB). - transfers/reference_lexicon_transfer.py: loads all 49 ref_* lookups into the lexicon (category per table), idempotent like init_lexicon; registered as a foundational transfer. - db/__init__.py, transfers/transfer.py, .env.example: wiring. Co-Authored-By: Claude Opus 4.8 --- .env.example | 2 + ...x0y1z2_nmw_legacy_staging_mirror_tables.py | 211 ++++++ ...y1z2a3_nmw_geothermal_dst_mirror_tables.py | 341 +++++++++ db/__init__.py | 1 + db/nmw_legacy.py | 647 ++++++++++++++++++ transfers/nmw_mirror_transfer.py | 250 +++++++ transfers/reference_lexicon_transfer.py | 362 ++++++++++ transfers/transfer.py | 12 +- 8 files changed, 1825 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py create mode 100644 alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py create mode 100644 db/nmw_legacy.py create mode 100644 transfers/nmw_mirror_transfer.py create mode 100644 transfers/reference_lexicon_transfer.py diff --git a/.env.example b/.env.example index 3f835882e..2c4534696 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,8 @@ TRANSFER_NGWMN_VIEWS=True TRANSFER_WATERLEVELS_PRESSURE_DAILY=True TRANSFER_WEATHER_DATA=True TRANSFER_MINOR_TRACE_CHEMISTRY=True +# NM_Wells 1:1 staging mirror load (separate source DB; off by default) +TRANSFER_NMW_MIRROR=False # asset storage GCS_BUCKET_NAME= diff --git a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py new file mode 100644 index 000000000..7fb64962b --- /dev/null +++ b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py @@ -0,0 +1,211 @@ +"""NM_Wells 1:1 staging mirror tables + +Revision ID: u7v8w9x0y1z2 +Revises: t6u7v8w9x0y1 +Create Date: 2026-06-06 00:00:00.000000 + +1:1 staging mirror of the legacy NM_Wells SQL Server "Migrate First / Main" +tables (see db/nmw_legacy.py and docs/nm_wells-migration.md). Faithful, +column-for-column copies; the transform into the Ocotillo data model is a +later phase. + + tbl_well_locations -> NMW_WellLocations + tbl_well_headers -> NMW_WellHeaders + tbl_well_records -> NMW_WellRecords + tbl_well_z_datum -> NMW_WellZDatum + tbl_well_samples -> NMW_WellSamples +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "u7v8w9x0y1z2" +down_revision: Union[str, Sequence[str], None] = "t6u7v8w9x0y1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "NMW_WellLocations", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Well_ID", sa.String(), nullable=True), + sa.Column("Import_ID", sa.Integer(), nullable=True), + sa.Column("Township", sa.Float(), nullable=True), + sa.Column("NorS_TDir", sa.String(), nullable=True), + sa.Column("Range", sa.Float(), nullable=True), + sa.Column("EorW_RDir", sa.String(), nullable=True), + sa.Column("Sectn", sa.SmallInteger(), nullable=True), + sa.Column("SectnPart", sa.String(), nullable=True), + sa.Column("UnitLetter", sa.String(), nullable=True), + sa.Column("UTM_zone", sa.String(), nullable=True), + sa.Column("State", sa.String(), nullable=True), + sa.Column("County", sa.String(), nullable=True), + sa.Column("Basin", sa.String(), nullable=True), + sa.Column("Footage_NS", sa.Float(), nullable=True), + sa.Column("NorS_FDir", sa.String(), nullable=True), + sa.Column("Footage_EW", sa.Float(), nullable=True), + sa.Column("EorW_FDir", sa.String(), nullable=True), + sa.Column("Lat_min", sa.SmallInteger(), nullable=True), + sa.Column("Lat_sec", sa.Float(), nullable=True), + sa.Column("Long_deg", sa.SmallInteger(), nullable=True), + sa.Column("Long_min", sa.SmallInteger(), nullable=True), + sa.Column("Long_sec", sa.Float(), nullable=True), + sa.Column("Lat_dd27", sa.Float(), nullable=True), + sa.Column("Long_dd27", sa.Float(), nullable=True), + sa.Column("Lat_dd83", sa.Float(), nullable=True), + sa.Column("Long_dd83", sa.Float(), nullable=True), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("SourceDatum", sa.String(), nullable=True), + sa.Column("SourceUnits", sa.String(), nullable=True), + sa.Column("LocAccType", sa.String(), nullable=True), + sa.Column("LocAccMeas", sa.String(), nullable=True), + sa.Column("LocAccVal", sa.Float(), nullable=True), + sa.Column("Duplicated", sa.SmallInteger(), nullable=True), + sa.Column("Exclude", sa.SmallInteger(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.Column("API", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WellLocations_WellDataID", "NMW_WellLocations", ["WellDataID"] + ) + + op.create_table( + "NMW_WellHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("WellSpotID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("API", sa.String(), nullable=True), + sa.Column("WellClass", sa.String(), nullable=True), + sa.Column("WellType", sa.String(), nullable=True), + sa.Column("WellOrient", sa.String(), nullable=True), + sa.Column("CurWellNam", sa.String(), nullable=True), + sa.Column("CurWellNum", sa.String(), nullable=True), + sa.Column("CurStatus", sa.String(), nullable=True), + sa.Column("PrdPoolCnt", sa.SmallInteger(), nullable=True), + sa.Column("CurOperatr", sa.String(), nullable=True), + sa.Column("CurOwner", sa.String(), nullable=True), + sa.Column("TotalDepth", sa.Float(), nullable=True), + sa.Column("Well_TVD", sa.Float(), nullable=True), + sa.Column("Fm_TD", sa.String(), nullable=True), + sa.Column("Age_TD", sa.String(), nullable=True), + sa.Column("SpudDate", sa.DateTime(), nullable=True), + sa.Column("ComplDate", sa.DateTime(), nullable=True), + sa.Column("PlugDate", sa.DateTime(), nullable=True), + sa.Column("PlugBack", sa.Float(), nullable=True), + sa.Column("BridgePlug", sa.String(), nullable=True), + sa.Column("ScoutTickt", sa.SmallInteger(), nullable=True), + sa.Column("DwnHoleSur", sa.SmallInteger(), nullable=True), + sa.Column("GeolLog", sa.SmallInteger(), nullable=True), + sa.Column("Geophyslog", sa.SmallInteger(), nullable=True), + sa.Column("GthrmExist", sa.SmallInteger(), nullable=True), + sa.Column("PetroData", sa.SmallInteger(), nullable=True), + sa.Column("CoreExists", sa.SmallInteger(), nullable=True), + sa.Column("Cuttings", sa.SmallInteger(), nullable=True), + sa.Column("SampleData", sa.SmallInteger(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("Import_ID", sa.String(), nullable=True), + sa.Column("Import_DB", sa.String(), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("WellDataID"), + ) + + op.create_table( + "NMW_WellRecords", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("RecrdClass", sa.String(), nullable=True), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("ActionDate", sa.DateTime(), nullable=True), + sa.Column("WellName", sa.String(), nullable=True), + sa.Column("WellNumber", sa.String(), nullable=True), + sa.Column("API_suffix", sa.String(), nullable=True), + sa.Column("EnteredBy", sa.String(), nullable=True), + sa.Column("EntryDate", sa.DateTime(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("RecrdSetID"), + ) + op.create_index("ix_NMW_WellRecords_WellDataID", "NMW_WellRecords", ["WellDataID"]) + + op.create_table( + "NMW_WellZDatum", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Elev_GL", sa.Float(), nullable=True), + sa.Column("Elev_DF", sa.Float(), nullable=True), + sa.Column("Elev_KB", sa.Float(), nullable=True), + sa.Column("Elev_unspc", sa.Float(), nullable=True), + sa.Column("DatumElev", sa.Float(), nullable=True), + sa.Column("DepthDatum", sa.String(), nullable=True), + sa.Column("DepthUnits", sa.String(), nullable=True), + sa.Column("Z_datum", sa.String(), nullable=True), + sa.Column("Z_units", sa.String(), nullable=True), + sa.Column("ElevSource", sa.String(), nullable=True), + sa.Column("ElvAccType", sa.String(), nullable=True), + sa.Column("ElvAccMeas", sa.String(), nullable=True), + sa.Column("ElvAccVal", sa.Float(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("GlobalID"), + ) + op.create_index("ix_NMW_WellZDatum_RecrdsetID", "NMW_WellZDatum", ["RecrdsetID"]) + + op.create_table( + "NMW_WellSamples", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SmpSetName", sa.String(), nullable=True), + sa.Column("SamplClass", sa.String(), nullable=True), + sa.Column("SampleType", sa.String(), nullable=True), + sa.Column("SampleFm", sa.String(), nullable=True), + sa.Column("SampleLoc", sa.String(), nullable=True), + sa.Column("SampleDate", sa.DateTime(), nullable=True), + sa.Column("From_Depth", sa.Float(), nullable=True), + sa.Column("To_Depth", sa.Float(), nullable=True), + sa.Column("SmpDpUnt", sa.String(), nullable=True), + sa.Column("From_TVD", sa.Float(), nullable=True), + sa.Column("To_TVD", sa.Float(), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("Porosity", sa.SmallInteger(), nullable=True), + sa.Column("Permeablty", sa.SmallInteger(), nullable=True), + sa.Column("Density", sa.SmallInteger(), nullable=True), + sa.Column("DST_Tests", sa.SmallInteger(), nullable=True), + sa.Column("ThinSect", sa.SmallInteger(), nullable=True), + sa.Column("Geochron", sa.SmallInteger(), nullable=True), + sa.Column("Geochem", sa.SmallInteger(), nullable=True), + sa.Column("Geothermal", sa.SmallInteger(), nullable=True), + sa.Column("WholeRock", sa.SmallInteger(), nullable=True), + sa.Column("Paleontlgy", sa.SmallInteger(), nullable=True), + sa.Column("EnteredBy", sa.String(), nullable=True), + sa.Column("EntryDate", sa.DateTime(), nullable=True), + sa.Column("Notes", sa.String(), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("SamplSetID"), + ) + op.create_index("ix_NMW_WellSamples_RecrdsetID", "NMW_WellSamples", ["RecrdsetID"]) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_NMW_WellSamples_RecrdsetID", table_name="NMW_WellSamples") + op.drop_table("NMW_WellSamples") + op.drop_index("ix_NMW_WellZDatum_RecrdsetID", table_name="NMW_WellZDatum") + op.drop_table("NMW_WellZDatum") + op.drop_index("ix_NMW_WellRecords_WellDataID", table_name="NMW_WellRecords") + op.drop_table("NMW_WellRecords") + op.drop_table("NMW_WellHeaders") + op.drop_index("ix_NMW_WellLocations_WellDataID", table_name="NMW_WellLocations") + op.drop_table("NMW_WellLocations") diff --git a/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py b/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py new file mode 100644 index 000000000..d88cd2d3b --- /dev/null +++ b/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py @@ -0,0 +1,341 @@ +"""NM_Wells geothermal + drill-stem-test 1:1 staging mirror tables + +Revision ID: v8w9x0y1z2a3 +Revises: u7v8w9x0y1z2 +Create Date: 2026-06-06 00:00:01.000000 + +1:1 staging mirror of the NM_Wells "Migrate First" Geothermal and Drill Stem +Test tables (see db/nmw_legacy.py and docs/nm_wells-migration.md). Columns and +lengths taken directly from the NM_Wells SQL dump DDL. + + Geothermal: + tbl_gt_bht_headers -> NMW_GtBhtHeaders + tbl_gt_bht_data -> NMW_GtBhtData + tbl_ws_intervals -> NMW_WsIntervals + tbl_gt_conductivity -> NMW_GtConductivity + tbl_gt_heat_flow -> NMW_GtHeatFlow + tbl_gt_sum_heat_flow -> NMW_GtSumHeatFlow + tbl_gt_temp_depths -> NMW_GtTempDepths + Drill Stem Tests: + tbl_ws_dst_headers -> NMW_WsDstHeaders + tbl_ws_dst_intervals -> NMW_WsDstIntervals + tbl_ws_dst_flow_history-> NMW_WsDstFlowHistory + tbl_ws_dst_fluid_properties -> NMW_WsDstFluidProperties + tbl_ws_dst_pressure -> NMW_WsDstPressure +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "v8w9x0y1z2a3" +down_revision: Union[str, Sequence[str], None] = "u7v8w9x0y1z2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "NMW_GtBhtHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("BoreDia", sa.Float(), nullable=True), + sa.Column("BoreUnits", sa.String(length=16), nullable=True), + sa.Column("DrillFluid", sa.String(length=16), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("FldSalinity", sa.Float(), nullable=True), + sa.Column("FldRstvity", sa.Float(), nullable=True), + sa.Column("Fluid_pH", sa.Float(), nullable=True), + sa.Column("FldDensity", sa.Float(), nullable=True), + sa.Column("FldLevel", sa.Float(), nullable=True), + sa.Column("FldViscsty", sa.Float(), nullable=True), + sa.Column("FluidLoss", sa.String(length=50), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("BHTGUID"), + ) + op.create_index( + "ix_NMW_GtBhtHeaders_SamplSetID", "NMW_GtBhtHeaders", ["SamplSetID"] + ) + + op.create_table( + "NMW_GtBhtData", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Depth", sa.Float(), nullable=True), + sa.Column("BHT", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=5), nullable=True), + sa.Column("HrsSnceCir", sa.Float(), nullable=True), + sa.Column("DateMeasrd", sa.DateTime(), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_GtBhtData_BHTGUID", "NMW_GtBhtData", ["BHTGUID"]) + + op.create_table( + "NMW_WsIntervals", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SampleID", sa.String(length=128), nullable=True), + sa.Column("From_Depth", sa.Float(), nullable=True), + sa.Column("To_Depth", sa.Float(), nullable=True), + sa.Column("From_TVD", sa.Float(), nullable=True), + sa.Column("To_TVD", sa.Float(), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("Intv_Notes", sa.String(length=255), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("IntrvlGUID"), + ) + op.create_index("ix_NMW_WsIntervals_SamplSetID", "NMW_WsIntervals", ["SamplSetID"]) + + op.create_table( + "NMW_GtConductivity", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Cnductvity", sa.Float(), nullable=True), + sa.Column("CnductUnit", sa.String(length=3), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtConductivity_IntrvlGUID", "NMW_GtConductivity", ["IntrvlGUID"] + ) + + op.create_table( + "NMW_GtHeatFlow", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Gradient", sa.Float(), nullable=True), + sa.Column("Ka", sa.Float(), nullable=True), + sa.Column("Ka_unit", sa.String(length=3), nullable=True), + sa.Column("Pm", sa.Float(), nullable=True), + sa.Column("Kpr", sa.Float(), nullable=True), + sa.Column("Kpr_unit", sa.String(length=3), nullable=True), + sa.Column("Q", sa.Float(), nullable=True), + sa.Column("Q_unit", sa.String(length=3), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_GtHeatFlow_IntrvlGUID", "NMW_GtHeatFlow", ["IntrvlGUID"]) + + op.create_table( + "NMW_GtSumHeatFlow", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("LithClass", sa.String(length=50), nullable=True), + sa.Column("UnitBasis", sa.String(length=16), nullable=True), + sa.Column("UnitName", sa.String(length=128), nullable=True), + sa.Column("GeoID", sa.String(length=16), nullable=True), + sa.Column("FromDepth", sa.Float(), nullable=True), + sa.Column("ToDepth", sa.Float(), nullable=True), + sa.Column("DepthUnit", sa.String(length=8), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("ThermlGrad", sa.Float(), nullable=True), + sa.Column("TGError", sa.Float(), nullable=True), + sa.Column("GradUnit", sa.String(length=3), nullable=True), + sa.Column("TGradRange", sa.String(length=15), nullable=True), + sa.Column("SampleType", sa.String(length=50), nullable=True), + sa.Column("NumSamples", sa.SmallInteger(), nullable=True), + sa.Column("ThermlCond", sa.Float(), nullable=True), + sa.Column("TCondError", sa.Float(), nullable=True), + sa.Column("TCondUnit", sa.String(length=3), nullable=True), + sa.Column("TCondRange", sa.String(length=15), nullable=True), + sa.Column("HeatFlow", sa.Float(), nullable=True), + sa.Column("HtFlowErr", sa.Float(), nullable=True), + sa.Column("HtFlowUnit", sa.String(length=3), nullable=True), + sa.Column("HtFlowEst", sa.Float(), nullable=True), + sa.Column("Quality", sa.String(length=50), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtSumHeatFlow_RecrdSetID", "NMW_GtSumHeatFlow", ["RecrdSetID"] + ) + op.create_index( + "ix_NMW_GtSumHeatFlow_SamplSetID", "NMW_GtSumHeatFlow", ["SamplSetID"] + ) + + op.create_table( + "NMW_GtTempDepths", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Depth", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("IntrvlGrad", sa.Float(), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtTempDepths_SamplSetID", "NMW_GtTempDepths", ["SamplSetID"] + ) + + op.create_table( + "NMW_WsDstHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("TestType", sa.String(length=50), nullable=True), + sa.Column("DSTOprator", sa.String(length=50), nullable=True), + sa.Column("PressUnits", sa.String(length=8), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("PipeDiaUnt", sa.String(length=8), nullable=True), + sa.Column("PipeLenUnt", sa.String(length=8), nullable=True), + sa.Column("ChokeSizUn", sa.String(length=8), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("DSTGUID"), + ) + op.create_index( + "ix_NMW_WsDstHeaders_SamplSetID", "NMW_WsDstHeaders", ["SamplSetID"] + ) + + op.create_table( + "NMW_WsDstIntervals", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("DSTName", sa.String(length=128), nullable=True), + sa.Column("TargetFm", sa.String(length=16), nullable=True), + sa.Column("DSTDate", sa.DateTime(), nullable=True), + sa.Column("DSTNumber", sa.SmallInteger(), nullable=True), + sa.Column("Status", sa.String(length=255), nullable=True), + sa.Column("StatusDate", sa.DateTime(), nullable=True), + sa.Column("PackrFrom", sa.Float(), nullable=True), + sa.Column("PackerTo", sa.Float(), nullable=True), + sa.Column("SrfChokeSz", sa.Float(), nullable=True), + sa.Column("BotChokeSz", sa.Float(), nullable=True), + sa.Column("PipeDia", sa.Float(), nullable=True), + sa.Column("PipeLength", sa.Float(), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("DSTInterval"), + ) + op.create_index("ix_NMW_WsDstIntervals_DSTGUID", "NMW_WsDstIntervals", ["DSTGUID"]) + + op.create_table( + "NMW_WsDstFlowHistory", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Operation", sa.String(length=255), nullable=True), + sa.Column("StartTime", sa.DateTime(), nullable=True), + sa.Column("EndTime", sa.DateTime(), nullable=True), + sa.Column("Duration", sa.Float(), nullable=True), + sa.Column("Pressure", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("RecovColmn", sa.Float(), nullable=True), + sa.Column("RecovType", sa.String(length=255), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstFlowHistory_DSTInterval", "NMW_WsDstFlowHistory", ["DSTInterval"] + ) + + op.create_table( + "NMW_WsDstFluidProperties", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SourceLoc", sa.String(length=255), nullable=True), + sa.Column("Resistivty", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("Chlorides", sa.Float(), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstFluidProperties_DSTInterval", + "NMW_WsDstFluidProperties", + ["DSTInterval"], + ) + + op.create_table( + "NMW_WsDstPressure", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("PrsGageDpt", sa.Float(), nullable=True), + sa.Column("BlankedOff", sa.SmallInteger(), nullable=True), + sa.Column("InShtInMin", sa.Float(), nullable=True), + sa.Column("FlwPrsInMin", sa.Float(), nullable=True), + sa.Column("PrsInShtIn", sa.Float(), nullable=True), + sa.Column("PrsInitClsdIn", sa.Float(), nullable=True), + sa.Column("FnShtInMin", sa.Float(), nullable=True), + sa.Column("FlwPrsFinMin", sa.Float(), nullable=True), + sa.Column("PrsFnShtIn", sa.Float(), nullable=True), + sa.Column("ShtInPrMth", sa.String(length=255), nullable=True), + sa.Column("HydrostPrsIn", sa.Float(), nullable=True), + sa.Column("HydStPrsFl", sa.Float(), nullable=True), + sa.Column("HydstPrMth", sa.String(length=255), nullable=True), + sa.Column("EquilPress", sa.Float(), nullable=True), + sa.Column("EqlPrsMth", sa.String(length=255), nullable=True), + sa.Column("FlowPrsMin", sa.Float(), nullable=True), + sa.Column("FlowPrsMax", sa.Float(), nullable=True), + sa.Column("FlowPrsMth", sa.String(length=255), nullable=True), + sa.Column("DSTFluid", sa.String(length=128), nullable=True), + sa.Column("FmTemp", sa.Float(), nullable=True), + sa.Column("TempCorrtn", sa.Float(), nullable=True), + sa.Column("TempFlowng", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=5), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstPressure_DSTInterval", "NMW_WsDstPressure", ["DSTInterval"] + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_NMW_WsDstPressure_DSTInterval", table_name="NMW_WsDstPressure") + op.drop_table("NMW_WsDstPressure") + op.drop_index( + "ix_NMW_WsDstFluidProperties_DSTInterval", table_name="NMW_WsDstFluidProperties" + ) + op.drop_table("NMW_WsDstFluidProperties") + op.drop_index( + "ix_NMW_WsDstFlowHistory_DSTInterval", table_name="NMW_WsDstFlowHistory" + ) + op.drop_table("NMW_WsDstFlowHistory") + op.drop_index("ix_NMW_WsDstIntervals_DSTGUID", table_name="NMW_WsDstIntervals") + op.drop_table("NMW_WsDstIntervals") + op.drop_index("ix_NMW_WsDstHeaders_SamplSetID", table_name="NMW_WsDstHeaders") + op.drop_table("NMW_WsDstHeaders") + op.drop_index("ix_NMW_GtTempDepths_SamplSetID", table_name="NMW_GtTempDepths") + op.drop_table("NMW_GtTempDepths") + op.drop_index("ix_NMW_GtSumHeatFlow_RecrdSetID", table_name="NMW_GtSumHeatFlow") + op.drop_index("ix_NMW_GtSumHeatFlow_SamplSetID", table_name="NMW_GtSumHeatFlow") + op.drop_table("NMW_GtSumHeatFlow") + op.drop_index("ix_NMW_GtHeatFlow_IntrvlGUID", table_name="NMW_GtHeatFlow") + op.drop_table("NMW_GtHeatFlow") + op.drop_index("ix_NMW_GtConductivity_IntrvlGUID", table_name="NMW_GtConductivity") + op.drop_table("NMW_GtConductivity") + op.drop_index("ix_NMW_WsIntervals_SamplSetID", table_name="NMW_WsIntervals") + op.drop_table("NMW_WsIntervals") + op.drop_index("ix_NMW_GtBhtData_BHTGUID", table_name="NMW_GtBhtData") + op.drop_table("NMW_GtBhtData") + op.drop_index("ix_NMW_GtBhtHeaders_SamplSetID", table_name="NMW_GtBhtHeaders") + op.drop_table("NMW_GtBhtHeaders") diff --git a/db/__init__.py b/db/__init__.py index a376381b1..4e2e7fb3a 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -59,6 +59,7 @@ from db.thing_geologic_formation_association import * from db.aquifer_type import * from db.nma_legacy import * +from db.nmw_legacy import * from db.transducer import * from sqlalchemy import ( diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py new file mode 100644 index 000000000..5186fa66d --- /dev/null +++ b/db/nmw_legacy.py @@ -0,0 +1,647 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""1:1 staging mirror of the legacy NM_Wells SQL Server database. + +PURPOSE +------- +These models are a FAITHFUL, column-for-column copy of the NM_Wells source +tables. They are a *staging layer*: data lands here unchanged from the SQL +dump, then a later transform phase maps it into the Ocotillo data model +(Location / Thing / FieldEvent / FieldActivity / Sample / Observation, plus +status_history, measuring_point_history, contact, publication, etc.). + +This file mirrors the convention of ``db/nma_legacy.py`` (the NM_Aquifer +mirror): ``NMW_`` table prefix, original source column names preserved via the +first positional arg to ``mapped_column``, snake_case Python attributes. + +SOURCE +------ +NM_Wells is delivered as a SQL dump. Physical source table names are +``tbl_well_*`` (snake_case). To feed the existing CSV->Pandas->ORM transfer +pipeline, export each source table to CSV (same flow as ``nma_csv_cache``). + +SCOPE (this commit) +------------------- +Mirrors the five "Migrate First / Main" tables that have an authoritative +field-level mapping in the planning workbook +("NM_Wells + Subsurface library.xlsx", sheet 3): + + tbl_well_locations -> NMW_WellLocations + tbl_well_headers -> NMW_WellHeaders + tbl_well_records -> NMW_WellRecords + tbl_well_z_datum -> NMW_WellZDatum + tbl_well_samples -> NMW_WellSamples + +Also mirrors the Geothermal and Drill Stem Test "Migrate First" tables +(columns + lengths taken directly from the NM_Wells SQL dump DDL, so these are +more precise than the five Main tables above whose lengths the sheet omitted): + + Geothermal: + tbl_gt_bht_headers -> NMW_GtBhtHeaders tbl_gt_bht_data -> NMW_GtBhtData + tbl_gt_conductivity -> NMW_GtConductivity tbl_gt_heat_flow -> NMW_GtHeatFlow + tbl_gt_sum_heat_flow-> NMW_GtSumHeatFlow tbl_gt_temp_depths -> NMW_GtTempDepths + tbl_ws_intervals -> NMW_WsIntervals + Drill Stem Tests: + tbl_ws_dst_headers -> NMW_WsDstHeaders tbl_ws_dst_intervals -> NMW_WsDstIntervals + tbl_ws_dst_flow_history -> NMW_WsDstFlowHistory + tbl_ws_dst_fluid_properties -> NMW_WsDstFluidProperties + tbl_ws_dst_pressure -> NMW_WsDstPressure + +Geothermal/DST relationship chains (kept as plain indexed GUID columns, NOT +enforced FKs, since this is staging): + well_samples.SamplSetID <- gt_bht_headers / gt_temp_depths / gt_sum_heat_flow + / ws_intervals / ws_dst_headers (SamplSetID) + gt_bht_headers.BHTGUID <- gt_bht_data.BHTGUID + ws_intervals.IntrvlGUID <- gt_conductivity / gt_heat_flow (IntrvlGUID) + ws_dst_headers.DSTGUID <- ws_dst_intervals.DSTGUID + ws_dst_intervals.DSTInterval <- ws_dst_flow_history / ws_dst_fluid_properties + / ws_dst_pressure (DSTInterval) + well_records.RecrdSetID <- gt_sum_heat_flow.RecrdSetID + +The transform of geothermal/DST into the Ocotillo model is not yet designed +(no field-level mapping in the workbook); see docs/nm_wells-migration.md. + +TRANSFORM NOTES +--------------- +Each column carries an inline note describing its eventual Ocotillo target +(from the mapping sheet). "Drop" = not carried into the Ocotillo model (kept +here only for staging fidelity / audit). See docs/nm_wells-migration.md for +the full plan and the cross-table relationship re-routing +(legacy RecrdSetID -> field_event). + +TYPE MAPPING (SQL Server -> SQLAlchemy) +--------------------------------------- + uniqueidentifier -> postgresql UUID(as_uuid=True) + int -> Integer + smallint -> SmallInteger + real / float -> Float + nvarchar -> String (source lengths not in the sheet; widened) + datetime2 -> DateTime + timestamp -> LargeBinary (SQL Server rowversion; staging only) + +TODO(verify): primary keys below are inferred from the mapping sheet / +relationship notes, not from source DDL. Confirm against the dump. +""" + +from sqlalchemy import ( + DateTime, + Float, + Integer, + LargeBinary, + SmallInteger, + String, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import mapped_column + +from db.base import Base + + +class NMW_WellLocations(Base): + """1:1 mirror of NM_Wells ``tbl_well_locations`` (Main / Migrate First). + + Transform target: ``location`` (point from Lat/Long_dd83, state, county) + plus a new ``NMW_Location`` table for the legacy PLSS/UTM attributes. + """ + + __tablename__ = "NMW_WellLocations" + + # TODO(verify PK): tbl has no clear GUID PK; OBJECTID is the identity col. + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # Drop + well_data_id = mapped_column( + "WellDataID", UUID(as_uuid=True), index=True + ) # -> NMW_Location.well_id (relates header/location/records) + well_id_legacy = mapped_column("Well_ID", String) # Drop + import_id = mapped_column("Import_ID", Integer) # Drop + township = mapped_column("Township", Float) # -> NMW_Location.township + nors_tdir = mapped_column("NorS_TDir", String) # -> NMW_Location.township_n_s + range_ = mapped_column("Range", Float) # -> NMW_Location.range + eorw_rdir = mapped_column("EorW_RDir", String) # -> NMW_Location.range_e_w + sectn = mapped_column("Sectn", SmallInteger) # -> NMW_Location.section + sectn_part = mapped_column("SectnPart", String) # -> NMW_Location.section_portion + unit_letter = mapped_column("UnitLetter", String) # -> NMW_Location.unit_letter + utm_zone = mapped_column("UTM_zone", String) # -> NMW_Location.utm_zone + state = mapped_column("State", String) # -> location.state + county = mapped_column("County", String) # -> location.county + basin = mapped_column("Basin", String) # -> NMW_Location.basin + footage_ns = mapped_column("Footage_NS", Float) # -> NMW_Location.footage_n_s + nors_fdir = mapped_column("NorS_FDir", String) # -> NMW_Location.direction_n_s + footage_ew = mapped_column("Footage_EW", Float) # -> NMW_Location.footage_e_w + eorw_fdir = mapped_column("EorW_FDir", String) # -> NMW_Location.direction_e_w + lat_min = mapped_column("Lat_min", SmallInteger) # Drop (mostly empty) + lat_sec = mapped_column("Lat_sec", Float) # Drop (mostly empty) + long_deg = mapped_column("Long_deg", SmallInteger) # Drop (mostly empty) + long_min = mapped_column("Long_min", SmallInteger) # Drop (mostly empty) + long_sec = mapped_column("Long_sec", Float) # Drop (mostly empty) + lat_dd27 = mapped_column("Lat_dd27", Float) # -> NMW_Location.latitutde_dd27 + long_dd27 = mapped_column("Long_dd27", Float) # -> NMW_Location.longitude_dd27 + lat_dd83 = mapped_column("Lat_dd83", Float) # -> location.point + long_dd83 = mapped_column("Long_dd83", Float) # -> location.point + source_id = mapped_column("SourceID", String) # -> publication.id + source_datum = mapped_column("SourceDatum", String) # -> NMW_Location.source_datum + source_units = mapped_column("SourceUnits", String) # -> NMW_Location.source_units + loc_acc_type = mapped_column("LocAccType", String) # Drop + loc_acc_meas = mapped_column("LocAccMeas", String) # Drop + loc_acc_val = mapped_column("LocAccVal", Float) # Drop + duplicated = mapped_column("Duplicated", SmallInteger) # Drop + exclude = mapped_column("Exclude", SmallInteger) # Drop + comments = mapped_column("Comments", String) # (unmapped) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + api = mapped_column("API", String) # Drop + + +class NMW_WellHeaders(Base): + """1:1 mirror of NM_Wells ``tbl_well_headers`` (Main / Migrate First). + + Transform target: ``thing`` (name/type/well_depth/completion_date), + ``status_history``, ``contact`` (operator + owner), ``publication``, + ``thing_geologic_formation_association``, ``thing_id_link.alternate_id``, + plus new ``well_detail`` and ``well_purpose`` tables. + """ + + __tablename__ = "NMW_WellHeaders" + + object_id = mapped_column("OBJECTID", Integer) # Drop + # WellDataID is the key relating header <-> location <-> records. + well_data_id = mapped_column( + "WellDataID", UUID(as_uuid=True), primary_key=True + ) # Keep + well_spot_id = mapped_column( + "WellSpotID", UUID(as_uuid=True) + ) # Drop (purpose unclear) + api = mapped_column("API", String) # -> thing_id_link.alternate_id + well_class = mapped_column("WellClass", String) # -> thing.type + well_type = mapped_column("WellType", String) # -> well_purpose.purpose + well_orient = mapped_column("WellOrient", String) # -> well_detail.well_orient + cur_well_nam = mapped_column("CurWellNam", String) # -> thing.name + cur_well_num = mapped_column("CurWellNum", String) # -> well_detail.well_number + cur_status = mapped_column("CurStatus", String) # -> status_history.status + prd_pool_cnt = mapped_column("PrdPoolCnt", SmallInteger) # Drop + cur_operatr = mapped_column("CurOperatr", String) # -> contact.name (type=operator) + cur_owner = mapped_column("CurOwner", String) # -> contact.name (type=owner) + total_depth = mapped_column("TotalDepth", Float) # -> thing.well_depth + well_tvd = mapped_column("Well_TVD", Float) # Drop + fm_td = mapped_column("Fm_TD", String) # -> thing_geologic_formation_association.id + age_td = mapped_column("Age_TD", String) # Drop + spud_date = mapped_column("SpudDate", DateTime) # Drop + compl_date = mapped_column("ComplDate", DateTime) # -> thing.well_completion_date + plug_date = mapped_column("PlugDate", DateTime) # Drop + plug_back = mapped_column("PlugBack", Float) # Drop + bridge_plug = mapped_column("BridgePlug", String) # Drop + scout_tickt = mapped_column("ScoutTickt", SmallInteger) # Drop + dwn_hole_sur = mapped_column("DwnHoleSur", SmallInteger) # Drop + geol_log = mapped_column("GeolLog", SmallInteger) # Drop + geophys_log = mapped_column("Geophyslog", SmallInteger) # Drop + gthrm_exist = mapped_column("GthrmExist", SmallInteger) # Drop + petro_data = mapped_column("PetroData", SmallInteger) # Drop + core_exists = mapped_column("CoreExists", SmallInteger) # Drop + cuttings = mapped_column("Cuttings", SmallInteger) # Drop + sample_data = mapped_column("SampleData", SmallInteger) # Drop + comments = mapped_column("Comments", String) # -> well_detail.comments + import_id = mapped_column("Import_ID", String) # Drop + import_db = mapped_column("Import_DB", String) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WellRecords(Base): + """1:1 mirror of NM_Wells ``tbl_well_records`` (Main / Migrate First). + + Transform target: ``field_event`` (+ ``field_activity``). The legacy + wells -> records relationship (RecrdSetID) is re-routed to + wells -> field_event during transform. RecrdClass tags which records are + geothermal. + """ + + __tablename__ = "NMW_WellRecords" + + object_id = mapped_column("OBJECTID", Integer) # Drop + recrd_set_id = mapped_column( + "RecrdSetID", UUID(as_uuid=True), primary_key=True + ) # -> field_event.id + well_data_id = mapped_column( + "WellDataID", UUID(as_uuid=True), index=True + ) # FK -> header/location WellDataID + recrd_class = mapped_column("RecrdClass", String) # -> field_activity.activity_type + source_id = mapped_column( + "SourceID", String + ) # -> publication.id (text in source, not a real FK) + action_date = mapped_column("ActionDate", DateTime) # -> field_event.event_date + well_name = mapped_column("WellName", String) # Drop + well_number = mapped_column("WellNumber", String) # Drop + api_suffix = mapped_column("API_suffix", String) # Drop + entered_by = mapped_column("EnteredBy", String) # Drop + entry_date = mapped_column("EntryDate", DateTime) # Drop + comments = mapped_column("Comments", String) # -> field_event.notes + + +class NMW_WellZDatum(Base): + """1:1 mirror of NM_Wells ``tbl_well_z_datum`` (Main / Migrate First). + + Transform target: ``measuring_point_history`` (elevation -> height, + datum -> description, units/source -> new fields). + """ + + __tablename__ = "NMW_WellZDatum" + + object_id = mapped_column("OBJECTID", Integer) # Drop + recrdset_id = mapped_column( + "RecrdsetID", UUID(as_uuid=True), index=True + ) # FK -> records + elev_gl = mapped_column( + "Elev_GL", Float + ) # -> measuring_point_history.measuring_point_height + elev_df = mapped_column( + "Elev_DF", Float + ) # -> measuring_point_history.measuring_point_height + elev_kb = mapped_column( + "Elev_KB", Float + ) # -> measuring_point_history.measuring_point_height + elev_unspc = mapped_column( + "Elev_unspc", Float + ) # -> measuring_point_history.measuring_point_height + datum_elev = mapped_column("DatumElev", Float) # Drop (redundant) + depth_datum = mapped_column( + "DepthDatum", String + ) # -> measuring_point_history.measuring_point_description + depth_units = mapped_column( + "DepthUnits", String + ) # -> measuring_point_history.measuring_point_units [new field] + z_datum = mapped_column("Z_datum", String) # Drop (only 7 records) + z_units = mapped_column("Z_units", String) # Drop + elev_source = mapped_column( + "ElevSource", String + ) # -> measuring_point_history.source [new field] + elv_acc_type = mapped_column("ElvAccType", String) # Drop + elv_acc_meas = mapped_column("ElvAccMeas", String) # Drop + elv_acc_val = mapped_column("ElvAccVal", Float) # Drop + comments = mapped_column("Comments", String) # Drop + # TODO(verify PK): GlobalID assumed PK. + global_id = mapped_column("GlobalID", UUID(as_uuid=True), primary_key=True) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WellSamples(Base): + """1:1 mirror of NM_Wells ``tbl_well_samples`` (Main / Migrate First). + + Transform target: ``sample`` (date/notes/created_by) + ``observation`` + (depth units). The many boolean attribute flags (Porosity, Geothermal, + etc.) are dropped (mostly empty in source). + """ + + __tablename__ = "NMW_WellSamples" + + object_id = mapped_column("OBJECTID", Integer) # Drop + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), primary_key=True + ) # -> sample.id + recrdset_id = mapped_column( + "RecrdsetID", UUID(as_uuid=True), index=True + ) # -> field_activity.id + smp_set_name = mapped_column("SmpSetName", String) # Drop + sampl_class = mapped_column("SamplClass", String) # Drop (mostly 'data') + sample_type = mapped_column("SampleType", String) # Drop (mostly empty) + sample_fm = mapped_column("SampleFm", String) # Drop (mostly empty) + sample_loc = mapped_column("SampleLoc", String) # Drop (no entries) + sample_date = mapped_column("SampleDate", DateTime) # -> sample.sample_date + from_depth = mapped_column("From_Depth", Float) # -> observation (depth) + to_depth = mapped_column("To_Depth", Float) # -> observation (depth) + smp_dp_unt = mapped_column("SmpDpUnt", String) # -> observation.unit + from_tvd = mapped_column("From_TVD", Float) # Drop + to_tvd = mapped_column("To_TVD", Float) # Drop + from_elev = mapped_column("From_Elev", Float) # Drop (empty) + to_elev = mapped_column("To_Elev", Float) # Drop (empty) + porosity = mapped_column("Porosity", SmallInteger) # Drop + permeablty = mapped_column("Permeablty", SmallInteger) # Drop + density = mapped_column("Density", SmallInteger) # Drop + dst_tests = mapped_column("DST_Tests", SmallInteger) # Drop + thin_sect = mapped_column("ThinSect", SmallInteger) # Drop + geochron = mapped_column("Geochron", SmallInteger) # Drop + geochem = mapped_column("Geochem", SmallInteger) # Drop + geothermal = mapped_column("Geothermal", SmallInteger) # Drop + whole_rock = mapped_column("WholeRock", SmallInteger) # Drop + paleontlgy = mapped_column("Paleontlgy", SmallInteger) # Drop + entered_by = mapped_column("EnteredBy", String) # -> sample.created_by_name + entry_date = mapped_column("EntryDate", DateTime) # -> sample.created_at + notes = mapped_column("Notes", String) # -> sample.notes + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +# ============================================================================= +# GEOTHERMAL (Area=Geothermal, "Migrate First") +# ============================================================================= + + +class NMW_GtBhtHeaders(Base): + """1:1 mirror of NM_Wells ``tbl_gt_bht_headers`` (bottom-hole-temp header).""" + + __tablename__ = "NMW_GtBhtHeaders" + + object_id = mapped_column("OBJECTID", Integer) # Drop (identity) + bht_guid = mapped_column("BHTGUID", UUID(as_uuid=True), primary_key=True) + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), index=True + ) # FK -> well_samples.SamplSetID + bore_dia = mapped_column("BoreDia", Float) + bore_units = mapped_column("BoreUnits", String(16)) + drill_fluid = mapped_column("DrillFluid", String(16)) + temp_unit = mapped_column("TempUnit", String(1)) + fld_salinity = mapped_column("FldSalinity", Float) + fld_rstvity = mapped_column("FldRstvity", Float) + fluid_ph = mapped_column("Fluid_pH", Float) + fld_density = mapped_column("FldDensity", Float) + fld_level = mapped_column("FldLevel", Float) + fld_viscsty = mapped_column("FldViscsty", Float) + fluid_loss = mapped_column("FluidLoss", String(50)) + notes = mapped_column("Notes", String(255)) + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_GtBhtData(Base): + """1:1 mirror of NM_Wells ``tbl_gt_bht_data`` (BHT readings).""" + + __tablename__ = "NMW_GtBhtData" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + bht_guid = mapped_column( + "BHTGUID", UUID(as_uuid=True), index=True + ) # FK -> gt_bht_headers.BHTGUID + depth = mapped_column("Depth", Float) + bht = mapped_column("BHT", Float) + temp_unit = mapped_column("TempUnit", String(5)) + hrs_snce_cir = mapped_column("HrsSnceCir", Float) + date_measrd = mapped_column("DateMeasrd", DateTime) + comments = mapped_column("Comments", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WsIntervals(Base): + """1:1 mirror of NM_Wells ``tbl_ws_intervals`` (sample depth intervals).""" + + __tablename__ = "NMW_WsIntervals" + + object_id = mapped_column("OBJECTID", Integer) # Drop (identity) + intrvl_guid = mapped_column("IntrvlGUID", UUID(as_uuid=True), primary_key=True) + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), index=True + ) # FK -> well_samples.SamplSetID + sample_id = mapped_column("SampleID", String(128)) + from_depth = mapped_column("From_Depth", Float) + to_depth = mapped_column("To_Depth", Float) + from_tvd = mapped_column("From_TVD", Float) + to_tvd = mapped_column("To_TVD", Float) + from_elev = mapped_column("From_Elev", Float) + to_elev = mapped_column("To_Elev", Float) + intv_notes = mapped_column("Intv_Notes", String(255)) + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_GtConductivity(Base): + """1:1 mirror of NM_Wells ``tbl_gt_conductivity`` (thermal conductivity).""" + + __tablename__ = "NMW_GtConductivity" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + intrvl_guid = mapped_column( + "IntrvlGUID", UUID(as_uuid=True), index=True + ) # FK -> ws_intervals.IntrvlGUID + cnductvity = mapped_column("Cnductvity", Float) + cnduct_unit = mapped_column("CnductUnit", String(3)) + comments = mapped_column("Comments", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_GtHeatFlow(Base): + """1:1 mirror of NM_Wells ``tbl_gt_heat_flow`` (per-interval heat flow).""" + + __tablename__ = "NMW_GtHeatFlow" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + intrvl_guid = mapped_column( + "IntrvlGUID", UUID(as_uuid=True), index=True + ) # FK -> ws_intervals.IntrvlGUID + gradient = mapped_column("Gradient", Float) + ka = mapped_column("Ka", Float) + ka_unit = mapped_column("Ka_unit", String(3)) + pm = mapped_column("Pm", Float) + kpr = mapped_column("Kpr", Float) + kpr_unit = mapped_column("Kpr_unit", String(3)) + q = mapped_column("Q", Float) + q_unit = mapped_column("Q_unit", String(3)) + comments = mapped_column("Comments", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_GtSumHeatFlow(Base): + """1:1 mirror of NM_Wells ``tbl_gt_sum_heat_flow`` (summary heat flow).""" + + __tablename__ = "NMW_GtSumHeatFlow" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + recrd_set_id = mapped_column( + "RecrdSetID", UUID(as_uuid=True), index=True + ) # FK -> well_records.RecrdSetID + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), index=True + ) # FK -> well_samples.SamplSetID + lith_class = mapped_column("LithClass", String(50)) + unit_basis = mapped_column("UnitBasis", String(16)) + unit_name = mapped_column("UnitName", String(128)) + geo_id = mapped_column("GeoID", String(16)) + from_depth = mapped_column("FromDepth", Float) + to_depth = mapped_column("ToDepth", Float) + depth_unit = mapped_column("DepthUnit", String(8)) + from_elev = mapped_column("From_Elev", Float) + to_elev = mapped_column("To_Elev", Float) + therml_grad = mapped_column("ThermlGrad", Float) + tg_error = mapped_column("TGError", Float) + grad_unit = mapped_column("GradUnit", String(3)) + tgrad_range = mapped_column("TGradRange", String(15)) + sample_type = mapped_column("SampleType", String(50)) + num_samples = mapped_column("NumSamples", SmallInteger) + therml_cond = mapped_column("ThermlCond", Float) + tcond_error = mapped_column("TCondError", Float) + tcond_unit = mapped_column("TCondUnit", String(3)) + tcond_range = mapped_column("TCondRange", String(15)) + heat_flow = mapped_column("HeatFlow", Float) + ht_flow_err = mapped_column("HtFlowErr", Float) + ht_flow_unit = mapped_column("HtFlowUnit", String(3)) + ht_flow_est = mapped_column("HtFlowEst", Float) + quality = mapped_column("Quality", String(50)) + comments = mapped_column("Comments", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_GtTempDepths(Base): + """1:1 mirror of NM_Wells ``tbl_gt_temp_depths`` (temp-vs-depth profile).""" + + __tablename__ = "NMW_GtTempDepths" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), index=True + ) # FK -> well_samples.SamplSetID + depth = mapped_column("Depth", Float) + temp = mapped_column("Temp", Float) + temp_unit = mapped_column("TempUnit", String(1)) + intrvl_grad = mapped_column("IntrvlGrad", Float) + comments = mapped_column("Comments", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +# ============================================================================= +# DRILL STEM TESTS (Area=Drill Stem Tests, "Migrate First") +# ============================================================================= + + +class NMW_WsDstHeaders(Base): + """1:1 mirror of NM_Wells ``tbl_ws_dst_headers`` (DST header).""" + + __tablename__ = "NMW_WsDstHeaders" + + object_id = mapped_column("OBJECTID", Integer) # Drop (identity) + dst_guid = mapped_column("DSTGUID", UUID(as_uuid=True), primary_key=True) + sampl_set_id = mapped_column( + "SamplSetID", UUID(as_uuid=True), index=True + ) # FK -> well_samples.SamplSetID + test_type = mapped_column("TestType", String(50)) + dst_oprator = mapped_column("DSTOprator", String(50)) + press_units = mapped_column("PressUnits", String(8)) + temp_unit = mapped_column("TempUnit", String(1)) + pipe_dia_unt = mapped_column("PipeDiaUnt", String(8)) + pipe_len_unt = mapped_column("PipeLenUnt", String(8)) + choke_siz_un = mapped_column("ChokeSizUn", String(8)) + notes = mapped_column("Notes", String(255)) + + +class NMW_WsDstIntervals(Base): + """1:1 mirror of NM_Wells ``tbl_ws_dst_intervals`` (DST interval).""" + + __tablename__ = "NMW_WsDstIntervals" + + object_id = mapped_column("OBJECTID", Integer) # Drop (identity) + dst_interval = mapped_column("DSTInterval", UUID(as_uuid=True), primary_key=True) + dst_guid = mapped_column( + "DSTGUID", UUID(as_uuid=True), index=True + ) # FK -> ws_dst_headers.DSTGUID + dst_name = mapped_column("DSTName", String(128)) + target_fm = mapped_column("TargetFm", String(16)) + dst_date = mapped_column("DSTDate", DateTime) + dst_number = mapped_column("DSTNumber", SmallInteger) + status = mapped_column("Status", String(255)) + status_date = mapped_column("StatusDate", DateTime) + packr_from = mapped_column("PackrFrom", Float) + packer_to = mapped_column("PackerTo", Float) + srf_choke_sz = mapped_column("SrfChokeSz", Float) + bot_choke_sz = mapped_column("BotChokeSz", Float) + pipe_dia = mapped_column("PipeDia", Float) + pipe_length = mapped_column("PipeLength", Float) + notes = mapped_column("Notes", String(255)) + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WsDstFlowHistory(Base): + """1:1 mirror of NM_Wells ``tbl_ws_dst_flow_history`` (DST flow events).""" + + __tablename__ = "NMW_WsDstFlowHistory" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + dst_interval = mapped_column( + "DSTInterval", UUID(as_uuid=True), index=True + ) # FK -> ws_dst_intervals.DSTInterval + operation = mapped_column("Operation", String(255)) + start_time = mapped_column("StartTime", DateTime) + end_time = mapped_column("EndTime", DateTime) + duration = mapped_column("Duration", Float) + pressure = mapped_column("Pressure", Float) + temp = mapped_column("Temp", Float) + recov_colmn = mapped_column("RecovColmn", Float) + recov_type = mapped_column("RecovType", String(255)) + notes = mapped_column("Notes", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WsDstFluidProperties(Base): + """1:1 mirror of NM_Wells ``tbl_ws_dst_fluid_properties`` (recovered fluid).""" + + __tablename__ = "NMW_WsDstFluidProperties" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + dst_interval = mapped_column( + "DSTInterval", UUID(as_uuid=True), index=True + ) # FK -> ws_dst_intervals.DSTInterval + source_loc = mapped_column("SourceLoc", String(255)) + resistivty = mapped_column("Resistivty", Float) + temp = mapped_column("Temp", Float) + chlorides = mapped_column("Chlorides", Float) + notes = mapped_column("Notes", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +class NMW_WsDstPressure(Base): + """1:1 mirror of NM_Wells ``tbl_ws_dst_pressure`` (DST pressure readings).""" + + __tablename__ = "NMW_WsDstPressure" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + dst_interval = mapped_column( + "DSTInterval", UUID(as_uuid=True), index=True + ) # FK -> ws_dst_intervals.DSTInterval + prs_gage_dpt = mapped_column("PrsGageDpt", Float) + blanked_off = mapped_column("BlankedOff", SmallInteger) + in_sht_in_min = mapped_column("InShtInMin", Float) + flw_prs_in_min = mapped_column("FlwPrsInMin", Float) + prs_in_sht_in = mapped_column("PrsInShtIn", Float) + prs_init_clsd_in = mapped_column("PrsInitClsdIn", Float) + fn_sht_in_min = mapped_column("FnShtInMin", Float) + flw_prs_fin_min = mapped_column("FlwPrsFinMin", Float) + prs_fn_sht_in = mapped_column("PrsFnShtIn", Float) + sht_in_pr_mth = mapped_column("ShtInPrMth", String(255)) + hydrost_prs_in = mapped_column("HydrostPrsIn", Float) + hyd_st_prs_fl = mapped_column("HydStPrsFl", Float) + hydst_pr_mth = mapped_column("HydstPrMth", String(255)) + equil_press = mapped_column("EquilPress", Float) + eql_prs_mth = mapped_column("EqlPrsMth", String(255)) + flow_prs_min = mapped_column("FlowPrsMin", Float) + flow_prs_max = mapped_column("FlowPrsMax", Float) + flow_prs_mth = mapped_column("FlowPrsMth", String(255)) + dst_fluid = mapped_column("DSTFluid", String(128)) + fm_temp = mapped_column("FmTemp", Float) + temp_corrtn = mapped_column("TempCorrtn", Float) + temp_flowng = mapped_column("TempFlowng", Float) + temp_unit = mapped_column("TempUnit", String(5)) + notes = mapped_column("Notes", String(255)) + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop + ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) + + +# ============================================================================= +# TODO(remaining "Migrate First" tables, no DDL/mapping yet) +# ----------------------------------------------------------------------------- +# Publications: tbl_sources +# Subsurface Library: dst_scan, log_scanned, Well_Header, well_operators +# See docs/nm_wells-migration.md for the full inventory + recommendations. +# ============================================================================= + + +# ============= EOF ============================================= diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py new file mode 100644 index 000000000..6718cfb7d --- /dev/null +++ b/transfers/nmw_mirror_transfer.py @@ -0,0 +1,250 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Load the NM_Wells SQL dump into the ``NMW_*`` 1:1 staging mirror tables. + +Phase 1 of the NM_Wells migration (see db/nmw_legacy.py and +docs/nm_wells-migration.md). This is a faithful copy: each source table's CSV +export is read and its rows are inserted into the matching ``NMW_*`` mirror +model with NO transformation beyond type coercion. The Phase 2 transform into +the Ocotillo model is separate. + +Generic + data-driven: one ``MirrorSpec`` per (model, source CSV). Column +handling is derived from each model's ``__table__`` metadata, so adding a new +mirror table requires only a model + a spec entry (no per-table code). + +Source CSVs are read with ``transfers.util.read_csv`` (looks in +``transfers/data/nma_csv_cache/.csv`` then GCS ``nma_csv/
.csv``). +CSV headers are expected to be the original SQL Server column names (OBJECTID, +WellDataID, GlobalID, ...), which match the mirror columns' DB names exactly. + +Idempotent: rows upsert via ``INSERT ... ON CONFLICT () DO NOTHING``. +""" + +import uuid +from dataclasses import dataclass + +import pandas as pd +from sqlalchemy import DateTime, Float, Integer, LargeBinary, SmallInteger, String +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from db.nmw_legacy import ( + NMW_GtBhtData, + NMW_GtBhtHeaders, + NMW_GtConductivity, + NMW_GtHeatFlow, + NMW_GtSumHeatFlow, + NMW_GtTempDepths, + NMW_WellHeaders, + NMW_WellLocations, + NMW_WellRecords, + NMW_WellSamples, + NMW_WellZDatum, + NMW_WsDstFlowHistory, + NMW_WsDstFluidProperties, + NMW_WsDstHeaders, + NMW_WsDstIntervals, + NMW_WsDstPressure, + NMW_WsIntervals, +) +from transfers.logger import logger +from transfers.util import read_csv + +_CHUNK_SIZE = 2000 + + +@dataclass +class MirrorSpec: + """Maps a mirror model to its NM_Wells source CSV/table name.""" + + model: type + source_table: str + + +# All NMW_* mirror tables. Order is irrelevant (no enforced cross-table FKs in +# the staging layer), but parents are listed before children for readability. +NMW_MIRROR_SPECS: list[MirrorSpec] = [ + # Main + MirrorSpec(NMW_WellLocations, "tbl_well_locations"), + MirrorSpec(NMW_WellHeaders, "tbl_well_headers"), + MirrorSpec(NMW_WellRecords, "tbl_well_records"), + MirrorSpec(NMW_WellZDatum, "tbl_well_z_datum"), + MirrorSpec(NMW_WellSamples, "tbl_well_samples"), + # Geothermal + MirrorSpec(NMW_GtBhtHeaders, "tbl_gt_bht_headers"), + MirrorSpec(NMW_GtBhtData, "tbl_gt_bht_data"), + MirrorSpec(NMW_WsIntervals, "tbl_ws_intervals"), + MirrorSpec(NMW_GtConductivity, "tbl_gt_conductivity"), + MirrorSpec(NMW_GtHeatFlow, "tbl_gt_heat_flow"), + MirrorSpec(NMW_GtSumHeatFlow, "tbl_gt_sum_heat_flow"), + MirrorSpec(NMW_GtTempDepths, "tbl_gt_temp_depths"), + # Drill Stem Tests + MirrorSpec(NMW_WsDstHeaders, "tbl_ws_dst_headers"), + MirrorSpec(NMW_WsDstIntervals, "tbl_ws_dst_intervals"), + MirrorSpec(NMW_WsDstFlowHistory, "tbl_ws_dst_flow_history"), + MirrorSpec(NMW_WsDstFluidProperties, "tbl_ws_dst_fluid_properties"), + MirrorSpec(NMW_WsDstPressure, "tbl_ws_dst_pressure"), +] + + +def _coerce(value, col_type): + """Coerce a single cell to the Python value for ``col_type`` (or None). + + Treats NaN/NaT as None. (pandas keeps NaN/NaT in typed columns even after a + ``.where(notnull, None)``, so the missing-value check must happen here.) + """ + if value is None: + return None + try: + if pd.isna(value): + return None + except (TypeError, ValueError): + pass # non-scalar / unhashable: fall through and coerce normally + if isinstance(col_type, UUID): + if isinstance(value, uuid.UUID): + return value + try: + return uuid.UUID(str(value).strip()) + except (ValueError, AttributeError, TypeError): + return None + if isinstance(col_type, (Integer, SmallInteger)): + try: + return int(value) + except (ValueError, TypeError): + return None + if isinstance(col_type, Float): + try: + return float(value) + except (ValueError, TypeError): + return None + if isinstance(col_type, DateTime): + # pandas Timestamp -> python datetime; anything else passed through. + return value.to_pydatetime() if hasattr(value, "to_pydatetime") else value + if isinstance(col_type, String): + s = str(value) + return s[: col_type.length] if col_type.length else s + # Fallback (should not hit for our mirror types). + return value + + +def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: + """Load one source CSV into its mirror table. Returns a stats dict.""" + table = spec.model.__table__ + name = spec.source_table + + try: + df = read_csv(name) + except Exception as e: # noqa: BLE001 - missing CSV / GCS miss must not abort + logger.warning("Skipping %s (could not read CSV): %s", name, e) + return {"table": name, "skipped": True, "reason": str(e)} + + if limit and limit > 0: + df = df.head(limit) + if df.empty: + logger.warning("Skipping %s (empty)", name) + return {"table": name, "skipped": True, "reason": "empty"} + + # Columns to load = mirror columns present in the CSV, excluding rowversion + # (LargeBinary) which is a SQL Server artifact with no meaningful CSV value. + cols = {c.name: c for c in table.columns if not isinstance(c.type, LargeBinary)} + present = [n for n in df.columns if n in cols] + missing_csv = [n for n in cols if n not in df.columns] + extra_csv = [n for n in df.columns if n not in cols] + if not present: + logger.warning( + "Skipping %s: no overlapping columns (csv has %s)", name, list(df.columns) + ) + return {"table": name, "skipped": True, "reason": "no matching columns"} + if missing_csv: + logger.warning("%s: mirror columns absent from CSV: %s", name, missing_csv) + if extra_csv: + logger.info("%s: ignoring %d unmapped CSV column(s)", name, len(extra_csv)) + + pk_cols = [c.name for c in table.primary_key] + # NaN/NaT are normalized to None inside _coerce (pandas keeps them in typed + # columns), so the raw dict records are fine here. + records = df[present].to_dict("records") + total = len(records) + inserted = 0 + + for start in range(0, total, _CHUNK_SIZE): + chunk = records[start : start + _CHUNK_SIZE] + rows = [] + for rec in chunk: + row = {n: _coerce(rec.get(n), cols[n].type) for n in present} + # Drop rows missing a PK value (cannot upsert). + if any(row.get(pk) is None for pk in pk_cols): + continue + rows.append(row) + if not rows: + continue + stmt = ( + pg_insert(spec.model) + .values(rows) + .on_conflict_do_nothing(index_elements=pk_cols) + ) + result = session.execute(stmt) + session.commit() + inserted += result.rowcount if result.rowcount and result.rowcount > 0 else 0 + + logger.info( + "Mirror %s -> %s: %d source rows, %d inserted", + name, + table.name, + total, + inserted, + ) + return { + "table": name, + "skipped": False, + "rows": total, + "inserted": inserted, + } + + +def transfer_nmw_mirror(session: Session, limit: int = None) -> tuple: + """Load all NM_Wells source CSVs into the ``NMW_*`` staging mirror. + + Same ``(session, limit)`` signature as the other session-based transfers. + Returns ``(num_tables_loaded, total_rows_inserted, errors)``. + """ + limit = int(limit or 0) + results = [] + errors = [] + for spec in NMW_MIRROR_SPECS: + try: + results.append(_load_table(session, spec, limit)) + except Exception as e: # noqa: BLE001 - isolate per-table failures + logger.critical("NMW mirror load failed for %s: %s", spec.source_table, e) + session.rollback() + errors.append({"table": spec.source_table, "error": str(e)}) + + loaded = [r for r in results if not r.get("skipped")] + skipped = [r for r in results if r.get("skipped")] + inserted = sum(r.get("inserted", 0) for r in loaded) + logger.info( + "NMW mirror load complete: %d tables loaded, %d skipped, %d rows inserted, " + "%d errors", + len(loaded), + len(skipped), + inserted, + len(errors), + ) + return len(loaded), inserted, errors + + +# ============= EOF ============================================= diff --git a/transfers/reference_lexicon_transfer.py b/transfers/reference_lexicon_transfer.py new file mode 100644 index 000000000..13c64e62b --- /dev/null +++ b/transfers/reference_lexicon_transfer.py @@ -0,0 +1,362 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Load the legacy NM_Wells ``ref_*`` reference tables into the lexicon. + +The planning workbook ("NM_Wells + Subsurface library.xlsx", sheet 1) flags the +``ref_*`` tables as "Add to lexicon". Each ref table is a small code/description +lookup; this transfer loads its rows as ``LexiconTerm`` rows and links them to a +``LexiconCategory`` named after the table (``ref_well_class`` -> ``well_class``). + +Idempotent: mirrors ``core.initializers.init_lexicon`` — categories and terms +upsert via ``ON CONFLICT DO NOTHING`` (both ``name``/``term`` are unique); +term<->category associations are inserted only when missing (no unique +constraint exists on that table). + +Source CSVs are read with ``transfers.util.read_csv`` (looks in +``transfers/data/nma_csv_cache/
.csv`` then GCS ``nma_csv/
.csv``). + +NOTE(columns): the ref tables' actual column names are not in the workbook, so +term/definition columns are AUTO-DETECTED per table (see ``_pick_columns``). If +auto-detection is wrong for a table, set ``term_col`` / ``definition_col`` +explicitly on its ``RefTableSpec`` below. The chosen columns are logged. + +NOTE(LU_*): the Subsurface Library ``LU_*`` lookups (LU_EnteredBy, LU_LogType, +LU_Status, LU_Type_Wellheader, LU_WorkType) are also "Add to lexicon"; add them +to ``REFERENCE_TABLE_SPECS`` once their CSVs are available. +""" + +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from db import ( + LexiconCategory, + LexiconTerm, + LexiconTermCategoryAssociation, +) +from transfers.logger import logger +from transfers.util import read_csv + +# lexicon_term.term (and its FK targets) is String(100). +_TERM_MAX_LEN = 100 + +# Column-name hints for auto-detecting the code/term vs definition columns. +_META_COLS = {"objectid", "ssma_timestamp", "globalid", "id", "import_id"} +_TERM_HINTS = ("code", "abbr", "symbol", "letter", "key", "short") +_DEF_HINTS = ( + "description", + "desc", + "definition", + "meaning", + "label", + "name", + "long", + "title", + "value", + "text", +) + + +@dataclass +class RefTableSpec: + """One legacy ref table -> one lexicon category. + + term_col / definition_col are optional overrides; when None the columns are + auto-detected from the CSV header. + """ + + source_table: str + category: str + term_col: Optional[str] = None + definition_col: Optional[str] = None + description: Optional[str] = None + + +def _spec(table: str) -> RefTableSpec: + """Build a spec with category = table name minus the ``ref_`` prefix.""" + category = table[4:] if table.startswith("ref_") else table + return RefTableSpec( + source_table=table, + category=category, + description=f"Imported from NM_Wells {table}", + ) + + +# All ref_* tables marked "Add to lexicon" in the workbook (sheet 1). +# ref_nm_quads (Review) and ref_date_drilled-style oddities are intentionally +# excluded; add/remove specs here as the mapping is refined. +REFERENCE_TABLE_SPECS: list[RefTableSpec] = [ + _spec(t) + for t in ( + "ref_altitude_datums", + "ref_altitude_methods", + "ref_basins", + "ref_coordinate_accuracy", + "ref_coordinate_datum", + "ref_coordinate_method", + "ref_county", + "ref_data_reliability", + "ref_date_drilled", + "ref_depth_types", + "ref_display_scales", + "ref_ground_levels", + "ref_gt_data_sources", + "ref_gt_well_types", + "ref_ign_comps", + "ref_indurations", + "ref_initials", + "ref_length_units", + "ref_lith_class", + "ref_lith_types", + "ref_ll_sources", + "ref_mm_facies", + "ref_perforation_types", + "ref_porosity_methods", + "ref_pres_units", + "ref_prod_meth_quality", + "ref_prod_methods", + "ref_prod_units", + "ref_sample_class", + "ref_sample_types", + "ref_states", + "ref_textures", + "ref_unit_basis", + "ref_unit_conductivity", + "ref_unit_depths", + "ref_unit_gradients", + "ref_unit_heat_flow", + "ref_unit_letters", + "ref_unit_temps", + "ref_well_action_class", + "ref_well_class", + "ref_well_commodity", + "ref_well_log_class", + "ref_well_orientations", + "ref_well_record_class", + "ref_well_status", + "ref_well_types", + "ref_work_types", + "ref_xy_units", + ) +] + + +def _pick_columns(df: pd.DataFrame, spec: RefTableSpec) -> tuple[str, str]: + """Resolve (term_col, definition_col) for a ref table. + + Honors explicit overrides on the spec, else auto-detects from the header + using name hints, ignoring meta columns (OBJECTID, GlobalID, ...). + """ + cols = [c for c in df.columns if str(c).strip().lower() not in _META_COLS] + if not cols: + cols = list(df.columns) + low = {c: str(c).strip().lower() for c in cols} + + term_col = spec.term_col + if term_col is None: + term_col = next( + (c for c in cols if any(h in low[c] for h in _TERM_HINTS)), cols[0] + ) + + def_col = spec.definition_col + if def_col is None: + def_col = next( + (c for c in cols if c != term_col and any(h in low[c] for h in _DEF_HINTS)), + None, + ) + if def_col is None: + def_col = cols[1] if len(cols) > 1 else term_col + + return term_col, def_col + + +def _clean(value) -> Optional[str]: + if value is None or pd.isna(value): + return None + s = str(value).strip() + return s or None + + +def _get_or_create_category(session: Session, spec: RefTableSpec) -> int: + """Return the lexicon_category.id for the spec, creating it if needed.""" + cat_id = session.execute( + select(LexiconCategory.id).where(LexiconCategory.name == spec.category) + ).scalar_one_or_none() + if cat_id is not None: + return cat_id + + session.execute( + pg_insert(LexiconCategory) + .values(name=spec.category, description=spec.description) + .on_conflict_do_nothing(index_elements=["name"]) + ) + session.commit() + return session.execute( + select(LexiconCategory.id).where(LexiconCategory.name == spec.category) + ).scalar_one() + + +def _transfer_one(session: Session, spec: RefTableSpec, limit: int = 0) -> dict: + """Load a single ref table into the lexicon. Returns a stats dict.""" + try: + df = read_csv(spec.source_table) + except Exception as e: # noqa: BLE001 - missing CSV / GCS miss should not abort + logger.warning("Skipping %s (could not read CSV): %s", spec.source_table, e) + return {"table": spec.source_table, "skipped": True, "reason": str(e)} + + if limit and limit > 0: + df = df.head(limit) + + if df.empty or not list(df.columns): + logger.warning("Skipping %s (empty)", spec.source_table) + return {"table": spec.source_table, "skipped": True, "reason": "empty"} + + term_col, def_col = _pick_columns(df, spec) + logger.info( + "%s -> category=%s term_col=%s definition_col=%s (%d rows)", + spec.source_table, + spec.category, + term_col, + def_col, + len(df), + ) + + category_id = _get_or_create_category(session, spec) + + # Build unique (term -> definition) map, dropping empties and overlong terms. + term_defs: dict[str, str] = {} + truncated = 0 + for row in df.itertuples(index=False): + term = _clean(getattr(row, term_col, None)) + if term is None: + continue + if len(term) > _TERM_MAX_LEN: + term = term[:_TERM_MAX_LEN] + truncated += 1 + definition = _clean(getattr(row, def_col, None)) or term + term_defs.setdefault(term, definition) + + if not term_defs: + logger.warning("Skipping %s (no usable terms)", spec.source_table) + return {"table": spec.source_table, "skipped": True, "reason": "no terms"} + if truncated: + logger.warning( + "%s: truncated %d term(s) to %d chars", + spec.source_table, + truncated, + _TERM_MAX_LEN, + ) + + term_names = list(term_defs) + existing_terms = dict( + session.execute( + select(LexiconTerm.term, LexiconTerm.id).where( + LexiconTerm.term.in_(term_names) + ) + ).all() + ) + new_rows = [ + {"term": t, "definition": d} + for t, d in term_defs.items() + if t not in existing_terms + ] + if new_rows: + session.execute( + pg_insert(LexiconTerm) + .values(new_rows) + .on_conflict_do_nothing(index_elements=["term"]) + ) + session.commit() + existing_terms = dict( + session.execute( + select(LexiconTerm.term, LexiconTerm.id).where( + LexiconTerm.term.in_(term_names) + ) + ).all() + ) + + term_ids = [tid for tid in existing_terms.values() if tid is not None] + existing_links = set() + if term_ids: + existing_links = set( + session.execute( + select(LexiconTermCategoryAssociation.term_id).where( + LexiconTermCategoryAssociation.category_id == category_id, + LexiconTermCategoryAssociation.term_id.in_(term_ids), + ) + ).scalars() + ) + + assoc_rows = [ + {"term_id": tid, "category_id": category_id} + for tid in term_ids + if tid not in existing_links + ] + if assoc_rows: + session.execute(pg_insert(LexiconTermCategoryAssociation).values(assoc_rows)) + session.commit() + + return { + "table": spec.source_table, + "skipped": False, + "rows": len(df), + "terms": len(term_defs), + "created_terms": len(new_rows), + "linked": len(assoc_rows), + } + + +def transfer_reference_tables(session: Session, limit: int = None) -> tuple: + """Foundational transfer: load all ``ref_*`` tables into the lexicon. + + Same ``(session, limit)`` signature as the other foundational transfers + (aquifer systems, geologic formations). Returns + ``(num_tables, total_created_terms, errors)``. + """ + limit = int(limit or 0) + results = [] + errors = [] + for spec in REFERENCE_TABLE_SPECS: + try: + results.append(_transfer_one(session, spec, limit)) + except Exception as e: # noqa: BLE001 - isolate per-table failures + logger.critical( + "Reference lexicon transfer failed for %s: %s", spec.source_table, e + ) + session.rollback() + errors.append({"table": spec.source_table, "error": str(e)}) + + loaded = [r for r in results if not r.get("skipped")] + skipped = [r for r in results if r.get("skipped")] + created = sum(r.get("created_terms", 0) for r in loaded) + linked = sum(r.get("linked", 0) for r in loaded) + logger.info( + "Reference lexicon transfer complete: %d tables loaded, %d skipped, " + "%d terms created, %d associations, %d errors", + len(loaded), + len(skipped), + created, + linked, + len(errors), + ) + return len(loaded), created, errors + + +# ============= EOF ============================================= diff --git a/transfers/transfer.py b/transfers/transfer.py index 419d4870a..c42910ada 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -55,6 +55,8 @@ from services.env import get_bool_env from transfers.aquifer_system_transfer import transfer_aquifer_systems from transfers.geologic_formation_transfer import transfer_geologic_formations +from transfers.reference_lexicon_transfer import transfer_reference_tables +from transfers.nmw_mirror_transfer import transfer_nmw_mirror from transfers.permissions_transfer import transfer_permissions from transfers.stratigraphy_legacy import StratigraphyLegacyTransferer from transfers.stratigraphy_transfer import transfer_stratigraphy @@ -360,11 +362,12 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: else: message("PHASE 1: FOUNDATIONAL TRANSFERS (PARALLEL)") foundational_tasks = [ + ("ReferenceLexicon", transfer_reference_tables), ("AquiferSystems", transfer_aquifer_systems), ("GeologicFormations", transfer_geologic_formations), ] - with ThreadPoolExecutor(max_workers=2) as executor: + with ThreadPoolExecutor(max_workers=len(foundational_tasks)) as executor: futures = { executor.submit( _execute_foundational_transfer_with_timing, name, func, limit @@ -383,6 +386,13 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: logger.critical(f"Foundational transfer {name} failed: {e}") raise # Fail fast - foundational transfers must succeed + # NM_Wells 1:1 staging mirror (separate source DB). Off by default so it + # does not run during the standard NM_Aquifer -> Ocotillo transfer. + if get_bool_env("TRANSFER_NMW_MIRROR", False): + message("NM_WELLS 1:1 STAGING MIRROR LOAD") + with session_ctx() as session: + transfer_nmw_mirror(session, limit=limit) + message("TRANSFERRING WELLS") use_parallel_wells = get_bool_env("TRANSFER_PARALLEL_WELLS", True) if use_parallel_wells: From 9fdb7686fcc1fd545628827266312b24d16b6b4a Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 16:27:31 -0600 Subject: [PATCH 02/41] fix(transfers): address review feedback on NM_Wells mirror - nmw_mirror_transfer: parse DateTime values with pd.to_datetime(errors=coerce) since read_csv does not parse_dates (avoids driver-dependent insert failures). - db/nmw_legacy: fix attribute typos (dst_operator, recov_column, resistivity) while preserving the legacy DB column names; fix latitude_dd27 comment typo. - reference_lexicon_transfer: correct stale exclusion comment (ref_date_drilled is included; only ref_nm_quads is excluded). Co-Authored-By: Claude Opus 4.8 --- db/nmw_legacy.py | 8 ++++---- transfers/nmw_mirror_transfer.py | 8 ++++++-- transfers/reference_lexicon_transfer.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index 5186fa66d..1256beddd 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -146,7 +146,7 @@ class NMW_WellLocations(Base): long_deg = mapped_column("Long_deg", SmallInteger) # Drop (mostly empty) long_min = mapped_column("Long_min", SmallInteger) # Drop (mostly empty) long_sec = mapped_column("Long_sec", Float) # Drop (mostly empty) - lat_dd27 = mapped_column("Lat_dd27", Float) # -> NMW_Location.latitutde_dd27 + lat_dd27 = mapped_column("Lat_dd27", Float) # -> NMW_Location.latitude_dd27 long_dd27 = mapped_column("Long_dd27", Float) # -> NMW_Location.longitude_dd27 lat_dd83 = mapped_column("Lat_dd83", Float) # -> location.point long_dd83 = mapped_column("Long_dd83", Float) # -> location.point @@ -523,7 +523,7 @@ class NMW_WsDstHeaders(Base): "SamplSetID", UUID(as_uuid=True), index=True ) # FK -> well_samples.SamplSetID test_type = mapped_column("TestType", String(50)) - dst_oprator = mapped_column("DSTOprator", String(50)) + dst_operator = mapped_column("DSTOprator", String(50)) press_units = mapped_column("PressUnits", String(8)) temp_unit = mapped_column("TempUnit", String(1)) pipe_dia_unt = mapped_column("PipeDiaUnt", String(8)) @@ -573,7 +573,7 @@ class NMW_WsDstFlowHistory(Base): duration = mapped_column("Duration", Float) pressure = mapped_column("Pressure", Float) temp = mapped_column("Temp", Float) - recov_colmn = mapped_column("RecovColmn", Float) + recov_column = mapped_column("RecovColmn", Float) recov_type = mapped_column("RecovType", String(255)) notes = mapped_column("Notes", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop @@ -590,7 +590,7 @@ class NMW_WsDstFluidProperties(Base): "DSTInterval", UUID(as_uuid=True), index=True ) # FK -> ws_dst_intervals.DSTInterval source_loc = mapped_column("SourceLoc", String(255)) - resistivty = mapped_column("Resistivty", Float) + resistivity = mapped_column("Resistivty", Float) temp = mapped_column("Temp", Float) chlorides = mapped_column("Chlorides", Float) notes = mapped_column("Notes", String(255)) diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 6718cfb7d..279a25f6f 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -132,8 +132,12 @@ def _coerce(value, col_type): except (ValueError, TypeError): return None if isinstance(col_type, DateTime): - # pandas Timestamp -> python datetime; anything else passed through. - return value.to_pydatetime() if hasattr(value, "to_pydatetime") else value + # read_csv does not parse_dates, so values are typically raw strings. + # Parse explicitly to avoid driver-dependent insert failures. + if hasattr(value, "to_pydatetime"): + return value.to_pydatetime() + ts = pd.to_datetime(value, errors="coerce") + return None if pd.isna(ts) else ts.to_pydatetime() if isinstance(col_type, String): s = str(value) return s[: col_type.length] if col_type.length else s diff --git a/transfers/reference_lexicon_transfer.py b/transfers/reference_lexicon_transfer.py index 13c64e62b..6a20461ae 100644 --- a/transfers/reference_lexicon_transfer.py +++ b/transfers/reference_lexicon_transfer.py @@ -100,8 +100,8 @@ def _spec(table: str) -> RefTableSpec: # All ref_* tables marked "Add to lexicon" in the workbook (sheet 1). -# ref_nm_quads (Review) and ref_date_drilled-style oddities are intentionally -# excluded; add/remove specs here as the mapping is refined. +# ref_nm_quads (Review, ~2k rows) is intentionally excluded; add/remove specs +# here as the mapping is refined. REFERENCE_TABLE_SPECS: list[RefTableSpec] = [ _spec(t) for t in ( From 565c49bbdbd263844fd4e428456c3a41f8d2a7ca Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 22:24:17 -0600 Subject: [PATCH 03/41] refactor(db): drop SSMA_TimeStamp from NM_Wells mirror The SSMA_TimeStamp column is a SQL Server rowversion artifact with no value as staging data (the loader already skipped it). Remove it from the NMW_* mirror models and both migrations; drop the now-unused LargeBinary import. Co-Authored-By: Claude Opus 4.8 --- ...9x0y1z2_nmw_legacy_staging_mirror_tables.py | 4 ---- ...0y1z2a3_nmw_geothermal_dst_mirror_tables.py | 11 ----------- db/nmw_legacy.py | 18 +----------------- 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py index 7fb64962b..b4a78ee34 100644 --- a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py +++ b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py @@ -71,7 +71,6 @@ def upgrade() -> None: sa.Column("Exclude", sa.SmallInteger(), nullable=True), sa.Column("Comments", sa.String(), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.Column("API", sa.String(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) @@ -115,7 +114,6 @@ def upgrade() -> None: sa.Column("Comments", sa.String(), nullable=True), sa.Column("Import_ID", sa.String(), nullable=True), sa.Column("Import_DB", sa.String(), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("WellDataID"), ) @@ -156,7 +154,6 @@ def upgrade() -> None: sa.Column("ElvAccVal", sa.Float(), nullable=True), sa.Column("Comments", sa.String(), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("GlobalID"), ) op.create_index("ix_NMW_WellZDatum_RecrdsetID", "NMW_WellZDatum", ["RecrdsetID"]) @@ -192,7 +189,6 @@ def upgrade() -> None: sa.Column("EnteredBy", sa.String(), nullable=True), sa.Column("EntryDate", sa.DateTime(), nullable=True), sa.Column("Notes", sa.String(), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("SamplSetID"), ) op.create_index("ix_NMW_WellSamples_RecrdsetID", "NMW_WellSamples", ["RecrdsetID"]) diff --git a/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py b/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py index d88cd2d3b..2a0bd1a97 100644 --- a/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py +++ b/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py @@ -56,7 +56,6 @@ def upgrade() -> None: sa.Column("FldViscsty", sa.Float(), nullable=True), sa.Column("FluidLoss", sa.String(length=50), nullable=True), sa.Column("Notes", sa.String(length=255), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("BHTGUID"), ) op.create_index( @@ -74,7 +73,6 @@ def upgrade() -> None: sa.Column("DateMeasrd", sa.DateTime(), nullable=True), sa.Column("Comments", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index("ix_NMW_GtBhtData_BHTGUID", "NMW_GtBhtData", ["BHTGUID"]) @@ -92,7 +90,6 @@ def upgrade() -> None: sa.Column("From_Elev", sa.Float(), nullable=True), sa.Column("To_Elev", sa.Float(), nullable=True), sa.Column("Intv_Notes", sa.String(length=255), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("IntrvlGUID"), ) op.create_index("ix_NMW_WsIntervals_SamplSetID", "NMW_WsIntervals", ["SamplSetID"]) @@ -105,7 +102,6 @@ def upgrade() -> None: sa.Column("CnductUnit", sa.String(length=3), nullable=True), sa.Column("Comments", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( @@ -126,7 +122,6 @@ def upgrade() -> None: sa.Column("Q_unit", sa.String(length=3), nullable=True), sa.Column("Comments", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index("ix_NMW_GtHeatFlow_IntrvlGUID", "NMW_GtHeatFlow", ["IntrvlGUID"]) @@ -162,7 +157,6 @@ def upgrade() -> None: sa.Column("Quality", sa.String(length=50), nullable=True), sa.Column("Comments", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( @@ -182,7 +176,6 @@ def upgrade() -> None: sa.Column("IntrvlGrad", sa.Float(), nullable=True), sa.Column("Comments", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( @@ -226,7 +219,6 @@ def upgrade() -> None: sa.Column("PipeDia", sa.Float(), nullable=True), sa.Column("PipeLength", sa.Float(), nullable=True), sa.Column("Notes", sa.String(length=255), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("DSTInterval"), ) op.create_index("ix_NMW_WsDstIntervals_DSTGUID", "NMW_WsDstIntervals", ["DSTGUID"]) @@ -245,7 +237,6 @@ def upgrade() -> None: sa.Column("RecovType", sa.String(length=255), nullable=True), sa.Column("Notes", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( @@ -262,7 +253,6 @@ def upgrade() -> None: sa.Column("Chlorides", sa.Float(), nullable=True), sa.Column("Notes", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( @@ -300,7 +290,6 @@ def upgrade() -> None: sa.Column("TempUnit", sa.String(length=5), nullable=True), sa.Column("Notes", sa.String(length=255), nullable=True), sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SSMA_TimeStamp", sa.LargeBinary(), nullable=True), sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index( diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index 1256beddd..ad5d82a71 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -90,7 +90,7 @@ real / float -> Float nvarchar -> String (source lengths not in the sheet; widened) datetime2 -> DateTime - timestamp -> LargeBinary (SQL Server rowversion; staging only) + timestamp -> dropped (SQL Server rowversion; no value as staging data) TODO(verify): primary keys below are inferred from the mapping sheet / relationship notes, not from source DDL. Confirm against the dump. @@ -100,7 +100,6 @@ DateTime, Float, Integer, - LargeBinary, SmallInteger, String, ) @@ -160,7 +159,6 @@ class NMW_WellLocations(Base): exclude = mapped_column("Exclude", SmallInteger) # Drop comments = mapped_column("Comments", String) # (unmapped) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) api = mapped_column("API", String) # Drop @@ -214,7 +212,6 @@ class NMW_WellHeaders(Base): comments = mapped_column("Comments", String) # -> well_detail.comments import_id = mapped_column("Import_ID", String) # Drop import_db = mapped_column("Import_DB", String) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WellRecords(Base): @@ -291,7 +288,6 @@ class NMW_WellZDatum(Base): comments = mapped_column("Comments", String) # Drop # TODO(verify PK): GlobalID assumed PK. global_id = mapped_column("GlobalID", UUID(as_uuid=True), primary_key=True) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WellSamples(Base): @@ -337,7 +333,6 @@ class NMW_WellSamples(Base): entered_by = mapped_column("EnteredBy", String) # -> sample.created_by_name entry_date = mapped_column("EntryDate", DateTime) # -> sample.created_at notes = mapped_column("Notes", String) # -> sample.notes - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) # ============================================================================= @@ -367,7 +362,6 @@ class NMW_GtBhtHeaders(Base): fld_viscsty = mapped_column("FldViscsty", Float) fluid_loss = mapped_column("FluidLoss", String(50)) notes = mapped_column("Notes", String(255)) - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_GtBhtData(Base): @@ -386,7 +380,6 @@ class NMW_GtBhtData(Base): date_measrd = mapped_column("DateMeasrd", DateTime) comments = mapped_column("Comments", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WsIntervals(Base): @@ -407,7 +400,6 @@ class NMW_WsIntervals(Base): from_elev = mapped_column("From_Elev", Float) to_elev = mapped_column("To_Elev", Float) intv_notes = mapped_column("Intv_Notes", String(255)) - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_GtConductivity(Base): @@ -423,7 +415,6 @@ class NMW_GtConductivity(Base): cnduct_unit = mapped_column("CnductUnit", String(3)) comments = mapped_column("Comments", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_GtHeatFlow(Base): @@ -445,7 +436,6 @@ class NMW_GtHeatFlow(Base): q_unit = mapped_column("Q_unit", String(3)) comments = mapped_column("Comments", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_GtSumHeatFlow(Base): @@ -486,7 +476,6 @@ class NMW_GtSumHeatFlow(Base): quality = mapped_column("Quality", String(50)) comments = mapped_column("Comments", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_GtTempDepths(Base): @@ -504,7 +493,6 @@ class NMW_GtTempDepths(Base): intrvl_grad = mapped_column("IntrvlGrad", Float) comments = mapped_column("Comments", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) # ============================================================================= @@ -555,7 +543,6 @@ class NMW_WsDstIntervals(Base): pipe_dia = mapped_column("PipeDia", Float) pipe_length = mapped_column("PipeLength", Float) notes = mapped_column("Notes", String(255)) - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WsDstFlowHistory(Base): @@ -577,7 +564,6 @@ class NMW_WsDstFlowHistory(Base): recov_type = mapped_column("RecovType", String(255)) notes = mapped_column("Notes", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WsDstFluidProperties(Base): @@ -595,7 +581,6 @@ class NMW_WsDstFluidProperties(Base): chlorides = mapped_column("Chlorides", Float) notes = mapped_column("Notes", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) class NMW_WsDstPressure(Base): @@ -632,7 +617,6 @@ class NMW_WsDstPressure(Base): temp_unit = mapped_column("TempUnit", String(5)) notes = mapped_column("Notes", String(255)) global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop - ssma_timestamp = mapped_column("SSMA_TimeStamp", LargeBinary) # Drop (rowversion) # ============================================================================= From 1f9b1fc0f3fbd4fbf8c7cc8a5896e492319b306d Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 22:28:17 -0600 Subject: [PATCH 04/41] fix(db): verify NM_Wells mirror PKs against dump; z_datum -> OBJECTID Confirmed source PKs from the NM_Wells SQL dump DDL: - WellHeaders/WellRecords/WellSamples have declared PRIMARY KEY constraints (WellDataID / RecrdSetID / SamplSetID) matching the models. - WellLocations and WellZDatum declare no PK, only unique indexes on OBJECTID and GlobalID. Switch WellZDatum PK from GlobalID to OBJECTID for consistency with WellLocations and safety (OBJECTID identity is never NULL; the GlobalID unique index permits one NULL). Update the migration accordingly. Remove the TODO(verify) note; PKs are now confirmed. Co-Authored-By: Claude Opus 4.8 --- ...x0y1z2_nmw_legacy_staging_mirror_tables.py | 6 +++--- db/nmw_legacy.py | 20 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py index b4a78ee34..b413554db 100644 --- a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py +++ b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py @@ -137,7 +137,7 @@ def upgrade() -> None: op.create_table( "NMW_WellZDatum", - sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("OBJECTID", sa.Integer(), nullable=False), sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("Elev_GL", sa.Float(), nullable=True), sa.Column("Elev_DF", sa.Float(), nullable=True), @@ -153,8 +153,8 @@ def upgrade() -> None: sa.Column("ElvAccMeas", sa.String(), nullable=True), sa.Column("ElvAccVal", sa.Float(), nullable=True), sa.Column("Comments", sa.String(), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=False), - sa.PrimaryKeyConstraint("GlobalID"), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), ) op.create_index("ix_NMW_WellZDatum_RecrdsetID", "NMW_WellZDatum", ["RecrdsetID"]) diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index ad5d82a71..9d65c7226 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -92,8 +92,14 @@ datetime2 -> DateTime timestamp -> dropped (SQL Server rowversion; no value as staging data) -TODO(verify): primary keys below are inferred from the mapping sheet / -relationship notes, not from source DDL. Confirm against the dump. +PRIMARY KEYS (verified against the NM_Wells SQL dump DDL) +-------------------------------------------------------- +- NMW_WellHeaders -> WellDataID, NMW_WellRecords -> RecrdSetID, + NMW_WellSamples -> SamplSetID: declared PRIMARY KEY constraints in source. +- NMW_WellLocations, NMW_WellZDatum: source declares no PK, only unique indexes + on OBJECTID and GlobalID; OBJECTID (identity, never NULL) is used. +- Geothermal/DST: declared PKs where present (BHTGUID, IntrvlGUID, DSTGUID, + DSTInterval); the rest are heaps keyed on the OBJECTID identity column. """ from sqlalchemy import ( @@ -118,7 +124,8 @@ class NMW_WellLocations(Base): __tablename__ = "NMW_WellLocations" - # TODO(verify PK): tbl has no clear GUID PK; OBJECTID is the identity col. + # No declared PK in source; OBJECTID (identity, unique index, always + # non-null) chosen over the also-unique GlobalID, which permits a NULL. object_id = mapped_column("OBJECTID", Integer, primary_key=True) # Drop well_data_id = mapped_column( "WellDataID", UUID(as_uuid=True), index=True @@ -254,7 +261,9 @@ class NMW_WellZDatum(Base): __tablename__ = "NMW_WellZDatum" - object_id = mapped_column("OBJECTID", Integer) # Drop + # No declared PK in source; OBJECTID (identity, unique index, always + # non-null) chosen over the also-unique GlobalID, which permits a NULL. + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # Drop recrdset_id = mapped_column( "RecrdsetID", UUID(as_uuid=True), index=True ) # FK -> records @@ -286,8 +295,7 @@ class NMW_WellZDatum(Base): elv_acc_meas = mapped_column("ElvAccMeas", String) # Drop elv_acc_val = mapped_column("ElvAccVal", Float) # Drop comments = mapped_column("Comments", String) # Drop - # TODO(verify PK): GlobalID assumed PK. - global_id = mapped_column("GlobalID", UUID(as_uuid=True), primary_key=True) # Drop + global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop class NMW_WellSamples(Base): From cfbf117bbc95441688a2bfedb0024de1138e4cf0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 22:39:24 -0600 Subject: [PATCH 05/41] feat(transfers): load NM_Wells mirror from a SQL Server data dump Add transfers/nmw_sql_dump.py: streams INSERT [dbo].[tbl_*] (...) VALUES (...) statements out of a SQL Server data-dump .sql file, yielding {column: value} dicts. Handles N'...' / escaped '', embedded commas/parens, CAST(expr AS type), multi-row VALUES, 0x binary -> None, and UTF-16/UTF-8 (BOM auto-detect). Refactor transfer_nmw_mirror to be source-agnostic: when NMW_SQL_DUMP points at a .sql data dump it loads from there, otherwise falls back to per-table CSVs. Same model-driven type coercion and chunked ON CONFLICT upsert for both. Note: the provided NMWells.sql is schema-only; NMW_SQL_DUMP expects a separate data dump containing INSERT statements. Co-Authored-By: Claude Opus 4.8 --- .env.example | 3 + transfers/nmw_mirror_transfer.py | 145 ++++++++++++-------- transfers/nmw_sql_dump.py | 223 +++++++++++++++++++++++++++++++ 3 files changed, 314 insertions(+), 57 deletions(-) create mode 100644 transfers/nmw_sql_dump.py diff --git a/.env.example b/.env.example index 2c4534696..8895581b0 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,9 @@ TRANSFER_WEATHER_DATA=True TRANSFER_MINOR_TRACE_CHEMISTRY=True # NM_Wells 1:1 staging mirror load (separate source DB; off by default) TRANSFER_NMW_MIRROR=False +# Optional: path to a NM_Wells SQL Server data-dump .sql file (INSERT statements). +# When set, the mirror loads from it; otherwise it falls back to CSV exports. +# NMW_SQL_DUMP=/path/to/NMWells_data.sql # asset storage GCS_BUCKET_NAME= diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 279a25f6f..2b84e39c2 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -21,18 +21,27 @@ model with NO transformation beyond type coercion. The Phase 2 transform into the Ocotillo model is separate. -Generic + data-driven: one ``MirrorSpec`` per (model, source CSV). Column +Generic + data-driven: one ``MirrorSpec`` per (model, source table). Column handling is derived from each model's ``__table__`` metadata, so adding a new mirror table requires only a model + a spec entry (no per-table code). -Source CSVs are read with ``transfers.util.read_csv`` (looks in -``transfers/data/nma_csv_cache/
.csv`` then GCS ``nma_csv/
.csv``). -CSV headers are expected to be the original SQL Server column names (OBJECTID, -WellDataID, GlobalID, ...), which match the mirror columns' DB names exactly. +Two row sources, selected at runtime: + +1. **SQL Server data dump** (preferred): set ``NMW_SQL_DUMP`` to a ``.sql`` file + containing ``INSERT [dbo].[tbl_*] (...) VALUES (...)`` statements. Rows are + streamed and parsed by ``transfers.nmw_sql_dump.iter_table_rows``. +2. **CSV exports** (fallback when ``NMW_SQL_DUMP`` is unset): per-table CSVs read + with ``transfers.util.read_csv`` (``transfers/data/nma_csv_cache/
.csv`` + then GCS ``nma_csv/
.csv``). + +In both cases the source column names are the original SQL Server names +(OBJECTID, WellDataID, ...), which match the mirror columns' DB names exactly. Idempotent: rows upsert via ``INSERT ... ON CONFLICT () DO NOTHING``. """ +import itertools +import os import uuid from dataclasses import dataclass @@ -62,8 +71,12 @@ NMW_WsIntervals, ) from transfers.logger import logger +from transfers.nmw_sql_dump import iter_table_rows from transfers.util import read_csv +# Path to a SQL Server data-dump .sql file. When set, rows are parsed from it; +# otherwise the loader falls back to per-table CSV exports. +_SQL_DUMP_ENV = "NMW_SQL_DUMP" _CHUNK_SIZE = 2000 @@ -145,70 +158,78 @@ def _coerce(value, col_type): return value +def _row_source(spec: MirrorSpec): + """Return ``(iterator_of_raw_dicts, source_label)`` for a spec. + + SQL dump if ``NMW_SQL_DUMP`` is set, otherwise CSV. Raises on a hard read + error so the caller can record/skip the table. + """ + dump = os.getenv(_SQL_DUMP_ENV) + if dump: + return iter_table_rows(dump, spec.source_table), f"sql:{os.path.basename(dump)}" + df = read_csv(spec.source_table) + return (rec for rec in df.to_dict("records")), "csv" + + +def _flush(session: Session, model, rows: list[dict], pk_cols: list[str]) -> int: + """Upsert a batch; return inserted row count.""" + if not rows: + return 0 + stmt = pg_insert(model).values(rows).on_conflict_do_nothing(index_elements=pk_cols) + result = session.execute(stmt) + session.commit() + return result.rowcount if result.rowcount and result.rowcount > 0 else 0 + + def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: - """Load one source CSV into its mirror table. Returns a stats dict.""" + """Load one source table (SQL dump or CSV) into its mirror. Stats dict.""" table = spec.model.__table__ name = spec.source_table + # Loadable columns from the model (rowversion/LargeBinary excluded defensively). + cols = {c.name: c for c in table.columns if not isinstance(c.type, LargeBinary)} + pk_cols = [c.name for c in table.primary_key] try: - df = read_csv(name) - except Exception as e: # noqa: BLE001 - missing CSV / GCS miss must not abort - logger.warning("Skipping %s (could not read CSV): %s", name, e) + rows_iter, src = _row_source(spec) + except Exception as e: # noqa: BLE001 - missing source must not abort the run + logger.warning("Skipping %s (could not read source): %s", name, e) return {"table": name, "skipped": True, "reason": str(e)} if limit and limit > 0: - df = df.head(limit) - if df.empty: - logger.warning("Skipping %s (empty)", name) - return {"table": name, "skipped": True, "reason": "empty"} + rows_iter = itertools.islice(rows_iter, limit) - # Columns to load = mirror columns present in the CSV, excluding rowversion - # (LargeBinary) which is a SQL Server artifact with no meaningful CSV value. - cols = {c.name: c for c in table.columns if not isinstance(c.type, LargeBinary)} - present = [n for n in df.columns if n in cols] - missing_csv = [n for n in cols if n not in df.columns] - extra_csv = [n for n in df.columns if n not in cols] - if not present: - logger.warning( - "Skipping %s: no overlapping columns (csv has %s)", name, list(df.columns) - ) - return {"table": name, "skipped": True, "reason": "no matching columns"} - if missing_csv: - logger.warning("%s: mirror columns absent from CSV: %s", name, missing_csv) - if extra_csv: - logger.info("%s: ignoring %d unmapped CSV column(s)", name, len(extra_csv)) - - pk_cols = [c.name for c in table.primary_key] - # NaN/NaT are normalized to None inside _coerce (pandas keeps them in typed - # columns), so the raw dict records are fine here. - records = df[present].to_dict("records") - total = len(records) + total = 0 inserted = 0 + batch: list[dict] = [] + warned_cols = False + for rec in rows_iter: + total += 1 + if not warned_cols: + missing = [n for n in cols if n not in rec] + if missing: + logger.warning( + "%s: mirror columns absent from source: %s", name, missing + ) + warned_cols = True + # NaN/NaT (CSV) and NULL (SQL) normalize to None inside _coerce. + row = {n: _coerce(rec.get(n), cols[n].type) for n in cols if n in rec} + if any(row.get(pk) is None for pk in pk_cols): + continue # cannot upsert without a PK value + batch.append(row) + if len(batch) >= _CHUNK_SIZE: + inserted += _flush(session, spec.model, batch, pk_cols) + batch = [] + inserted += _flush(session, spec.model, batch, pk_cols) - for start in range(0, total, _CHUNK_SIZE): - chunk = records[start : start + _CHUNK_SIZE] - rows = [] - for rec in chunk: - row = {n: _coerce(rec.get(n), cols[n].type) for n in present} - # Drop rows missing a PK value (cannot upsert). - if any(row.get(pk) is None for pk in pk_cols): - continue - rows.append(row) - if not rows: - continue - stmt = ( - pg_insert(spec.model) - .values(rows) - .on_conflict_do_nothing(index_elements=pk_cols) - ) - result = session.execute(stmt) - session.commit() - inserted += result.rowcount if result.rowcount and result.rowcount > 0 else 0 + if total == 0: + logger.warning("Skipping %s (no source rows from %s)", name, src) + return {"table": name, "skipped": True, "reason": "no rows", "source": src} logger.info( - "Mirror %s -> %s: %d source rows, %d inserted", + "Mirror %s -> %s [%s]: %d source rows, %d inserted", name, table.name, + src, total, inserted, ) @@ -217,16 +238,26 @@ def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: "skipped": False, "rows": total, "inserted": inserted, + "source": src, } def transfer_nmw_mirror(session: Session, limit: int = None) -> tuple: - """Load all NM_Wells source CSVs into the ``NMW_*`` staging mirror. + """Load all NM_Wells source tables into the ``NMW_*`` staging mirror. - Same ``(session, limit)`` signature as the other session-based transfers. - Returns ``(num_tables_loaded, total_rows_inserted, errors)``. + Source is a SQL dump (``NMW_SQL_DUMP``) when set, else per-table CSVs. Same + ``(session, limit)`` signature as the other session-based transfers. Returns + ``(num_tables_loaded, total_rows_inserted, errors)``. """ limit = int(limit or 0) + dump = os.getenv(_SQL_DUMP_ENV) + if dump: + if not os.path.exists(dump): + raise FileNotFoundError(f"{_SQL_DUMP_ENV} set but file not found: {dump}") + logger.info("NMW mirror source: SQL dump %s", dump) + else: + logger.info("NMW mirror source: CSV exports (set %s for a dump)", _SQL_DUMP_ENV) + results = [] errors = [] for spec in NMW_MIRROR_SPECS: diff --git a/transfers/nmw_sql_dump.py b/transfers/nmw_sql_dump.py new file mode 100644 index 000000000..39637c076 --- /dev/null +++ b/transfers/nmw_sql_dump.py @@ -0,0 +1,223 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Stream rows out of a SQL Server data-dump ``.sql`` file. + +Parses ``INSERT [dbo].[
] () VALUES ()[, () ...]`` +statements (the format produced by SSMS "Generate Scripts -> data" / ``bcp`` +INSERT mode) for one target table at a time, yielding ``{column: value}`` +dicts. Values are decoded to plain Python: + + NULL -> None + N'...' / '...' -> str (doubled '' unescaped) + 123 / -1.5 -> int / float + CAST(expr AS type) -> the inner expr, recursively + 0x.... -> None (binary / rowversion; not mirrored) + +Type coercion to the target column type happens in nmw_mirror_transfer._coerce, +so this module keeps values loosely typed. + +Streaming: the file is read line by line (constant memory), accumulating across +lines only when a statement's parentheses are unbalanced (strings containing +newlines). The file is scanned once per table. + +Encoding is auto-detected from the BOM (SSMS writes UTF-16 LE); falls back to +utf-8. +""" + +import re +from typing import Iterator, Optional + + +def _detect_encoding(path: str) -> str: + with open(path, "rb") as f: + head = f.read(4) + if head[:2] in (b"\xff\xfe", b"\xfe\xff"): + return "utf-16" + if head[:3] == b"\xef\xbb\xbf": + return "utf-8-sig" + return "utf-8" + + +def _split_top_level(s: str) -> list[str]: + """Split a comma list at paren-depth 0, respecting single-quoted strings.""" + parts: list[str] = [] + buf: list[str] = [] + depth = 0 + in_quote = False + i = 0 + n = len(s) + while i < n: + c = s[i] + if in_quote: + buf.append(c) + if c == "'": + if i + 1 < n and s[i + 1] == "'": # escaped '' + buf.append("'") + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + buf.append(c) + elif c == "(": + depth += 1 + buf.append(c) + elif c == ")": + depth -= 1 + buf.append(c) + elif c == "," and depth == 0: + parts.append("".join(buf).strip()) + buf = [] + else: + buf.append(c) + i += 1 + if buf: + parts.append("".join(buf).strip()) + return parts + + +def _iter_value_groups(s: str) -> Iterator[str]: + """Yield the inside of each top-level ``( ... )`` group in a VALUES list.""" + depth = 0 + in_quote = False + start = -1 + i = 0 + n = len(s) + while i < n: + c = s[i] + if in_quote: + if c == "'": + if i + 1 < n and s[i + 1] == "'": + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + elif c == "(": + if depth == 0: + start = i + 1 + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0 and start >= 0: + yield s[start:i] + start = -1 + i += 1 + + +_CAST_RE = re.compile(r"(?is)^CAST\s*\((.*)\s+AS\s+[^)]+\)$") + + +def _parse_value(tok: str): + t = tok.strip() + if not t or t.upper() == "NULL": + return None + m = _CAST_RE.match(t) + if m: + return _parse_value(m.group(1).strip()) + # N'...' or '...' + if t[:1] == "'" or t[:2].upper() == "N'": + q = t.find("'") + inner = t[q + 1 :] + if inner.endswith("'"): + inner = inner[:-1] + return inner.replace("''", "'") + if t[:2].lower() == "0x": # binary / rowversion + return None + if re.fullmatch(r"[-+]?\d+", t): + return int(t) + try: + return float(t) + except ValueError: + return t + + +_INSERT_RE = re.compile( + r"(?is)INSERT\s+(?:\[dbo\]\.)?\[?(?P
\w+)\]?\s*\((?P.*?)\)\s*VALUES\s*(?P.*)$" +) + + +def _balanced(stmt: str) -> bool: + """True if parens are balanced outside single-quoted strings.""" + depth = 0 + in_quote = False + i = 0 + n = len(stmt) + while i < n: + c = stmt[i] + if in_quote: + if c == "'": + if i + 1 < n and stmt[i + 1] == "'": + i += 2 + continue + in_quote = False + elif c == "'": + in_quote = True + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + return depth <= 0 and not in_quote + + +def iter_table_rows(path: str, table: str) -> Iterator[dict]: + """Yield ``{column: value}`` dicts for every INSERT into ``table``.""" + enc = _detect_encoding(path) + target = f"[{table}]".lower() + target_plain = table.lower() + pending: Optional[str] = None + + with open(path, encoding=enc, errors="ignore") as f: + for line in f: + if pending is None: + low = line.lower() + if "insert" not in low: + continue + # cheap table filter before the heavier regex + if ( + target not in low + and f"].[{target_plain}]" not in low + and f" {target_plain} " not in low + ): + if target_plain not in low: + continue + pending = line + else: + pending += line + + if not _balanced(pending): + continue # statement spans more lines + + stmt = pending + pending = None + m = _INSERT_RE.search(stmt) + if not m or m.group("table").lower() != target_plain: + continue + cols = [c.strip().strip("[]") for c in _split_top_level(m.group("cols"))] + vals_part = m.group("vals").strip().rstrip(";") + for group in _iter_value_groups(vals_part): + vals = [_parse_value(v) for v in _split_top_level(group)] + if len(vals) != len(cols): + continue # malformed row; skip + yield dict(zip(cols, vals)) + + +# ============= EOF ============================================= From 8bc427f708173a801f1554b66a9da2d2c21c8de0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 22:42:09 -0600 Subject: [PATCH 06/41] refactor(transfers): standalone transfer_geothermal; deprecate transfer.py Move the NM_Wells (geothermal) orchestration out of transfers/transfer.py into a new standalone transfers/transfer_geothermal.py. Revert all NM_Wells wiring from transfer.py and mark that module deprecated (module docstring + DeprecationWarning in transfer_all) so new migrations get their own orchestrator. transfer_geothermal.py runs the reference->lexicon load (TRANSFER_GEOTHERMAL_REFERENCE) and the NMW_* mirror load (TRANSFER_NMW_MIRROR); both default on. Run: python -m transfers.transfer_geothermal. Co-Authored-By: Claude Opus 4.8 --- .env.example | 6 +- transfers/transfer.py | 26 ++++---- transfers/transfer_geothermal.py | 105 +++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 transfers/transfer_geothermal.py diff --git a/.env.example b/.env.example index 8895581b0..23ad212e5 100644 --- a/.env.example +++ b/.env.example @@ -40,8 +40,10 @@ TRANSFER_NGWMN_VIEWS=True TRANSFER_WATERLEVELS_PRESSURE_DAILY=True TRANSFER_WEATHER_DATA=True TRANSFER_MINOR_TRACE_CHEMISTRY=True -# NM_Wells 1:1 staging mirror load (separate source DB; off by default) -TRANSFER_NMW_MIRROR=False +# NM_Wells (geothermal) migration: run `python -m transfers.transfer_geothermal` +# (separate from the deprecated transfers/transfer.py NM_Aquifer driver). +TRANSFER_GEOTHERMAL_REFERENCE=True # load ref_* lookups into the lexicon +TRANSFER_NMW_MIRROR=True # load the NMW_* 1:1 staging mirror # Optional: path to a NM_Wells SQL Server data-dump .sql file (INSERT statements). # When set, the mirror loads from it; otherwise it falls back to CSV exports. # NMW_SQL_DUMP=/path/to/NMWells_data.sql diff --git a/transfers/transfer.py b/transfers/transfer.py index c42910ada..f0ed4314d 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -13,8 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +"""DEPRECATED: legacy NM_Aquifer -> Ocotillo transfer orchestrator. + +This module (the original AMPAPI / NM_Aquifer migration driver) is deprecated. +Do not add new migrations here. New migrations get their own standalone +orchestrator script; e.g. the NM_Wells geothermal migration lives in +``transfers/transfer_geothermal.py``. +""" import os import time +import warnings from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import contextmanager from dataclasses import dataclass @@ -55,8 +63,6 @@ from services.env import get_bool_env from transfers.aquifer_system_transfer import transfer_aquifer_systems from transfers.geologic_formation_transfer import transfer_geologic_formations -from transfers.reference_lexicon_transfer import transfer_reference_tables -from transfers.nmw_mirror_transfer import transfer_nmw_mirror from transfers.permissions_transfer import transfer_permissions from transfers.stratigraphy_legacy import StratigraphyLegacyTransferer from transfers.stratigraphy_transfer import transfer_stratigraphy @@ -326,6 +332,12 @@ def _drop_and_rebuild_db() -> None: @timeit def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: + warnings.warn( + "transfers.transfer is deprecated; new migrations get their own " + "orchestrator (e.g. transfers/transfer_geothermal.py).", + DeprecationWarning, + stacklevel=2, + ) message("STARTING TRANSFER", new_line_at_top=False) if get_bool_env("DROP_AND_REBUILD_DB", False): logger.info("Dropping schema and rebuilding database from migrations") @@ -362,12 +374,11 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: else: message("PHASE 1: FOUNDATIONAL TRANSFERS (PARALLEL)") foundational_tasks = [ - ("ReferenceLexicon", transfer_reference_tables), ("AquiferSystems", transfer_aquifer_systems), ("GeologicFormations", transfer_geologic_formations), ] - with ThreadPoolExecutor(max_workers=len(foundational_tasks)) as executor: + with ThreadPoolExecutor(max_workers=2) as executor: futures = { executor.submit( _execute_foundational_transfer_with_timing, name, func, limit @@ -386,13 +397,6 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: logger.critical(f"Foundational transfer {name} failed: {e}") raise # Fail fast - foundational transfers must succeed - # NM_Wells 1:1 staging mirror (separate source DB). Off by default so it - # does not run during the standard NM_Aquifer -> Ocotillo transfer. - if get_bool_env("TRANSFER_NMW_MIRROR", False): - message("NM_WELLS 1:1 STAGING MIRROR LOAD") - with session_ctx() as session: - transfer_nmw_mirror(session, limit=limit) - message("TRANSFERRING WELLS") use_parallel_wells = get_bool_env("TRANSFER_PARALLEL_WELLS", True) if use_parallel_wells: diff --git a/transfers/transfer_geothermal.py b/transfers/transfer_geothermal.py new file mode 100644 index 000000000..a9d9d0b47 --- /dev/null +++ b/transfers/transfer_geothermal.py @@ -0,0 +1,105 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Standalone orchestrator for the NM_Wells (geothermal) migration. + +Separate from the deprecated ``transfers/transfer.py`` (NM_Aquifer driver). This +script runs the NM_Wells Phase-1 staging migration: + +1. Reference -> lexicon load (``ref_*`` lookups), gated by + ``TRANSFER_GEOTHERMAL_REFERENCE`` (default True). +2. NM_Wells 1:1 staging mirror load into the ``NMW_*`` tables, gated by + ``TRANSFER_NMW_MIRROR`` (default True). Row source is a SQL Server data dump + when ``NMW_SQL_DUMP`` is set, otherwise per-table CSV exports. + +Assumes the schema already exists (run ``alembic upgrade head`` first). Does not +drop/rebuild the database. + +Run: + python -m transfers.transfer_geothermal +Env: + TRANSFER_LIMIT=1000 # rows per table (0/unset = all) + NMW_SQL_DUMP=/path/to/data.sql # optional; else CSV + TRANSFER_GEOTHERMAL_REFERENCE=1 + TRANSFER_NMW_MIRROR=1 +""" + +import os + +from dotenv import load_dotenv + +# Load .env FIRST, before any database imports. Do not override env vars already +# set by the runtime (e.g. Cloud Run jobs). +load_dotenv(override=False) + +# In managed runtimes DB_DRIVER is sometimes omitted while CLOUD_SQL_* are set. +if ( + not (os.getenv("DB_DRIVER") or "").strip() + and (os.getenv("CLOUD_SQL_INSTANCE_NAME") or "").strip() +): + os.environ["DB_DRIVER"] = "cloudsql" + +from db.engine import session_ctx # noqa: E402 +from services.env import get_bool_env # noqa: E402 +from transfers.logger import logger # noqa: E402 +from transfers.nmw_mirror_transfer import transfer_nmw_mirror # noqa: E402 +from transfers.reference_lexicon_transfer import transfer_reference_tables # noqa: E402 + + +def run_geothermal_transfer(limit: int = None) -> dict: + """Run the NM_Wells geothermal staging migration. Returns a summary dict.""" + limit = int(limit if limit is not None else os.getenv("TRANSFER_LIMIT", 0) or 0) + summary: dict = {} + + logger.info("========== NM_WELLS (GEOTHERMAL) MIGRATION ==========") + logger.info("limit=%s", limit or "all") + + if get_bool_env("TRANSFER_GEOTHERMAL_REFERENCE", True): + logger.info("---- Reference tables -> lexicon ----") + with session_ctx() as session: + tables, created, errors = transfer_reference_tables(session, limit=limit) + summary["reference"] = { + "tables": tables, + "terms_created": created, + "errors": len(errors), + } + else: + logger.info("Skipping reference->lexicon (TRANSFER_GEOTHERMAL_REFERENCE=0)") + + if get_bool_env("TRANSFER_NMW_MIRROR", True): + logger.info("---- NM_Wells 1:1 staging mirror ----") + with session_ctx() as session: + tables, inserted, errors = transfer_nmw_mirror(session, limit=limit) + summary["mirror"] = { + "tables": tables, + "rows_inserted": inserted, + "errors": len(errors), + } + else: + logger.info("Skipping NM_Wells mirror (TRANSFER_NMW_MIRROR=0)") + + logger.info("NM_Wells migration complete: %s", summary) + return summary + + +def main() -> None: + run_geothermal_transfer() + + +if __name__ == "__main__": + main() + + +# ============= EOF ============================================= From 6d7e92c2b28a9f3d915d8f275c712484e48b79a5 Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Sun, 7 Jun 2026 04:42:33 +0000 Subject: [PATCH 07/41] Formatting changes --- transfers/transfer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transfers/transfer.py b/transfers/transfer.py index f0ed4314d..b1cae6ba2 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -20,6 +20,7 @@ orchestrator script; e.g. the NM_Wells geothermal migration lives in ``transfers/transfer_geothermal.py``. """ + import os import time import warnings From 83eb17029b15b1f4ce7bfb74bbbdd1dacf865305 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 6 Jun 2026 22:46:01 -0600 Subject: [PATCH 08/41] feat(transfers): ref lexicon loads from the same SQL dump as the mirror reference_lexicon_transfer now selects its row source the same way as nmw_mirror_transfer: a SQL Server data dump when NMW_SQL_DUMP is set (parsed by nmw_sql_dump.iter_table_rows), otherwise per-table CSV. _pick_columns operates on a column-name list and rows are processed as dicts so both sources share one path. Co-Authored-By: Claude Opus 4.8 --- transfers/reference_lexicon_transfer.py | 81 +++++++++++++++++++------ 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/transfers/reference_lexicon_transfer.py b/transfers/reference_lexicon_transfer.py index 6a20461ae..3ae6ecb81 100644 --- a/transfers/reference_lexicon_transfer.py +++ b/transfers/reference_lexicon_transfer.py @@ -25,8 +25,9 @@ term<->category associations are inserted only when missing (no unique constraint exists on that table). -Source CSVs are read with ``transfers.util.read_csv`` (looks in -``transfers/data/nma_csv_cache/
.csv`` then GCS ``nma_csv/
.csv``). +Row source is the same as the mirror loader: a SQL Server data dump when +``NMW_SQL_DUMP`` is set (parsed by ``transfers.nmw_sql_dump.iter_table_rows``), +otherwise per-table CSV exports via ``transfers.util.read_csv``. NOTE(columns): the ref tables' actual column names are not in the workbook, so term/definition columns are AUTO-DETECTED per table (see ``_pick_columns``). If @@ -38,6 +39,8 @@ to ``REFERENCE_TABLE_SPECS`` once their CSVs are available. """ +import itertools +import os from dataclasses import dataclass from typing import Optional @@ -52,8 +55,12 @@ LexiconTermCategoryAssociation, ) from transfers.logger import logger +from transfers.nmw_sql_dump import iter_table_rows from transfers.util import read_csv +# Same source selector as the mirror loader: a SQL Server data dump when +# NMW_SQL_DUMP is set, otherwise per-table CSV exports. +_SQL_DUMP_ENV = "NMW_SQL_DUMP" # lexicon_term.term (and its FK targets) is String(100). _TERM_MAX_LEN = 100 @@ -158,15 +165,15 @@ def _spec(table: str) -> RefTableSpec: ] -def _pick_columns(df: pd.DataFrame, spec: RefTableSpec) -> tuple[str, str]: +def _pick_columns(columns: list[str], spec: RefTableSpec) -> tuple[str, str]: """Resolve (term_col, definition_col) for a ref table. - Honors explicit overrides on the spec, else auto-detects from the header - using name hints, ignoring meta columns (OBJECTID, GlobalID, ...). + Honors explicit overrides on the spec, else auto-detects from the column + names using name hints, ignoring meta columns (OBJECTID, GlobalID, ...). """ - cols = [c for c in df.columns if str(c).strip().lower() not in _META_COLS] + cols = [c for c in columns if str(c).strip().lower() not in _META_COLS] if not cols: - cols = list(df.columns) + cols = list(columns) low = {c: str(c).strip().lower() for c in cols} term_col = spec.term_col @@ -213,29 +220,53 @@ def _get_or_create_category(session: Session, spec: RefTableSpec) -> int: ).scalar_one() +def _iter_source_rows(table: str, limit: int = 0): + """Yield raw ``{column: value}`` dicts for a ref table. + + SQL dump when NMW_SQL_DUMP is set (same source as the mirror loader), + otherwise per-table CSV. Mirrors transfers.nmw_mirror_transfer._row_source. + """ + dump = os.getenv(_SQL_DUMP_ENV) + if dump: + it = iter_table_rows(dump, table) + else: + df = read_csv(table) + it = (rec for rec in df.to_dict("records")) + if limit and limit > 0: + it = itertools.islice(it, limit) + return it + + def _transfer_one(session: Session, spec: RefTableSpec, limit: int = 0) -> dict: """Load a single ref table into the lexicon. Returns a stats dict.""" try: - df = read_csv(spec.source_table) - except Exception as e: # noqa: BLE001 - missing CSV / GCS miss should not abort - logger.warning("Skipping %s (could not read CSV): %s", spec.source_table, e) + rows = list(_iter_source_rows(spec.source_table, limit)) + except Exception as e: # noqa: BLE001 - missing source should not abort the run + logger.warning("Skipping %s (could not read source): %s", spec.source_table, e) return {"table": spec.source_table, "skipped": True, "reason": str(e)} - if limit and limit > 0: - df = df.head(limit) - - if df.empty or not list(df.columns): + if not rows: logger.warning("Skipping %s (empty)", spec.source_table) return {"table": spec.source_table, "skipped": True, "reason": "empty"} - term_col, def_col = _pick_columns(df, spec) + # Column names from the union of row keys (CSV rows and SSMS INSERTs are + # column-consistent, but be defensive). + columns: list[str] = [] + seen = set() + for rec in rows: + for k in rec: + if k not in seen: + seen.add(k) + columns.append(k) + + term_col, def_col = _pick_columns(columns, spec) logger.info( "%s -> category=%s term_col=%s definition_col=%s (%d rows)", spec.source_table, spec.category, term_col, def_col, - len(df), + len(rows), ) category_id = _get_or_create_category(session, spec) @@ -243,14 +274,14 @@ def _transfer_one(session: Session, spec: RefTableSpec, limit: int = 0) -> dict: # Build unique (term -> definition) map, dropping empties and overlong terms. term_defs: dict[str, str] = {} truncated = 0 - for row in df.itertuples(index=False): - term = _clean(getattr(row, term_col, None)) + for rec in rows: + term = _clean(rec.get(term_col)) if term is None: continue if len(term) > _TERM_MAX_LEN: term = term[:_TERM_MAX_LEN] truncated += 1 - definition = _clean(getattr(row, def_col, None)) or term + definition = _clean(rec.get(def_col)) or term term_defs.setdefault(term, definition) if not term_defs: @@ -316,7 +347,7 @@ def _transfer_one(session: Session, spec: RefTableSpec, limit: int = 0) -> dict: return { "table": spec.source_table, "skipped": False, - "rows": len(df), + "rows": len(rows), "terms": len(term_defs), "created_terms": len(new_rows), "linked": len(assoc_rows), @@ -331,6 +362,16 @@ def transfer_reference_tables(session: Session, limit: int = None) -> tuple: ``(num_tables, total_created_terms, errors)``. """ limit = int(limit or 0) + dump = os.getenv(_SQL_DUMP_ENV) + if dump: + if not os.path.exists(dump): + raise FileNotFoundError(f"{_SQL_DUMP_ENV} set but file not found: {dump}") + logger.info("Reference lexicon source: SQL dump %s", dump) + else: + logger.info( + "Reference lexicon source: CSV exports (set %s for a dump)", _SQL_DUMP_ENV + ) + results = [] errors = [] for spec in REFERENCE_TABLE_SPECS: From f4610982204cc323870c15679db5f725258d2db2 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:22:05 -0600 Subject: [PATCH 09/41] docs(db): flag NMW_* attributes that become lexicon terms/enums Add LEXICON_REF_BY_COLUMN mapping every coded mirror attribute to its ref_* source table (which reference_lexicon_transfer loads as a lexicon category whose rows become terms). These 40 attributes will become lexicon_term FKs / enums in the Phase-2 transform. Add LEXICON_CANDIDATES_NO_REF for 8 coded columns that have no ref_* table and will need a new category/enum (DrillFluid, TestType, Operation, etc.). Validated: every column + ref table exists. Co-Authored-By: Claude Opus 4.8 --- db/nmw_legacy.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index 9d65c7226..689a704df 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -100,6 +100,20 @@ on OBJECTID and GlobalID; OBJECTID (identity, never NULL) is used. - Geothermal/DST: declared PKs where present (BHTGUID, IntrvlGUID, DSTGUID, DSTInterval); the rest are heaps keyed on the OBJECTID identity column. + +LEXICON FLAGGING (Phase 2) +-------------------------- +Every ``ref_*`` table is loaded as a ``LexiconCategory`` whose rows become +``LexiconTerm``s (see transfers/reference_lexicon_transfer.py). The mirror +columns that hold those coded values will, in the Phase-2 Ocotillo model, +become ``lexicon_term`` foreign keys / enums. + +``LEXICON_REF_BY_COLUMN`` below flags every such attribute, mapping +``{tablename: {source_column: ref_source_table}}``. The lexicon *category* for +each ref table is assigned by ``transfers/reference_lexicon_transfer.py`` (one +category per ref table), so this map records the stable ref-table name rather +than the derived category string. ``LEXICON_CANDIDATES_NO_REF`` lists coded +columns that have no ``ref_*`` table and will need a NEW category / enum. """ from sqlalchemy import ( @@ -115,6 +129,89 @@ from db.base import Base +# Attributes that will become lexicon_term FKs / enums in the Phase-2 transform. +# {tablename: {source_column: ref_source_table}}. The lexicon category per ref +# table is assigned by transfers/reference_lexicon_transfer.py. +LEXICON_REF_BY_COLUMN: dict[str, dict[str, str]] = { + "NMW_WellLocations": { + "UnitLetter": "ref_unit_letters", + "State": "ref_states", + "County": "ref_county", + "Basin": "ref_basins", + "SourceDatum": "ref_coordinate_datum", + "SourceUnits": "ref_xy_units", + "LocAccType": "ref_coordinate_accuracy", + "LocAccMeas": "ref_coordinate_method", + }, + "NMW_WellHeaders": { + "WellClass": "ref_well_class", + "WellType": "ref_well_types", + "WellOrient": "ref_well_orientations", + "CurStatus": "ref_well_status", + }, + "NMW_WellRecords": { + "RecrdClass": "ref_well_record_class", + }, + "NMW_WellZDatum": { + "DepthDatum": "ref_ground_levels", + "DepthUnits": "ref_unit_depths", + "Z_datum": "ref_altitude_datums", + "Z_units": "ref_unit_depths", + "ElevSource": "ref_altitude_methods", + }, + "NMW_WellSamples": { + "SamplClass": "ref_sample_class", + "SampleType": "ref_sample_types", + "SmpDpUnt": "ref_unit_depths", + }, + "NMW_GtBhtHeaders": { + "BoreUnits": "ref_length_units", + "TempUnit": "ref_unit_temps", + }, + "NMW_GtBhtData": { + "TempUnit": "ref_unit_temps", + }, + "NMW_GtConductivity": { + "CnductUnit": "ref_unit_conductivity", + }, + "NMW_GtHeatFlow": { + "Kpr_unit": "ref_unit_conductivity", + "Q_unit": "ref_unit_heat_flow", + }, + "NMW_GtSumHeatFlow": { + "LithClass": "ref_lith_class", + "UnitBasis": "ref_unit_basis", + "DepthUnit": "ref_unit_depths", + "GradUnit": "ref_unit_gradients", + "SampleType": "ref_sample_types", + "TCondUnit": "ref_unit_conductivity", + "HtFlowUnit": "ref_unit_heat_flow", + }, + "NMW_GtTempDepths": { + "TempUnit": "ref_unit_temps", + }, + "NMW_WsDstHeaders": { + "PressUnits": "ref_pres_units", + "TempUnit": "ref_unit_temps", + "PipeDiaUnt": "ref_length_units", + "PipeLenUnt": "ref_length_units", + "ChokeSizUn": "ref_length_units", + }, +} + +# Coded/categorical columns with NO existing ref_* table; Phase 2 must create a +# new lexicon category or enum for these. {tablename: [source_column, ...]}. +LEXICON_CANDIDATES_NO_REF: dict[str, list[str]] = { + "NMW_GtBhtHeaders": ["DrillFluid"], + "NMW_GtHeatFlow": ["Ka_unit"], + "NMW_GtSumHeatFlow": ["Quality"], + "NMW_WsDstHeaders": ["TestType"], + "NMW_WsDstIntervals": ["Status"], + "NMW_WsDstFlowHistory": ["Operation", "RecovType"], + "NMW_WsDstPressure": ["DSTFluid"], +} + + class NMW_WellLocations(Base): """1:1 mirror of NM_Wells ``tbl_well_locations`` (Main / Migrate First). From cff5d52c81b20f6bf4a55505c0e62d91a5d1851c Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Sun, 7 Jun 2026 06:22:26 +0000 Subject: [PATCH 10/41] Formatting changes --- db/nmw_legacy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index 689a704df..4c53c32c9 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -128,7 +128,6 @@ from db.base import Base - # Attributes that will become lexicon_term FKs / enums in the Phase-2 transform. # {tablename: {source_column: ref_source_table}}. The lexicon category per ref # table is assigned by transfers/reference_lexicon_transfer.py. From a63179e7a28e56ec616ed0cc3397536c086cb2ff Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:24:30 -0600 Subject: [PATCH 11/41] chore(transfers): clean up _spec category derivation Remove the dead `category = table[4:]` line and fix the stale docstring; the category is nmw_
(e.g. nmw_ref_states). Co-Authored-By: Claude Opus 4.8 --- transfers/reference_lexicon_transfer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transfers/reference_lexicon_transfer.py b/transfers/reference_lexicon_transfer.py index 3ae6ecb81..bfd391e21 100644 --- a/transfers/reference_lexicon_transfer.py +++ b/transfers/reference_lexicon_transfer.py @@ -97,11 +97,10 @@ class RefTableSpec: def _spec(table: str) -> RefTableSpec: - """Build a spec with category = table name minus the ``ref_`` prefix.""" - category = table[4:] if table.startswith("ref_") else table + """Build a spec with category ``nmw_
`` (e.g. ``nmw_ref_states``).""" return RefTableSpec( source_table=table, - category=category, + category=f"nmw_{table}", description=f"Imported from NM_Wells {table}", ) From 64e4fdfb6a23213bb4b85866878a3194478e60c4 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:39:05 -0600 Subject: [PATCH 12/41] feat(alembic): add geothermal OGC views (BHT + temperature-depth profile) Two pygeoapi point layers over the NMW_* staging mirror, geometry from NMW_WellLocations Lat/Long_dd83: - ogc_geothermal_wells_bht: one feature per geothermal well with bottom-hole temperature data (NMW_GtBhtData), aggregate BHT stats. - ogc_geothermal_wells_temperature_profile: one feature per geothermal well with a downhole temperature-vs-depth series (NMW_GtTempDepths) as an ordered JSON array. Wells link via gt_*.SamplSetID -> NMW_WellSamples -> NMW_WellRecords -> NMW_WellLocations. Guards required tables; drops views on downgrade. Co-Authored-By: Claude Opus 4.8 --- .../w9x0y1z2a3b4_add_geothermal_ogc_views.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py diff --git a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py new file mode 100644 index 000000000..5749ce20a --- /dev/null +++ b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py @@ -0,0 +1,154 @@ +"""add geothermal OGC views (bottom-hole temps + temperature-depth profile) + +Revision ID: w9x0y1z2a3b4 +Revises: v8w9x0y1z2a3 +Create Date: 2026-06-07 00:00:00.000000 + +Two pygeoapi point layers over the NM_Wells staging mirror (db/nmw_legacy.py): + + ogc_geothermal_wells_bht + One feature per geothermal well that has bottom-hole-temperature data + (NMW_GtBhtData), with aggregate BHT stats. + + ogc_geothermal_wells_temperature_profile + One feature per geothermal well that has a downhole temperature-vs-depth + series (NMW_GtTempDepths), with the ordered series as a JSON array. + +Well geometry is built from NMW_WellLocations Lat/Long_dd83 (WGS84). Geothermal +data links to a well via: + gt_*.SamplSetID -> NMW_WellSamples.SamplSetID + NMW_WellSamples.RecrdsetID -> NMW_WellRecords.RecrdSetID + NMW_WellRecords.WellDataID -> NMW_WellLocations/Headers.WellDataID +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "w9x0y1z2a3b4" +down_revision: Union[str, Sequence[str], None] = "v8w9x0y1z2a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_BHT_VIEW = "ogc_geothermal_wells_bht" +_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" + +_REQUIRED_TABLES = ( + "NMW_WellLocations", + "NMW_WellHeaders", + "NMW_WellRecords", + "NMW_WellSamples", + "NMW_GtBhtData", + "NMW_GtBhtHeaders", + "NMW_GtTempDepths", +) + + +def _create_bht_view() -> str: + return """ + CREATE VIEW ogc_geothermal_wells_bht AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """ + + +def _create_profile_view() -> str: + return """ + CREATE VIEW ogc_geothermal_wells_temperature_profile AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + AND td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + + +def upgrade() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing = set(inspector.get_table_names(schema="public")) + missing = [t for t in _REQUIRED_TABLES if t not in existing] + if missing: + raise RuntimeError( + "Cannot create geothermal OGC views. Missing required tables: " + + ", ".join(missing) + ) + + op.execute(text(f"DROP VIEW IF EXISTS {_BHT_VIEW}")) + op.execute(text(_create_bht_view())) + op.execute( + text( + f"COMMENT ON VIEW {_BHT_VIEW} IS " + "'Geothermal wells with bottom-hole-temperature data (pygeoapi).'" + ) + ) + + op.execute(text(f"DROP VIEW IF EXISTS {_PROFILE_VIEW}")) + op.execute(text(_create_profile_view())) + op.execute( + text( + f"COMMENT ON VIEW {_PROFILE_VIEW} IS " + "'Geothermal wells with downhole temperature-vs-depth series " + "(pygeoapi).'" + ) + ) + + +def downgrade() -> None: + op.execute(text(f"DROP VIEW IF EXISTS {_PROFILE_VIEW}")) + op.execute(text(f"DROP VIEW IF EXISTS {_BHT_VIEW}")) From 9b8037affe6876a16631ea5d38cb8a25dadedd3c Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:46:03 -0600 Subject: [PATCH 13/41] feat(alembic): update comments for geothermal OGC views --- alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py index 5749ce20a..832f295ec 100644 --- a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py +++ b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py @@ -4,7 +4,7 @@ Revises: v8w9x0y1z2a3 Create Date: 2026-06-07 00:00:00.000000 -Two pygeoapi point layers over the NM_Wells staging mirror (db/nmw_legacy.py): +Two point layers over the NM_Wells staging mirror (db/nmw_legacy.py): ogc_geothermal_wells_bht One feature per geothermal well that has bottom-hole-temperature data @@ -134,7 +134,7 @@ def upgrade() -> None: op.execute( text( f"COMMENT ON VIEW {_BHT_VIEW} IS " - "'Geothermal wells with bottom-hole-temperature data (pygeoapi).'" + "'Geothermal wells with bottom-hole-temperature data.'" ) ) @@ -143,8 +143,7 @@ def upgrade() -> None: op.execute( text( f"COMMENT ON VIEW {_PROFILE_VIEW} IS " - "'Geothermal wells with downhole temperature-vs-depth series " - "(pygeoapi).'" + "'Geothermal wells with downhole temperature-vs-depth series.'" ) ) From 24a199c09f282bac4bca1285671b611d2989e357 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:54:04 -0600 Subject: [PATCH 14/41] perf(alembic): materialize geothermal temperature-profile OGC view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The temperature-vs-depth profile view scans/groups NMW_GtTempDepths (~370k source rows) and builds a per-well JSON series — too heavy to recompute per pygeoapi request. Convert it to a MATERIALIZED view with a unique index on well_data_id (enables REFRESH CONCURRENTLY) and a GiST index on geom. The BHT view stays a regular view (small source). REFRESH after a data reload. Co-Authored-By: Claude Opus 4.8 --- .../w9x0y1z2a3b4_add_geothermal_ogc_views.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py index 832f295ec..6e89bff9b 100644 --- a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py +++ b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py @@ -10,9 +10,11 @@ One feature per geothermal well that has bottom-hole-temperature data (NMW_GtBhtData), with aggregate BHT stats. - ogc_geothermal_wells_temperature_profile + ogc_geothermal_wells_temperature_profile (MATERIALIZED) One feature per geothermal well that has a downhole temperature-vs-depth - series (NMW_GtTempDepths), with the ordered series as a JSON array. + series (NMW_GtTempDepths, ~370k source rows), with the ordered series as + a JSON array. Materialized + indexed (unique well_data_id, GiST geom); + REFRESH MATERIALIZED VIEW after a data reload. Well geometry is built from NMW_WellLocations Lat/Long_dd83 (WGS84). Geothermal data links to a well via: @@ -81,8 +83,12 @@ def _create_bht_view() -> str: def _create_profile_view() -> str: + # Materialized: the source NMW_GtTempDepths is large (~370k source rows) and + # this groups + builds a JSON series per well, too heavy to recompute per + # pygeoapi request. Staging data loads once, so staleness is a non-issue; + # REFRESH MATERIALIZED VIEW after a reload. return """ - CREATE VIEW ogc_geothermal_wells_temperature_profile AS + CREATE MATERIALIZED VIEW ogc_geothermal_wells_temperature_profile AS SELECT r."WellDataID" AS well_data_id, hdr."CurWellNam" AS well_name, @@ -138,16 +144,30 @@ def upgrade() -> None: ) ) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {_PROFILE_VIEW}")) op.execute(text(f"DROP VIEW IF EXISTS {_PROFILE_VIEW}")) op.execute(text(_create_profile_view())) op.execute( text( - f"COMMENT ON VIEW {_PROFILE_VIEW} IS " + f"COMMENT ON MATERIALIZED VIEW {_PROFILE_VIEW} IS " "'Geothermal wells with downhole temperature-vs-depth series.'" ) ) + # Unique index on the feature id enables REFRESH ... CONCURRENTLY; GiST on + # the geometry for fast pygeoapi bbox queries. + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " + f"ON {_PROFILE_VIEW} (well_data_id)" + ) + ) + op.execute( + text( + f"CREATE INDEX ix_{_PROFILE_VIEW}_geom ON {_PROFILE_VIEW} USING gist (geom)" + ) + ) def downgrade() -> None: - op.execute(text(f"DROP VIEW IF EXISTS {_PROFILE_VIEW}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {_PROFILE_VIEW}")) op.execute(text(f"DROP VIEW IF EXISTS {_BHT_VIEW}")) From 158d97e11d892f44436859ad2e51984373cd9845 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:55:33 -0600 Subject: [PATCH 15/41] feat(alembic): add geothermal heat-flow OGC view pygeoapi point layer ogc_geothermal_wells_heat_flow: one feature per geothermal well with summary heat-flow determinations (NMW_GtSumHeatFlow) - aggregate heat flow, thermal gradient, thermal conductivity and quality. Geometry from NMW_WellLocations; linked via NMW_GtSumHeatFlow.RecrdSetID -> NMW_WellRecords. Co-Authored-By: Claude Opus 4.8 --- ...3b4c5_add_geothermal_heat_flow_ogc_view.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py diff --git a/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py b/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py new file mode 100644 index 000000000..4dbf8c7c4 --- /dev/null +++ b/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py @@ -0,0 +1,92 @@ +"""add geothermal heat-flow OGC view + +Revision ID: x0y1z2a3b4c5 +Revises: w9x0y1z2a3b4 +Create Date: 2026-06-07 00:00:01.000000 + +pygeoapi point layer of geothermal wells with summary heat-flow determinations +(NMW_GtSumHeatFlow), one feature per well with aggregate heat-flow / gradient / +conductivity stats. Geometry from NMW_WellLocations Lat/Long_dd83. + +Link: NMW_GtSumHeatFlow.RecrdSetID -> NMW_WellRecords.RecrdSetID -> +NMW_WellLocations/Headers.WellDataID. +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "x0y1z2a3b4c5" +down_revision: Union[str, Sequence[str], None] = "w9x0y1z2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_VIEW = "ogc_geothermal_wells_heat_flow" + +_REQUIRED_TABLES = ( + "NMW_WellLocations", + "NMW_WellHeaders", + "NMW_WellRecords", + "NMW_GtSumHeatFlow", +) + + +def _create_view() -> str: + return """ + CREATE VIEW ogc_geothermal_wells_heat_flow AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(shf.*) AS heat_flow_count, + max(shf."HeatFlow") AS max_heat_flow, + avg(shf."HeatFlow") AS avg_heat_flow, + max(shf."HtFlowUnit") AS heat_flow_unit, + max(shf."ThermlGrad") AS max_thermal_gradient, + max(shf."GradUnit") AS gradient_unit, + max(shf."ThermlCond") AS max_thermal_conductivity, + max(shf."TCondUnit") AS conductivity_unit, + max(shf."Quality") AS quality, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtSumHeatFlow" AS shf + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + + +def upgrade() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing = set(inspector.get_table_names(schema="public")) + missing = [t for t in _REQUIRED_TABLES if t not in existing] + if missing: + raise RuntimeError( + "Cannot create geothermal heat-flow OGC view. Missing required " + "tables: " + ", ".join(missing) + ) + + op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) + op.execute(text(_create_view())) + op.execute( + text( + f"COMMENT ON VIEW {_VIEW} IS " + "'Geothermal wells with summary heat-flow determinations (pygeoapi).'" + ) + ) + + +def downgrade() -> None: + op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) From a92cba95ad527c7b0971f2d1b11811c5a2c85b2d Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 00:58:39 -0600 Subject: [PATCH 16/41] feat(alembic): add geothermal per-interval heat-flow OGC view pygeoapi point layer ogc_geothermal_wells_interval_heat_flow from NMW_GtHeatFlow (per-interval values: Q heat flow, gradient, Kpr conductivity, Ka diffusivity), one feature per well. Distinct from ogc_geothermal_wells_heat_flow (summary, NMW_GtSumHeatFlow). Linked via IntrvlGUID -> NMW_WsIntervals -> NMW_WellSamples -> NMW_WellRecords -> NMW_WellLocations. Co-Authored-By: Claude Opus 4.8 --- ..._geothermal_interval_heat_flow_ogc_view.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py diff --git a/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py b/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py new file mode 100644 index 000000000..7b5b5163a --- /dev/null +++ b/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py @@ -0,0 +1,98 @@ +"""add geothermal per-interval heat-flow OGC view + +Revision ID: y1z2a3b4c5d6 +Revises: x0y1z2a3b4c5 +Create Date: 2026-06-07 00:00:02.000000 + +pygeoapi point layer of geothermal wells with per-interval heat-flow values +(NMW_GtHeatFlow), one feature per well with aggregate heat-flow / gradient / +conductivity / diffusivity stats. Distinct from ogc_geothermal_wells_heat_flow, +which is the summary (NMW_GtSumHeatFlow) layer. + +Link: NMW_GtHeatFlow.IntrvlGUID -> NMW_WsIntervals.IntrvlGUID -> +NMW_WellSamples.SamplSetID -> NMW_WellRecords.RecrdSetID -> +NMW_WellLocations/Headers.WellDataID. +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "y1z2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "x0y1z2a3b4c5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_VIEW = "ogc_geothermal_wells_interval_heat_flow" + +_REQUIRED_TABLES = ( + "NMW_WellLocations", + "NMW_WellHeaders", + "NMW_WellRecords", + "NMW_WellSamples", + "NMW_WsIntervals", + "NMW_GtHeatFlow", +) + + +def _create_view() -> str: + return """ + CREATE VIEW ogc_geothermal_wells_interval_heat_flow AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(hf.*) AS interval_count, + max(hf."Q") AS max_heat_flow, + avg(hf."Q") AS avg_heat_flow, + max(hf."Q_unit") AS heat_flow_unit, + max(hf."Gradient") AS max_gradient, + max(hf."Kpr") AS max_thermal_conductivity, + max(hf."Kpr_unit") AS conductivity_unit, + max(hf."Ka") AS max_diffusivity, + max(hf."Ka_unit") AS diffusivity_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtHeatFlow" AS hf + JOIN "NMW_WsIntervals" AS i ON i."IntrvlGUID" = hf."IntrvlGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = i."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + + +def upgrade() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing = set(inspector.get_table_names(schema="public")) + missing = [t for t in _REQUIRED_TABLES if t not in existing] + if missing: + raise RuntimeError( + "Cannot create geothermal interval heat-flow OGC view. Missing " + "required tables: " + ", ".join(missing) + ) + + op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) + op.execute(text(_create_view())) + op.execute( + text( + f"COMMENT ON VIEW {_VIEW} IS " + "'Geothermal wells with per-interval heat-flow values (pygeoapi).'" + ) + ) + + +def downgrade() -> None: + op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) From a4a5952da259b01ab1395f95ca38343f6e527abb Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 01:04:26 -0600 Subject: [PATCH 17/41] feat(alembic): heat-flow OGC views return per-feature measurement series - Rename ogc_geothermal_wells_heat_flow -> ogc_geothermal_wells_summary_heat_flow. - Add a `measurements` JSON series to both heat-flow views: one element per determination/interval (depth range, heat flow, gradient, conductivity, etc.), ordered by depth, alongside the existing per-well aggregates. Co-Authored-By: Claude Opus 4.8 --- ...3b4c5_add_geothermal_heat_flow_ogc_view.py | 27 +++++++++++++++---- ..._geothermal_interval_heat_flow_ogc_view.py | 20 +++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py b/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py index 4dbf8c7c4..5422102ae 100644 --- a/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py +++ b/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py @@ -4,9 +4,10 @@ Revises: w9x0y1z2a3b4 Create Date: 2026-06-07 00:00:01.000000 -pygeoapi point layer of geothermal wells with summary heat-flow determinations -(NMW_GtSumHeatFlow), one feature per well with aggregate heat-flow / gradient / -conductivity stats. Geometry from NMW_WellLocations Lat/Long_dd83. +pygeoapi point layer ogc_geothermal_wells_summary_heat_flow: geothermal wells +with summary heat-flow determinations (NMW_GtSumHeatFlow), one feature per well +with aggregate stats plus a `measurements` JSON series (one element per +determination, ordered by depth). Geometry from NMW_WellLocations Lat/Long_dd83. Link: NMW_GtSumHeatFlow.RecrdSetID -> NMW_WellRecords.RecrdSetID -> NMW_WellLocations/Headers.WellDataID. @@ -23,7 +24,7 @@ branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None -_VIEW = "ogc_geothermal_wells_heat_flow" +_VIEW = "ogc_geothermal_wells_summary_heat_flow" _REQUIRED_TABLES = ( "NMW_WellLocations", @@ -35,7 +36,7 @@ def _create_view() -> str: return """ - CREATE VIEW ogc_geothermal_wells_heat_flow AS + CREATE VIEW ogc_geothermal_wells_summary_heat_flow AS SELECT r."WellDataID" AS well_data_id, hdr."CurWellNam" AS well_name, @@ -49,6 +50,22 @@ def _create_view() -> str: max(shf."ThermlCond") AS max_thermal_conductivity, max(shf."TCondUnit") AS conductivity_unit, max(shf."Quality") AS quality, + json_agg( + json_build_object( + 'from_depth', shf."FromDepth", + 'to_depth', shf."ToDepth", + 'depth_unit', shf."DepthUnit", + 'heat_flow', shf."HeatFlow", + 'heat_flow_error', shf."HtFlowErr", + 'heat_flow_unit', shf."HtFlowUnit", + 'thermal_gradient', shf."ThermlGrad", + 'gradient_unit', shf."GradUnit", + 'thermal_conductivity', shf."ThermlCond", + 'conductivity_unit', shf."TCondUnit", + 'quality', shf."Quality" + ) + ORDER BY shf."FromDepth" + ) AS measurements, ST_SetSRID( ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 ) AS geom diff --git a/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py b/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py index 7b5b5163a..12f570183 100644 --- a/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py +++ b/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py @@ -5,9 +5,9 @@ Create Date: 2026-06-07 00:00:02.000000 pygeoapi point layer of geothermal wells with per-interval heat-flow values -(NMW_GtHeatFlow), one feature per well with aggregate heat-flow / gradient / -conductivity / diffusivity stats. Distinct from ogc_geothermal_wells_heat_flow, -which is the summary (NMW_GtSumHeatFlow) layer. +(NMW_GtHeatFlow), one feature per well with aggregate stats plus a +`measurements` JSON series (one element per interval, ordered by depth). +Distinct from ogc_geothermal_wells_summary_heat_flow (NMW_GtSumHeatFlow). Link: NMW_GtHeatFlow.IntrvlGUID -> NMW_WsIntervals.IntrvlGUID -> NMW_WellSamples.SamplSetID -> NMW_WellRecords.RecrdSetID -> @@ -53,6 +53,20 @@ def _create_view() -> str: max(hf."Kpr_unit") AS conductivity_unit, max(hf."Ka") AS max_diffusivity, max(hf."Ka_unit") AS diffusivity_unit, + json_agg( + json_build_object( + 'from_depth', i."From_Depth", + 'to_depth', i."To_Depth", + 'heat_flow', hf."Q", + 'heat_flow_unit', hf."Q_unit", + 'gradient', hf."Gradient", + 'thermal_conductivity', hf."Kpr", + 'conductivity_unit', hf."Kpr_unit", + 'diffusivity', hf."Ka", + 'diffusivity_unit', hf."Ka_unit" + ) + ORDER BY i."From_Depth" + ) AS measurements, ST_SetSRID( ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 ) AS geom From 890232801a5076c1a5c9f32b36bf2debf4077c7c Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 21:58:06 -0600 Subject: [PATCH 18/41] feat(transfers): load NM_Wells mirror via sqlparse CSV + Postgres COPY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When NMW_SQL_DUMP is set, the mirror now parses the dump with sqlparse (nmw_sql_dump.write_table_csv) into a CSV per table, then bulk-loads each via Postgres COPY ... FROM STDIN (truncate + COPY; Postgres casts text -> types) — far faster than row-by-row ORM inserts. CSV dir defaults to a temp dir (override NMW_CSV_DIR). The CSV-exports fallback (no dump) keeps the row-insert path. Adds sqlparse dependency. Co-Authored-By: Claude Opus 4.8 --- .env.example | 5 +- pyproject.toml | 1 + transfers/nmw_mirror_transfer.py | 66 ++++++++++++-- transfers/nmw_sql_dump.py | 147 ++++++++++++++++--------------- uv.lock | 13 ++- 5 files changed, 151 insertions(+), 81 deletions(-) diff --git a/.env.example b/.env.example index 23ad212e5..a3dca336f 100644 --- a/.env.example +++ b/.env.example @@ -45,8 +45,11 @@ TRANSFER_MINOR_TRACE_CHEMISTRY=True TRANSFER_GEOTHERMAL_REFERENCE=True # load ref_* lookups into the lexicon TRANSFER_NMW_MIRROR=True # load the NMW_* 1:1 staging mirror # Optional: path to a NM_Wells SQL Server data-dump .sql file (INSERT statements). -# When set, the mirror loads from it; otherwise it falls back to CSV exports. +# When set, the mirror parses it to a CSV per table (sqlparse) and bulk-loads via +# Postgres COPY; otherwise it falls back to CSV exports + row inserts. # NMW_SQL_DUMP=/path/to/NMWells_data.sql +# Optional: dir for the per-table CSVs written from the dump (default: temp dir). +# NMW_CSV_DIR=/path/to/nmw_csv # asset storage GCS_BUCKET_NAME= diff --git a/pyproject.toml b/pyproject.toml index 813650d65..d6a96541b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ dependencies = [ "utm==0.8.1", "uvicorn==0.42.0", "yarl==1.23.0", + "sqlparse>=0.5.5", ] [tool.uv] diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 2b84e39c2..9bb0a961f 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -28,11 +28,14 @@ Two row sources, selected at runtime: 1. **SQL Server data dump** (preferred): set ``NMW_SQL_DUMP`` to a ``.sql`` file - containing ``INSERT [dbo].[tbl_*] (...) VALUES (...)`` statements. Rows are - streamed and parsed by ``transfers.nmw_sql_dump.iter_table_rows``. + of ``INSERT [dbo].[tbl_*] (...) VALUES (...)`` statements. Each table is + written to a CSV by ``transfers.nmw_sql_dump.write_table_csv`` (sqlparse) and + bulk-loaded with Postgres ``COPY ... FROM STDIN`` (truncate + COPY; Postgres + casts text -> column types). CSV output dir defaults to a temp dir, override + with ``NMW_CSV_DIR``. 2. **CSV exports** (fallback when ``NMW_SQL_DUMP`` is unset): per-table CSVs read with ``transfers.util.read_csv`` (``transfers/data/nma_csv_cache/
.csv`` - then GCS ``nma_csv/
.csv``). + then GCS ``nma_csv/
.csv``), inserted row-by-row with type coercion. In both cases the source column names are the original SQL Server names (OBJECTID, WellDataID, ...), which match the mirror columns' DB names exactly. @@ -42,11 +45,12 @@ import itertools import os +import tempfile import uuid from dataclasses import dataclass import pandas as pd -from sqlalchemy import DateTime, Float, Integer, LargeBinary, SmallInteger, String +from sqlalchemy import DateTime, Float, Integer, LargeBinary, SmallInteger, String, text from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session @@ -71,12 +75,15 @@ NMW_WsIntervals, ) from transfers.logger import logger -from transfers.nmw_sql_dump import iter_table_rows +from transfers.nmw_sql_dump import iter_table_rows, write_table_csv from transfers.util import read_csv # Path to a SQL Server data-dump .sql file. When set, rows are parsed from it; # otherwise the loader falls back to per-table CSV exports. _SQL_DUMP_ENV = "NMW_SQL_DUMP" +# Optional output dir for the per-table CSVs written from the dump (COPY path). +# Defaults to a fresh temp dir. +_CSV_DIR_ENV = "NMW_CSV_DIR" _CHUNK_SIZE = 2000 @@ -181,6 +188,45 @@ def _flush(session: Session, model, rows: list[dict], pk_cols: list[str]) -> int return result.rowcount if result.rowcount and result.rowcount > 0 else 0 +def _copy_csv_into_table( + session: Session, table_name: str, header: list[str], csv_path: str +) -> None: + """Bulk-load a CSV into ``table_name`` via Postgres COPY (pg8000 stream).""" + collist = ", ".join(f'"{c}"' for c in header) + sql = ( + f'COPY "{table_name}" ({collist}) FROM STDIN ' + "WITH (FORMAT CSV, HEADER true, NULL '')" + ) + raw = session.connection().connection # underlying pg8000 DBAPI connection + cursor = raw.cursor() + with open(csv_path, "rb") as f: + cursor.execute(sql, stream=f) + + +def _copy_load_table( + session: Session, spec: MirrorSpec, dump: str, out_dir: str, limit: int = 0 +) -> dict: + """Dump -> per-table CSV (sqlparse) -> COPY into the mirror table.""" + table = spec.model.__table__ + name = spec.source_table + # Load only model columns (rowversion/LargeBinary excluded). COPY relies on + # Postgres to cast text -> column types, so no Python coercion is needed. + columns = [c.name for c in table.columns if not isinstance(c.type, LargeBinary)] + out_csv = os.path.join(out_dir, f"{name}.csv") + + n, header = write_table_csv(dump, name, out_csv, columns=columns, limit=limit) + if n == 0: + logger.warning("Skipping %s (no rows in dump)", name) + return {"table": name, "skipped": True, "reason": "no rows", "source": "sql"} + + # Staging reload: truncate then COPY (no upsert; tables are a 1:1 snapshot). + session.execute(text(f'TRUNCATE TABLE "{table.name}"')) + _copy_csv_into_table(session, table.name, header, out_csv) + session.commit() + logger.info("COPY %s -> %s: %d rows (%s)", name, table.name, n, out_csv) + return {"table": name, "skipped": False, "rows": n, "inserted": n, "source": "sql"} + + def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: """Load one source table (SQL dump or CSV) into its mirror. Stats dict.""" table = spec.model.__table__ @@ -251,10 +297,13 @@ def transfer_nmw_mirror(session: Session, limit: int = None) -> tuple: """ limit = int(limit or 0) dump = os.getenv(_SQL_DUMP_ENV) + out_dir = None if dump: if not os.path.exists(dump): raise FileNotFoundError(f"{_SQL_DUMP_ENV} set but file not found: {dump}") - logger.info("NMW mirror source: SQL dump %s", dump) + out_dir = os.getenv(_CSV_DIR_ENV) or tempfile.mkdtemp(prefix="nmw_csv_") + os.makedirs(out_dir, exist_ok=True) + logger.info("NMW mirror source: SQL dump %s -> CSV %s -> COPY", dump, out_dir) else: logger.info("NMW mirror source: CSV exports (set %s for a dump)", _SQL_DUMP_ENV) @@ -262,7 +311,10 @@ def transfer_nmw_mirror(session: Session, limit: int = None) -> tuple: errors = [] for spec in NMW_MIRROR_SPECS: try: - results.append(_load_table(session, spec, limit)) + if dump: + results.append(_copy_load_table(session, spec, dump, out_dir, limit)) + else: + results.append(_load_table(session, spec, limit)) except Exception as e: # noqa: BLE001 - isolate per-table failures logger.critical("NMW mirror load failed for %s: %s", spec.source_table, e) session.rollback() diff --git a/transfers/nmw_sql_dump.py b/transfers/nmw_sql_dump.py index 39637c076..29f72e735 100644 --- a/transfers/nmw_sql_dump.py +++ b/transfers/nmw_sql_dump.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -"""Stream rows out of a SQL Server data-dump ``.sql`` file. +"""Parse a SQL Server data-dump ``.sql`` file into per-table CSVs. -Parses ``INSERT [dbo].[
] () VALUES ()[, () ...]`` -statements (the format produced by SSMS "Generate Scripts -> data" / ``bcp`` -INSERT mode) for one target table at a time, yielding ``{column: value}`` -dicts. Values are decoded to plain Python: +``INSERT [dbo].[
] () VALUES ()[, () ...]`` statements +(SSMS "Generate Scripts -> data" / bcp INSERT mode) are split with ``sqlparse`` +and decoded to plain Python values: NULL -> None N'...' / '...' -> str (doubled '' unescaped) @@ -26,20 +25,21 @@ CAST(expr AS type) -> the inner expr, recursively 0x.... -> None (binary / rowversion; not mirrored) -Type coercion to the target column type happens in nmw_mirror_transfer._coerce, -so this module keeps values loosely typed. - -Streaming: the file is read line by line (constant memory), accumulating across -lines only when a statement's parentheses are unbalanced (strings containing -newlines). The file is scanned once per table. +``iter_table_rows`` yields ``{column: value}`` dicts; ``write_table_csv`` writes +one table to a CSV suitable for a Postgres ``COPY ... FROM`` bulk load (NULL -> +empty field, so load with ``NULL ''``). Encoding is auto-detected from the BOM (SSMS writes UTF-16 LE); falls back to utf-8. """ +import csv +import itertools import re from typing import Iterator, Optional +import sqlparse + def _detect_encoding(path: str) -> str: with open(path, "rb") as f: @@ -150,74 +150,77 @@ def _parse_value(tok: str): _INSERT_RE = re.compile( - r"(?is)INSERT\s+(?:\[dbo\]\.)?\[?(?P
\w+)\]?\s*\((?P.*?)\)\s*VALUES\s*(?P.*)$" + r"(?is)INSERT\s+(?:\[dbo\]\.)?\[?(?P
\w+)\]?\s*" + r"\((?P.*?)\)\s*VALUES\s*(?P.*)$" ) -def _balanced(stmt: str) -> bool: - """True if parens are balanced outside single-quoted strings.""" - depth = 0 - in_quote = False - i = 0 - n = len(stmt) - while i < n: - c = stmt[i] - if in_quote: - if c == "'": - if i + 1 < n and stmt[i + 1] == "'": - i += 2 - continue - in_quote = False - elif c == "'": - in_quote = True - elif c == "(": - depth += 1 - elif c == ")": - depth -= 1 - i += 1 - return depth <= 0 and not in_quote +def _iter_insert_statements(path: str, table: str) -> Iterator[str]: + """Yield raw INSERT statement strings for ``table`` using sqlparse.""" + enc = _detect_encoding(path) + target = table.lower() + with open(path, encoding=enc, errors="ignore") as f: + # parsestream splits the dump into statements lazily. + for statement in sqlparse.parsestream(f): + s = str(statement).strip() + if not s: + continue + low = s.lower() + if "insert" not in low or target not in low: + continue + yield s def iter_table_rows(path: str, table: str) -> Iterator[dict]: """Yield ``{column: value}`` dicts for every INSERT into ``table``.""" - enc = _detect_encoding(path) - target = f"[{table}]".lower() - target_plain = table.lower() - pending: Optional[str] = None - - with open(path, encoding=enc, errors="ignore") as f: - for line in f: - if pending is None: - low = line.lower() - if "insert" not in low: - continue - # cheap table filter before the heavier regex - if ( - target not in low - and f"].[{target_plain}]" not in low - and f" {target_plain} " not in low - ): - if target_plain not in low: - continue - pending = line - else: - pending += line - - if not _balanced(pending): - continue # statement spans more lines - - stmt = pending - pending = None - m = _INSERT_RE.search(stmt) - if not m or m.group("table").lower() != target_plain: - continue - cols = [c.strip().strip("[]") for c in _split_top_level(m.group("cols"))] - vals_part = m.group("vals").strip().rstrip(";") - for group in _iter_value_groups(vals_part): - vals = [_parse_value(v) for v in _split_top_level(group)] - if len(vals) != len(cols): - continue # malformed row; skip - yield dict(zip(cols, vals)) + for stmt in _iter_insert_statements(path, table): + m = _INSERT_RE.search(stmt) + if not m or m.group("table").lower() != table.lower(): + continue + cols = [c.strip().strip("[]") for c in _split_top_level(m.group("cols"))] + vals_part = m.group("vals").strip().rstrip(";") + for group in _iter_value_groups(vals_part): + vals = [_parse_value(v) for v in _split_top_level(group)] + if len(vals) != len(cols): + continue # malformed row; skip + yield dict(zip(cols, vals)) + + +def _csv_cell(value) -> str: + """Render a parsed value for a COPY-friendly CSV (None -> empty field).""" + return "" if value is None else str(value) + + +def write_table_csv( + path: str, + table: str, + out_csv: str, + columns: Optional[list[str]] = None, + limit: int = 0, +) -> tuple[int, list[str]]: + """Write one source table's rows to ``out_csv``. Returns (n_rows, header). + + ``columns`` restricts/orders the output columns (e.g. the target model's + columns); missing source values become empty fields. If omitted, the first + row's keys define the header. None -> empty so Postgres COPY ``NULL ''`` + treats it as NULL. + """ + rows = iter_table_rows(path, table) + if limit and limit > 0: + rows = itertools.islice(rows, limit) + + header: Optional[list[str]] = None + writer = None + n = 0 + with open(out_csv, "w", newline="", encoding="utf-8") as fo: + for rec in rows: + if header is None: + header = list(columns) if columns else list(rec.keys()) + writer = csv.writer(fo) + writer.writerow(header) + writer.writerow([_csv_cell(rec.get(c)) for c in header]) + n += 1 + return n, (header or list(columns or [])) # ============= EOF ============================================= diff --git a/uv.lock b/uv.lock index 72a152f8b..9746570db 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]] @@ -1544,6 +1544,7 @@ dependencies = [ { name = "sqlalchemy-continuum" }, { name = "sqlalchemy-searchable" }, { name = "sqlalchemy-utils" }, + { name = "sqlparse" }, { name = "starlette" }, { name = "starlette-admin", extra = ["i18n"] }, { name = "typer" }, @@ -1658,6 +1659,7 @@ requires-dist = [ { name = "sqlalchemy-continuum", specifier = "==1.6.0" }, { name = "sqlalchemy-searchable", specifier = "==2.1.0" }, { name = "sqlalchemy-utils", specifier = "==0.42.1" }, + { name = "sqlparse", specifier = ">=0.5.5" }, { name = "starlette", specifier = "==0.52.1" }, { name = "starlette-admin", extras = ["i18n"], specifier = "==0.16.0" }, { name = "typer", specifier = "==0.24.1" }, @@ -2866,6 +2868,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/25/7400c18c3ee97914cc99c90007795c00a4ec5b60c853b49db7ba24d11179/sqlalchemy_utils-0.42.1-py3-none-any.whl", hash = "sha256:243cfe1b3a1dae3c74118ae633f1d1e0ed8c787387bc33e556e37c990594ac80", size = 91761, upload-time = "2025-12-13T03:14:15.014Z" }, ] +[[package]] +name = "sqlparse" +version = "0.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815, upload-time = "2025-12-19T07:17:45.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, +] + [[package]] name = "starlette" version = "0.52.1" From 84f9e9f84c0a989a9297599a9258fcb6df86922a Mon Sep 17 00:00:00 2001 From: jakeross Date: Sun, 7 Jun 2026 22:04:08 -0600 Subject: [PATCH 19/41] feat(transfers): refresh materialized OGC views after mirror load Add refresh_materialized_views (REFRESH the geothermal materialized views, currently ogc_geothermal_wells_temperature_profile; skip any not present). The transfer_geothermal orchestrator calls it after the NMW_* mirror load so the materialized view reflects the freshly loaded data. Co-Authored-By: Claude Opus 4.8 --- transfers/nmw_mirror_transfer.py | 25 +++++++++++++++++++++++++ transfers/transfer_geothermal.py | 8 +++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 9bb0a961f..d541d0f0e 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -86,6 +86,10 @@ _CSV_DIR_ENV = "NMW_CSV_DIR" _CHUNK_SIZE = 2000 +# Materialized OGC views over the geothermal mirror that need a REFRESH after a +# (re)load. Regular views reflect the tables live and need no refresh. +_MATERIALIZED_VIEWS = ("ogc_geothermal_wells_temperature_profile",) + @dataclass class MirrorSpec: @@ -334,4 +338,25 @@ def transfer_nmw_mirror(session: Session, limit: int = None) -> tuple: return len(loaded), inserted, errors +def refresh_materialized_views(session: Session) -> list[str]: + """REFRESH the geothermal materialized OGC views (skip any not present). + + Call after a mirror (re)load so the materialized views reflect new data. + Plain (non-concurrent) REFRESH — runs inside the session transaction. + """ + refreshed = [] + for view in _MATERIALIZED_VIEWS: + exists = session.execute( + text("SELECT to_regclass(:n)"), {"n": f"public.{view}"} + ).scalar() + if not exists: + logger.warning("Skip refresh; materialized view missing: %s", view) + continue + logger.info("REFRESH MATERIALIZED VIEW %s", view) + session.execute(text(f'REFRESH MATERIALIZED VIEW "{view}"')) + session.commit() + refreshed.append(view) + return refreshed + + # ============= EOF ============================================= diff --git a/transfers/transfer_geothermal.py b/transfers/transfer_geothermal.py index a9d9d0b47..6945ea01d 100644 --- a/transfers/transfer_geothermal.py +++ b/transfers/transfer_geothermal.py @@ -54,7 +54,10 @@ from db.engine import session_ctx # noqa: E402 from services.env import get_bool_env # noqa: E402 from transfers.logger import logger # noqa: E402 -from transfers.nmw_mirror_transfer import transfer_nmw_mirror # noqa: E402 +from transfers.nmw_mirror_transfer import ( # noqa: E402 + refresh_materialized_views, + transfer_nmw_mirror, +) from transfers.reference_lexicon_transfer import transfer_reference_tables # noqa: E402 @@ -87,6 +90,9 @@ def run_geothermal_transfer(limit: int = None) -> dict: "rows_inserted": inserted, "errors": len(errors), } + logger.info("---- Refresh materialized OGC views ----") + with session_ctx() as session: + summary["refreshed_views"] = refresh_materialized_views(session) else: logger.info("Skipping NM_Wells mirror (TRANSFER_NMW_MIRROR=0)") From d24667eec36d7435e3155ea9b98f59d08107787e Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 8 Jun 2026 11:14:56 -0600 Subject: [PATCH 20/41] chore: regenerate requirements.txt with sqlparse after staging merge Co-Authored-By: Claude Opus 4.8 --- requirements.txt | 1015 +++++++++------------------------------------- 1 file changed, 192 insertions(+), 823 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0335fdcbd..23420654a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,81 +17,45 @@ aiohappyeyeballs==2.6.2 \ # aiohttp # ocotilloapi aiohttp==3.14.1 \ - --hash=sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5 \ --hash=sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983 \ - --hash=sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521 \ --hash=sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340 \ --hash=sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d \ --hash=sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a \ --hash=sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4 \ - --hash=sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a \ - --hash=sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f \ - --hash=sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee \ --hash=sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8 \ - --hash=sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb \ --hash=sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397 \ - --hash=sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05 \ --hash=sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8 \ --hash=sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09 \ - --hash=sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2 \ --hash=sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba \ --hash=sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf \ - --hash=sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271 \ --hash=sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5 \ - --hash=sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847 \ - --hash=sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264 \ - --hash=sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf \ - --hash=sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6 \ - --hash=sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df \ --hash=sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035 \ - --hash=sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126 \ --hash=sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6 \ --hash=sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35 \ - --hash=sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4 \ --hash=sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333 \ --hash=sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203 \ - --hash=sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c \ --hash=sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1 \ --hash=sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251 \ --hash=sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365 \ - --hash=sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b \ - --hash=sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621 \ --hash=sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94 \ --hash=sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da \ - --hash=sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491 \ --hash=sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe \ - --hash=sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d \ --hash=sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080 \ - --hash=sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42 \ - --hash=sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c \ --hash=sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397 \ --hash=sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9 \ - --hash=sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8 \ --hash=sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345 \ --hash=sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3 \ --hash=sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602 \ - --hash=sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2 \ - --hash=sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966 \ - --hash=sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192 \ - --hash=sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95 \ - --hash=sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3 \ - --hash=sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b \ --hash=sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444 \ - --hash=sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6 \ - --hash=sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573 \ - --hash=sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af \ --hash=sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15 \ --hash=sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe \ --hash=sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2 \ --hash=sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496 \ --hash=sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876 \ - --hash=sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817 \ --hash=sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448 \ - --hash=sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e \ --hash=sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6 \ --hash=sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd \ --hash=sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f \ - --hash=sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe \ --hash=sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c \ --hash=sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca \ --hash=sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c \ @@ -99,41 +63,23 @@ aiohttp==3.14.1 \ --hash=sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc \ --hash=sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0 \ --hash=sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0 \ - --hash=sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2 \ --hash=sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844 \ --hash=sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719 \ --hash=sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1 \ - --hash=sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3 \ --hash=sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178 \ - --hash=sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3 \ --hash=sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95 \ - --hash=sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730 \ - --hash=sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842 \ - --hash=sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd \ - --hash=sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d \ --hash=sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96 \ - --hash=sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85 \ --hash=sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1 \ - --hash=sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199 \ --hash=sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a \ --hash=sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588 \ --hash=sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec \ --hash=sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004 \ - --hash=sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480 \ - --hash=sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04 \ - --hash=sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8 \ - --hash=sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce \ - --hash=sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087 \ - --hash=sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505 \ --hash=sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780 \ - --hash=sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4 \ --hash=sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d \ --hash=sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca \ --hash=sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665 \ --hash=sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296 \ --hash=sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c \ - --hash=sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a \ - --hash=sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7 \ --hash=sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451 \ --hash=sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3 # via @@ -226,9 +172,9 @@ authlib==1.7.2 \ --hash=sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231 \ --hash=sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f # via ocotilloapi -babel==2.18.0 \ - --hash=sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d \ - --hash=sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35 +babel==2.17.0 \ + --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \ + --hash=sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2 # via # pygeoapi # starlette-admin @@ -305,123 +251,51 @@ cffi==2.0.0 \ --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ - --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ - --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ - --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ - --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ - --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ - --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ - --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ - --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ - --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ - --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ - --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ - --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ - --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ - --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ - --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ - --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ - --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ - --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ - --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ - --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ - --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ - --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ - --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ - --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ - --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ - --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ - --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ - --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ - --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ - --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ - --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ - --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ - --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ - --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ - --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ - --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ - --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ - --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ - --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ - --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ - --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ - --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ - --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ - --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ - --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ - --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ - --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ - --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ - --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ - --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf + --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 # via # cryptography # ocotilloapi -cfgv==3.5.0 \ - --hash=sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0 \ - --hash=sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132 +cfgv==3.4.0 \ + --hash=sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9 \ + --hash=sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560 # via pre-commit charset-normalizer==3.4.7 \ - --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ - --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ - --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ - --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ - --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ - --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ - --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ - --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ - --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ - --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ - --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ - --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ - --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ - --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ - --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ - --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ - --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ - --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ - --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ - --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ - --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ - --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ - --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ @@ -429,99 +303,43 @@ charset-normalizer==3.4.7 \ --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ - --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ - --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ - --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ - --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ - --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ - --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ - --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ - --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ - --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ - --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ - --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ - --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ - --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ - --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ - --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ - --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ - --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ - --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ - --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ - --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ - --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ - --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ - --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ - --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ - --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ - --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ - --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ - --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ - --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ - --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ - --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ - --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ - --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ - --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ - --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ - --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ - --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ - --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ - --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ - --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ - --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ - --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ - --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ - --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ - --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ - --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ - --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ - --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ - --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ - --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ - --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ - --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ - --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ - --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ - --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ - --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ - --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 # via # ocotilloapi # requests @@ -535,7 +353,6 @@ click==8.4.1 \ # pygeoapi # pygeofilter # rasterio - # typer # uvicorn cligj==0.7.2 \ --hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \ @@ -551,123 +368,61 @@ colorama==0.4.6 ; sys_platform == 'win32' \ # via # click # pytest -coverage==7.14.1 \ - --hash=sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86 \ - --hash=sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd \ - --hash=sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d \ - --hash=sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5 \ - --hash=sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42 \ - --hash=sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de \ - --hash=sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548 \ - --hash=sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1 \ - --hash=sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7 \ - --hash=sha256:1238cb94638e610e972c60dac68e813f868dc7d6e982535270558443058d9d59 \ - --hash=sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906 \ - --hash=sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af \ - --hash=sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1 \ - --hash=sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d \ - --hash=sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1 \ - --hash=sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be \ - --hash=sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02 \ - --hash=sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42 \ - --hash=sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129 \ - --hash=sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e \ - --hash=sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be \ - --hash=sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e \ - --hash=sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65 \ - --hash=sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54 \ - --hash=sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1 \ - --hash=sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5 \ - --hash=sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df \ - --hash=sha256:3c18ebc343e15be53049b3a2dce38fe82d58f37e20ab9094b3a39c0aa4f6bb47 \ - --hash=sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f \ - --hash=sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf \ - --hash=sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37 \ - --hash=sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4 \ - --hash=sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f \ - --hash=sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84 \ - --hash=sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1 \ - --hash=sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c \ - --hash=sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8 \ - --hash=sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e \ - --hash=sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec \ - --hash=sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d \ - --hash=sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54 \ - --hash=sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890 \ - --hash=sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b \ - --hash=sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d \ - --hash=sha256:62a9f70b52e0b5a95cfef4a5c5641b06983cadc5e538a3feeb5c00211f523ac2 \ - --hash=sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33 \ - --hash=sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9 \ - --hash=sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e \ - --hash=sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6 \ - --hash=sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce \ - --hash=sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247 \ - --hash=sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901 \ - --hash=sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36 \ - --hash=sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69 \ - --hash=sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416 \ - --hash=sha256:7f02d09f70776579b926d889a4c9c235070a1f47c40458aeaca563fae5acfdb5 \ - --hash=sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500 \ - --hash=sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad \ - --hash=sha256:84ac9499e48700399a5dd0ea7085b5091961fec52c68d66b4ec0d3cf7f4441b1 \ - --hash=sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b \ - --hash=sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b \ - --hash=sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a \ - --hash=sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e \ - --hash=sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee \ - --hash=sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07 \ - --hash=sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a \ - --hash=sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d \ - --hash=sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c \ - --hash=sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343 \ - --hash=sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4 \ - --hash=sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2 \ - --hash=sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8 \ - --hash=sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf \ - --hash=sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb \ - --hash=sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c \ - --hash=sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff \ - --hash=sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e \ - --hash=sha256:b84ffdf877644e7096aa936991efeed873f7f3df57b9cd001312b7668ab08550 \ - --hash=sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860 \ - --hash=sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793 \ - --hash=sha256:c643734307300234fafa36bf2a040a7235f8f177ea1fd6ec1423aea6fb7b929f \ - --hash=sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851 \ - --hash=sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7 \ - --hash=sha256:c912c259304cfb5ee584481cfb7ce1ff932b4d61e6c9140b8f19cb7b5ed82332 \ - --hash=sha256:ce66d8e46da2bb5ee313a745cbd2e391d319176c1f7a9451bfcd3a2fb920859b \ - --hash=sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2 \ - --hash=sha256:cfe5a5fec635799ef33428f1e5e61bafa45a92a96190ba731561ba558ccc214d \ - --hash=sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a \ - --hash=sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef \ - --hash=sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474 \ - --hash=sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee \ - --hash=sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43 \ - --hash=sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034 \ - --hash=sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3 \ - --hash=sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c \ - --hash=sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d \ - --hash=sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7 \ - --hash=sha256:e854312c4103f2ad4c0dc023b69b77ebfd2c89db5f86c4c94dc2353f9a92167e \ - --hash=sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d \ - --hash=sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4 \ - --hash=sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9 \ - --hash=sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52 \ - --hash=sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a \ - --hash=sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c \ - --hash=sha256:fc459e5d73be2d6332fcfe8dbf3d8994671fe33c700f4565988ecfa511547253 \ - --hash=sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c + # typer +coverage==7.10.2 \ + --hash=sha256:0100b19f230df72c90fdb36db59d3f39232391e8d89616a7de30f677da4f532b \ + --hash=sha256:04c74f9ef1f925456a9fd23a7eef1103126186d0500ef9a0acb0bd2514bdc7cc \ + --hash=sha256:11333094c1bff621aa811b67ed794865cbcaa99984dedea4bd9cf780ad64ecba \ + --hash=sha256:12e52b5aa00aa720097d6947d2eb9e404e7c1101ad775f9661ba165ed0a28303 \ + --hash=sha256:14fb5b6641ab5b3c4161572579f0f2ea8834f9d3af2f7dd8fbaecd58ef9175cc \ + --hash=sha256:1a2e934e9da26341d342d30bfe91422bbfdb3f1f069ec87f19b2909d10d8dcc4 \ + --hash=sha256:228946da741558904e2c03ce870ba5efd9cd6e48cbc004d9a27abee08100a15a \ + --hash=sha256:248b5394718e10d067354448dc406d651709c6765669679311170da18e0e9af8 \ + --hash=sha256:2d358f259d8019d4ef25d8c5b78aca4c7af25e28bd4231312911c22a0e824a57 \ + --hash=sha256:2e980e4179f33d9b65ac4acb86c9c0dde904098853f27f289766657ed16e07b3 \ + --hash=sha256:5250bda76e30382e0a2dcd68d961afcab92c3a7613606e6269855c6979a1b0bb \ + --hash=sha256:52d708b5fd65589461381fa442d9905f5903d76c086c6a4108e8e9efdca7a7ed \ + --hash=sha256:5b9d538e8e04916a5df63052d698b30c74eb0174f2ca9cd942c981f274a18eaf \ + --hash=sha256:5c61675a922b569137cf943770d7ad3edd0202d992ce53ac328c5ff68213ccf4 \ + --hash=sha256:5d6e6d84e6dd31a8ded64759626627247d676a23c1b892e1326f7c55c8d61055 \ + --hash=sha256:651015dcd5fd9b5a51ca79ece60d353cacc5beaf304db750407b29c89f72fe2b \ + --hash=sha256:65b451949cb789c346f9f9002441fc934d8ccedcc9ec09daabc2139ad13853f7 \ + --hash=sha256:6eb586fa7d2aee8d65d5ae1dd71414020b2f447435c57ee8de8abea0a77d5074 \ + --hash=sha256:718044729bf1fe3e9eb9f31b52e44ddae07e434ec050c8c628bf5adc56fe4bdd \ + --hash=sha256:71d40b3ac0f26fa9ffa6ee16219a714fed5c6ec197cdcd2018904ab5e75bcfa3 \ + --hash=sha256:75cc1a3f8c88c69bf16a871dab1fe5a7303fdb1e9f285f204b60f1ee539b8fc0 \ + --hash=sha256:81bf6a32212f9f66da03d63ecb9cd9bd48e662050a937db7199dbf47d19831de \ + --hash=sha256:835f39e618099325e7612b3406f57af30ab0a0af350490eff6421e2e5f608e46 \ + --hash=sha256:8f34b09f68bdadec122ffad312154eda965ade433559cc1eadd96cca3de5c824 \ + --hash=sha256:916369b3b914186b2c5e5ad2f7264b02cff5df96cdd7cdad65dccd39aa5fd9f0 \ + --hash=sha256:95db3750dd2e6e93d99fa2498f3a1580581e49c494bddccc6f85c5c21604921f \ + --hash=sha256:95e23987b52d02e7c413bf2d6dc6288bd5721beb518052109a13bfdc62c8033b \ + --hash=sha256:96e5921342574a14303dfdb73de0019e1ac041c863743c8fe1aa6c2b4a257226 \ + --hash=sha256:9c1cd71483ea78331bdfadb8dcec4f4edfb73c7002c1206d8e0af6797853f5be \ + --hash=sha256:9f75dbf4899e29a37d74f48342f29279391668ef625fdac6d2f67363518056a1 \ + --hash=sha256:a3e853cc04987c85ec410905667eed4bf08b1d84d80dfab2684bb250ac8da4f6 \ + --hash=sha256:a7df481e7508de1c38b9b8043da48d94931aefa3e32b47dd20277e4978ed5b95 \ + --hash=sha256:a91e027d66eff214d88d9afbe528e21c9ef1ecdf4956c46e366c50f3094696d0 \ + --hash=sha256:abb57fdd38bf6f7dcc66b38dafb7af7c5fdc31ac6029ce373a6f7f5331d6f60f \ + --hash=sha256:aca7b5645afa688de6d4f8e89d30c577f62956fefb1bad021490d63173874186 \ + --hash=sha256:c2e117e64c26300032755d4520cd769f2623cde1a1d1c3515b05a3b8add0ade1 \ + --hash=sha256:ca07fa78cc9d26bc8c4740de1abd3489cf9c47cc06d9a8ab3d552ff5101af4c0 \ + --hash=sha256:d800705f6951f75a905ea6feb03fff8f3ea3468b81e7563373ddc29aa3e5d1ca \ + --hash=sha256:daaf98009977f577b71f8800208f4d40d4dcf5c2db53d4d822787cdc198d76e1 \ + --hash=sha256:e8415918856a3e7d57a4e0ad94651b761317de459eb74d34cc1bb51aad80f07e \ + --hash=sha256:e96649ac34a3d0e6491e82a2af71098e43be2874b619547c3282fc11d3840a4b \ + --hash=sha256:ea8d8fe546c528535c761ba424410bbeb36ba8a0f24be653e94b70c93fd8a8ca \ + --hash=sha256:f256173b48cc68486299d510a3e729a96e62c889703807482dbf56946befb5c8 \ + --hash=sha256:f287a25a8ca53901c613498e4a40885b19361a2fe8fbfdbb7f8ef2cad2a23f03 \ + --hash=sha256:f35481d42c6d146d48ec92d4e239c23f97b53a3f1fbd2302e7c64336f28641fe \ + --hash=sha256:fe024d40ac31eb8d5aae70215b41dafa264676caa4404ae155f77d2fa95c37bb # via pytest-cov cryptography==46.0.7 \ - --hash=sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65 \ --hash=sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832 \ --hash=sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067 \ --hash=sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de \ - --hash=sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4 \ --hash=sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0 \ --hash=sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b \ - --hash=sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968 \ --hash=sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef \ --hash=sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b \ --hash=sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4 \ @@ -693,10 +448,8 @@ cryptography==46.0.7 \ --hash=sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1 \ --hash=sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2 \ --hash=sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0 \ - --hash=sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455 \ --hash=sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842 \ --hash=sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457 \ - --hash=sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15 \ --hash=sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2 \ --hash=sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c \ --hash=sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb \ @@ -705,7 +458,6 @@ cryptography==46.0.7 \ --hash=sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902 \ --hash=sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246 \ --hash=sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022 \ - --hash=sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f \ --hash=sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e \ --hash=sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298 \ --hash=sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce @@ -713,14 +465,15 @@ cryptography==46.0.7 \ # authlib # cloud-sql-python-connector # google-auth + # joserfc # ocotilloapi -dateparser==1.4.0 \ - --hash=sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378 \ - --hash=sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4 +dateparser==1.3.0 \ + --hash=sha256:5bccf5d1ec6785e5be71cc7ec80f014575a09b4923e762f850e57443bddbf1a5 \ + --hash=sha256:8dc678b0a526e103379f02ae44337d424bd366aac727d3c6cf52ce1b01efbb5a # via pygeofilter -distlib==0.4.1 \ - --hash=sha256:9c2c552c68cbadc619f2d0ed3a69e27c351a3f4c9baa9ffb7df9e9cdc3d19a97 \ - --hash=sha256:c3804d0d2d4b5fcd44036eb860cb6660485fcdf5c2aba53dc324d805837ea65b +distlib==0.4.0 \ + --hash=sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 \ + --hash=sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d # via virtualenv dnspython==2.8.0 \ --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af \ @@ -732,9 +485,9 @@ dnspython==2.8.0 \ dotenv==0.9.9 \ --hash=sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9 # via ocotilloapi -ecdsa==0.19.2 \ - --hash=sha256:62635b0ac1ca2e027f82122b5b81cb706edc38cd91c63dda28e4f3455a2bf930 \ - --hash=sha256:840f5dc5e375c68f36c1a7a5b9caad28f95daa65185c9253c0c08dd952bb7399 +ecdsa==0.19.1 \ + --hash=sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3 \ + --hash=sha256:478cba7b62555866fcb3bb3fe985e06decbdb68ef55713c4e5ab98c57d508e61 # via python-jose email-validator==2.3.0 \ --hash=sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4 \ @@ -752,9 +505,9 @@ fastapi-pagination==0.15.14 \ --hash=sha256:61209b30172f928887a2537a85d144a2ae970edfadf160aab7c1fb15676dd651 \ --hash=sha256:b1c2ae46ae9952199f75d07726e3f11909ecd32bf12701a11f3e1080f05c4e91 # via ocotilloapi -filelock==3.29.1 \ - --hash=sha256:85199dfd706869641b72b2e8955d5416a4b2b7dc4b0e8e6d97b4cc1299a6983b \ - --hash=sha256:d97e6b1b9757569626c58caa07dc4beb1613f4a2938b1e8cc81afca398906c9e +filelock==3.18.0 \ + --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \ + --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de # via # pygeoapi # virtualenv @@ -892,11 +645,7 @@ googleapis-common-protos==1.75.0 \ # google-api-core # ocotilloapi greenlet==3.5.1 \ - --hash=sha256:001775efe7b8e758861294c7a27c28af87f3f3f1c20468a2bc618c45b346c061 \ - --hash=sha256:00929c98ec525fd9bf075875d8c5f6a983a90906cdf78a66e6de2d8e466c2a19 \ - --hash=sha256:017a544f0385d441e88714160d089d6900ef46c9eff9d99b6715a5ef2d127747 \ --hash=sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1 \ - --hash=sha256:1072b4f9edcc1e192d9283a66a3e68d6b84c561de33a83d7858beb9ba1effe10 \ --hash=sha256:10a9a1c0bfbc93d41156ffcb90c75fbc05544054faf15dcc1fdf9765f8b607f0 \ --hash=sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9 \ --hash=sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64 \ @@ -906,71 +655,46 @@ greenlet==3.5.1 \ --hash=sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e \ --hash=sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c \ --hash=sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d \ - --hash=sha256:3c8bb982ad117d29478ef8f5533e97df21f1e2befd17a299257b0c96d1371c0b \ --hash=sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986 \ --hash=sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78 \ --hash=sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2 \ --hash=sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a \ --hash=sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc \ --hash=sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b \ - --hash=sha256:540dae7b956209af4d70a3be35927b4055f617763771e5e84a5255bea934d2f5 \ --hash=sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829 \ --hash=sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea \ --hash=sha256:67821bb03e4e98664490edb787ff6af501194c29bbee0f5c1dfdcf1dc3d9d436 \ --hash=sha256:6c09df69dc1712d131332054a858a3e5cca400967fa3a672e2324fbb0971448c \ - --hash=sha256:6ebeb75c81211f5c702576cf81f315e77e23cfdb2c7c6fcb9dd143e6de35c360 \ - --hash=sha256:73f78f9b9f0a5c06e5c946ba1e8e36f5114923b6be109ee618c54f079c3ea14f \ --hash=sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244 \ --hash=sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283 \ --hash=sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54 \ - --hash=sha256:7eacb17a9d41538a2bc4912eba5ef13823c83cb69e4d141d0813debe7163187f \ --hash=sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2 \ - --hash=sha256:80eb4b04dadc4e67df3fae179a32c4706a3f495bc7f22fc8a81115d5f5512188 \ - --hash=sha256:88e300d136eac057b2397aa1cfd7328b4c87c7eb66a09c7bc6a1292234db474e \ - --hash=sha256:89101bfd5011e069be974903cb3a4e4523845e4ece2d62dcd8d358933c0ef249 \ --hash=sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3 \ - --hash=sha256:8a271fcd66c74615cda6a964fda3f304267a12e50a084472218a39bb0376f563 \ --hash=sha256:8d8a23250ea3ec7b36de8fa4b541e9e2db3ee82915cc060ab0631609ad8b28de \ --hash=sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6 \ --hash=sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368 \ --hash=sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26 \ --hash=sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de \ - --hash=sha256:9d59e840387076a51016777a9328b3f2c427c6f9208a6e958bad251be50a648d \ - --hash=sha256:a0cbed8bb44e23c5b199f888f4e4ce096b45ad9f25ff74a7ad0213875e936bb2 \ --hash=sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e \ - --hash=sha256:a203a8bd0acb0701653d3bbb26e404854a68674139ed5cbb778830f42b09bb33 \ --hash=sha256:a4764e0bfc6a4d114c865b32520805c16a990ef5f286a514413b05d5ecd6a23d \ --hash=sha256:a57b0d05a0448eed231d59c0ceb287dde984551e54cbc51ac2d4865712838e9c \ --hash=sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd \ - --hash=sha256:a5ea42a752d47a145eae922b605cd1634665ac3d5ec1e72402d5048e8d60d207 \ --hash=sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed \ - --hash=sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b \ --hash=sha256:b0703c2cef53e01baec47f7a3868009913ad71ec678bbecb42a6f40895e4ce62 \ - --hash=sha256:b9152fca4a6466e114aaec745ae61cba739903a109754a9d4e1262f01e9259b1 \ --hash=sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0 \ - --hash=sha256:c3d35f87c7253b715d13d679e0783d845910144f282cb939fe1ba4ac8616269c \ - --hash=sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823 \ --hash=sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab \ - --hash=sha256:cc6ab7e555c8a112ad3a76e368e86e12a2754bcae1652a5602e133ec7b635523 \ --hash=sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd \ - --hash=sha256:d0932b81d72f552ded9d810d00021b64d89f2195a91ce115b893f943b7a4ab3c \ --hash=sha256:d40a890035c0058cadbdc4af7569800fd28a0e527a0fdbb7b5f9418f176846ce \ --hash=sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c \ --hash=sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07 \ --hash=sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135 \ --hash=sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e \ - --hash=sha256:ded7b068c7c31c1a8657d4fd42d886b3e051ae29f88b80c5ff9d502257b0f071 \ - --hash=sha256:e5cc9606aa5f4e0bde0d3bd502b44f743864c3ffa5cfa1011b1e30f5aa02366f \ --hash=sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5 \ - --hash=sha256:e6cd99ea59dd5d89f0c956606571d79bfe6f68c9eb7f4a4083a41a7f1587edee \ --hash=sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e \ --hash=sha256:ea37d5a157eb9493820d3792ac4ece28619a394391d2b9f2f78057d396ff0f0f \ --hash=sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad \ - --hash=sha256:ed8cdb691169715a9a492844a83246f090182247d1a5031dc78a403f68ba1e97 \ --hash=sha256:ef08c1567c78074b22d1a200183d52d04a14df447bf70bcbb6a3507a48e776fc \ - --hash=sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e \ - --hash=sha256:fa4f98af3a528f0c3fd592a26df7f376f93329c8f4d987f6bb979057af8bf5e2 \ - --hash=sha256:ffea73584b216150eab159b6d12348fb253e68757974de1e2c40d8a318ac89ed + --hash=sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e # via # ocotilloapi # sqlalchemy @@ -997,9 +721,9 @@ httpx==0.28.1 \ # via # apitally # ocotilloapi -identify==2.6.19 \ - --hash=sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a \ - --hash=sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842 +identify==2.6.12 \ + --hash=sha256:ad9672d5a72e0d2ff7c5c8809b62dfa60458626352fb0eb7b55e69bdc45334a2 \ + --hash=sha256:d8de45749f1efb108badef65ee8386f0f7bb19a7f26185f74de6367bffbaf0e6 # via pre-commit idna==3.18 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ @@ -1011,9 +735,9 @@ idna==3.18 \ # ocotilloapi # requests # yarl -importlib-metadata==9.0.0 \ - --hash=sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7 \ - --hash=sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc +importlib-metadata==8.7.1 \ + --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ + --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 # via opentelemetry-api iniconfig==2.3.0 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ @@ -1035,6 +759,10 @@ jinja2==3.1.6 \ # ocotilloapi # pygeoapi # starlette-admin +joserfc==1.7.1 \ + --hash=sha256:77d0b76514879c68c6f433bc5b7357a4ab72008ff1e33d8379fd11d72bd8ca81 \ + --hash=sha256:b3e3d655612e2e1ef67b2600f2f420e12e537b020208fab1761fad647319c164 + # via authlib jsonschema==4.26.0 \ --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce @@ -1053,9 +781,9 @@ mako==1.3.12 \ # via # alembic # ocotilloapi -markdown-it-py==4.2.0 \ - --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ - --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +markdown-it-py==4.0.0 \ + --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \ + --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 # via rich markupsafe==3.0.3 \ --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ @@ -1192,53 +920,34 @@ multidict==6.7.1 \ # aiohttp # ocotilloapi # yarl -nodeenv==1.10.0 \ - --hash=sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827 \ - --hash=sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb +nodeenv==1.9.1 \ + --hash=sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f \ + --hash=sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9 # via pre-commit numpy==2.4.6 \ - --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ - --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ - --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ - --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ - --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ - --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ - --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ - --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ - --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ - --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ - --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ - --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ - --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ - --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ - --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ - --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ - --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ - --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ - --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ @@ -1248,26 +957,16 @@ numpy==2.4.6 \ --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ - --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ - --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ - --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ - --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ - --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ - --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ - --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ - --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ - --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ - --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 # via # ocotilloapi @@ -1275,19 +974,19 @@ numpy==2.4.6 \ # pandas-stubs # rasterio # shapely -opentelemetry-api==1.42.1 \ - --hash=sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714 \ - --hash=sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716 +opentelemetry-api==1.39.1 \ + --hash=sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950 \ + --hash=sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c # via # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-sdk==1.42.1 \ - --hash=sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d \ - --hash=sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7 +opentelemetry-sdk==1.39.1 \ + --hash=sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c \ + --hash=sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6 # via apitally -opentelemetry-semantic-conventions==0.63b1 \ - --hash=sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9 \ - --hash=sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682 +opentelemetry-semantic-conventions==0.60b1 \ + --hash=sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953 \ + --hash=sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb # via opentelemetry-sdk packaging==26.2 \ --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ @@ -1327,100 +1026,60 @@ phonenumbers==9.0.32 \ # via ocotilloapi pillow==12.2.0 \ --hash=sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9 \ - --hash=sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5 \ - --hash=sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987 \ --hash=sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9 \ --hash=sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b \ - --hash=sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f \ --hash=sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd \ - --hash=sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e \ --hash=sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e \ --hash=sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe \ --hash=sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795 \ --hash=sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601 \ - --hash=sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1 \ --hash=sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed \ --hash=sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea \ - --hash=sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5 \ - --hash=sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97 \ --hash=sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453 \ --hash=sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98 \ - --hash=sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa \ --hash=sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b \ - --hash=sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d \ - --hash=sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705 \ --hash=sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8 \ - --hash=sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024 \ - --hash=sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0 \ --hash=sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286 \ --hash=sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150 \ --hash=sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2 \ - --hash=sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3 \ - --hash=sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b \ --hash=sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f \ --hash=sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463 \ - --hash=sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940 \ --hash=sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166 \ --hash=sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed \ - --hash=sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f \ --hash=sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795 \ - --hash=sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780 \ --hash=sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7 \ --hash=sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1 \ - --hash=sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5 \ --hash=sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295 \ --hash=sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b \ --hash=sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354 \ - --hash=sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60 \ - --hash=sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65 \ - --hash=sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005 \ --hash=sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c \ --hash=sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be \ - --hash=sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5 \ --hash=sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06 \ --hash=sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae \ --hash=sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c \ - --hash=sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c \ --hash=sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612 \ - --hash=sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e \ - --hash=sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab \ - --hash=sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808 \ --hash=sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f \ --hash=sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e \ - --hash=sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909 \ - --hash=sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec \ - --hash=sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe \ --hash=sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50 \ --hash=sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4 \ - --hash=sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f \ - --hash=sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff \ --hash=sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5 \ --hash=sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb \ - --hash=sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414 \ --hash=sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1 \ - --hash=sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032 \ - --hash=sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76 \ - --hash=sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136 \ - --hash=sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e \ --hash=sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c \ --hash=sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3 \ --hash=sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea \ --hash=sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f \ --hash=sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104 \ - --hash=sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176 \ --hash=sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24 \ --hash=sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3 \ --hash=sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4 \ --hash=sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed \ --hash=sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43 \ - --hash=sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421 \ - --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \ - --hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \ - --hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5 + --hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 # via ocotilloapi -platformdirs==4.10.0 \ - --hash=sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7 \ - --hash=sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a +platformdirs==4.3.8 \ + --hash=sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc \ + --hash=sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 # via virtualenv pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ @@ -1435,99 +1094,55 @@ pre-commit==4.6.0 \ # via ocotilloapi propcache==0.5.2 \ --hash=sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427 \ - --hash=sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5 \ --hash=sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa \ --hash=sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7 \ --hash=sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a \ --hash=sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0 \ - --hash=sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660 \ --hash=sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94 \ --hash=sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917 \ - --hash=sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42 \ --hash=sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3 \ --hash=sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa \ - --hash=sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d \ - --hash=sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33 \ --hash=sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a \ - --hash=sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511 \ - --hash=sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0 \ - --hash=sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84 \ - --hash=sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c \ - --hash=sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66 \ --hash=sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821 \ --hash=sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb \ - --hash=sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e \ --hash=sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853 \ --hash=sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56 \ --hash=sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55 \ - --hash=sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6 \ --hash=sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704 \ - --hash=sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82 \ --hash=sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f \ - --hash=sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64 \ --hash=sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999 \ - --hash=sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b \ --hash=sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb \ --hash=sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d \ --hash=sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4 \ - --hash=sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab \ - --hash=sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f \ --hash=sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03 \ --hash=sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5 \ - --hash=sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba \ --hash=sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979 \ - --hash=sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b \ - --hash=sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144 \ - --hash=sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d \ --hash=sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e \ - --hash=sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67 \ --hash=sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117 \ --hash=sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa \ - --hash=sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb \ --hash=sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96 \ --hash=sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5 \ - --hash=sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 \ --hash=sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191 \ - --hash=sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78 \ --hash=sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078 \ --hash=sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837 \ - --hash=sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a \ - --hash=sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba \ - --hash=sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe \ --hash=sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c \ - --hash=sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf \ - --hash=sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c \ - --hash=sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9 \ --hash=sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8 \ --hash=sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe \ - --hash=sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031 \ - --hash=sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913 \ --hash=sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d \ - --hash=sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf \ - --hash=sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f \ --hash=sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539 \ - --hash=sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b \ --hash=sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285 \ - --hash=sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959 \ --hash=sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d \ --hash=sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4 \ --hash=sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f \ --hash=sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836 \ - --hash=sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274 \ - --hash=sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d \ --hash=sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f \ --hash=sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e \ --hash=sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe \ - --hash=sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1 \ --hash=sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a \ --hash=sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39 \ - --hash=sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7 \ - --hash=sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a \ --hash=sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164 \ - --hash=sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e \ --hash=sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2 \ --hash=sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0 \ - --hash=sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0 \ --hash=sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335 \ --hash=sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568 \ --hash=sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4 \ @@ -1535,15 +1150,8 @@ propcache==0.5.2 \ --hash=sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2 \ --hash=sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370 \ --hash=sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4 \ - --hash=sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b \ - --hash=sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42 \ - --hash=sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a \ - --hash=sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e \ --hash=sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757 \ --hash=sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825 \ - --hash=sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0 \ - --hash=sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27 \ - --hash=sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf \ --hash=sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f \ --hash=sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d \ --hash=sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366 \ @@ -1604,71 +1212,27 @@ psutil==7.2.2 \ # via apitally psycopg2-binary==2.9.12 \ --hash=sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f \ - --hash=sha256:049366c6d884bdcd65d66e6ca1fdbebe670b56c6c9ba46f164e6667e90881964 \ - --hash=sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c \ --hash=sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2 \ - --hash=sha256:127467c6e476dd876634f17c3d870530e73ff454ff99bff73d36e80af28e1115 \ --hash=sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c \ - --hash=sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be \ - --hash=sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6 \ - --hash=sha256:36512911ebb2b60a0c3e44d0bb5048c1980aced91235d133b7874f3d1d93487c \ - --hash=sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c \ - --hash=sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c \ - --hash=sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d \ --hash=sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019 \ --hash=sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7 \ - --hash=sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3 \ --hash=sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777 \ --hash=sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd \ --hash=sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5 \ - --hash=sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39 \ --hash=sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c \ - --hash=sha256:5c7cb4cbf894a1d36c720d713de507952c7c58f66d30834708f03dbe5c822ccf \ - --hash=sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b \ - --hash=sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433 \ - --hash=sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d \ - --hash=sha256:63a3ebbd543d3d1eda088ac99164e8c5bac15293ee91f20281fd17d050aee1c4 \ --hash=sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290 \ --hash=sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2 \ - --hash=sha256:6f9cae1f848779b5b01f417e762c40d026ea93eb0648249a604728cda991dde3 \ - --hash=sha256:718e1fc18edf573b02cb8aea868de8d8d33f99ce9620206aa9144b67b0985e94 \ --hash=sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d \ - --hash=sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b \ --hash=sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965 \ --hash=sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9 \ --hash=sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f \ --hash=sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354 \ - --hash=sha256:864c261b3690e1207d14bbfe0a61e27567981b80c47a778561e49f676f7ce433 \ - --hash=sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9 \ - --hash=sha256:8ffdb59fe88f99589e34354a130217aa1fd2d615612402d6edc8b3dbc7a44463 \ --hash=sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5 \ --hash=sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be \ - --hash=sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580 \ - --hash=sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4 \ - --hash=sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f \ - --hash=sha256:a46fe069b65255df410f856d842bc235f90e22ffdf532dda625fd4213d3fd9b1 \ - --hash=sha256:a7e39a65b7d2a20e4ba2e0aaad1960b61cc2888d6ab047769f8347bd3c9ad915 \ --hash=sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033 \ - --hash=sha256:ab29414b25dcb698bf26bf213e3348abdcd07bbd5de032a5bec15bd75b298b03 \ - --hash=sha256:ace94261f43850e9e79f6c56636c5e0147978ab79eda5e5e5ebf13ae146fc8fe \ - --hash=sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326 \ --hash=sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0 \ - --hash=sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e \ - --hash=sha256:ba3df2fc42a1cfa45b72cf096d4acb2b885937eedc61461081d53538d4a82a86 \ --hash=sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5 \ - --hash=sha256:c5ee5213445dd45312459029b8c4c0a695461eb517b753d2582315bd07995f5e \ - --hash=sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06 \ - --hash=sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936 \ - --hash=sha256:cfa2517c94ea3af6deb46f81e1bbd884faa63e28481eb2f889989dd8d95e5f03 \ - --hash=sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56 \ - --hash=sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6 \ - --hash=sha256:d6fcbba8c9fed08a73b8ac61ea79e4821e45b1e92bb466230c5e746bbf3d5256 \ - --hash=sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8 \ - --hash=sha256:ee2d84ef5eb6c04702d2e9c372ad557fb027f26a5d82804f749dfb14c7fdd2ab \ --hash=sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980 \ - --hash=sha256:f625abb7020e4af3432d95342daa1aa0db3fa369eed19807aa596367ba791b10 \ - --hash=sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a \ - --hash=sha256:fb1828cf3da68f99e45ebce1355d65d2d12b6a78fb5dfb16247aad6bdef5f5d2 \ --hash=sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e # via ocotilloapi pyasn1==0.6.3 \ @@ -1750,9 +1314,9 @@ pygeoapi==0.23.4 \ --hash=sha256:7f0fd854575a0da049b64907b56fc0f77ab97768414c1397897e60a0e563438d \ --hash=sha256:935a22761eb0d8736f7b0f2c8384672f5341577509803e35f33f6e78299221ae # via ocotilloapi -pygeofilter==0.4.0 \ - --hash=sha256:cbb4a5f14af0b87e4f0c0c81c659ff64e44351c98e9f61d36af515d896fa8a05 \ - --hash=sha256:ddb74c8233f4fd1b62b80a0ecf4e4f9aff178b8c61334754288d1622c8db71ec +pygeofilter==0.3.3 \ + --hash=sha256:8b9fec05ba144943a1e415b6ac3752ad6011f44aad7d1bb27e7ef48b073460bd \ + --hash=sha256:e719fcb929c6b60bca99de0cfde5f95bc3245cab50516c103dae1d4f12c4c7b6 # via pygeoapi pygeoif==1.6.0 \ --hash=sha256:02f84807dadbaf1941c4bb2a9ef1ebac99b1b0404597d2602efdbb58910c69c9 \ @@ -1861,80 +1425,17 @@ pytz==2025.2 \ # ocotilloapi # pandas # pygeoapi -pyyaml==6.0.3 \ - --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ - --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ - --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ - --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ - --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ - --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ - --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ - --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ - --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ - --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ - --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ - --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ - --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ - --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ - --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ - --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ - --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ - --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ - --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ - --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ - --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ - --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ - --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ - --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ - --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ - --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ - --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ - --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ - --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ - --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ - --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ - --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ - --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ - --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ - --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ - --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ - --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ - --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ - --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ - --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ - --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ - --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ - --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ - --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ - --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ - --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ - --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ - --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ - --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ - --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ - --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ - --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ - --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ - --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ - --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ - --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ - --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ - --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ - --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ - --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ - --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ - --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ - --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ - --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ - --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ - --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ - --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ - --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ - --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ - --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ - --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ - --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ - --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 +pyyaml==6.0.2 \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba # via # pre-commit # pygeoapi @@ -1971,121 +1472,72 @@ referencing==0.37.0 \ # via # jsonschema # jsonschema-specifications -regex==2026.5.9 \ - --hash=sha256:002205cafd2a9e78c6290c7d1df277bf3277b3b7a30e0b4bb0dac2e2e3f7cb2d \ - --hash=sha256:01f0f5f55f4b64dacec85dc116d3c05fd23ad3ff037bbc73a2085775953c2611 \ - --hash=sha256:01f28d868834624c934b8d2e0aa1c8341337e37831f4a012f18a5afcba4cbaf3 \ - --hash=sha256:075160bf16658e16d35233300b8453aac25de4cbea808d22348b6979668e924d \ - --hash=sha256:0de5cf193997384ed2ca6f1cd4f78055b255d93d82d5a8cd6ba0d11c10b167e4 \ - --hash=sha256:0e1b1b4e496afbb24f4a62aba855ee4f88f25578927697b340702e48c9ee6bc2 \ - --hash=sha256:0f03aa6898aaaac4592479821df16e68e8d0e29e903e65d8f2dfb2f19028a989 \ - --hash=sha256:0f9eede6a5cbdc02d4978090186390936e1776a7d1359b21e41014c609880bcf \ - --hash=sha256:1268eddd8486dc561d08eee1156e40aa3a8fe10f4bdec8fa653b455fcbffd12c \ - --hash=sha256:15ee42209947f4ca045412eae98416317238163618ace2a8e54f99586a466733 \ - --hash=sha256:164eba9b755ea6f244b0d881196fbc1fac09714e9782c9e2732b813142033c8e \ - --hash=sha256:19c16ceb4a267a8789e25733e583983eeab9f0f8664e66b0bd1c5d21f14c2d4b \ - --hash=sha256:1bd7587a2948b4085195d5a3374eaf4a425dc3e55784c038175355ecf3bbbf8a \ - --hash=sha256:1e6da47d679b7010ef27556b6e0f99771b744936db1792a10ceac6547ae1503e \ - --hash=sha256:205109e96b3cf5adf8f4cd62bedde9487feb282b9497a3535451e5a24cd706a0 \ - --hash=sha256:2099f7e7ff7b6aa3192312650a56e91cc091e49d50b04e4f6f8b6e28b3b27f1c \ - --hash=sha256:246de9d60aa3f8538b519834dd95cbf276ea263d6a7bd5a3666dc3fa0230505b \ - --hash=sha256:24b2355ef5cc9aa5b8f07d17704face1c166fdcc2290fa7bd6e6c925655a8346 \ - --hash=sha256:2a661a7d270a61f7cf460caee8b9fa2d5ef9e5c681234bcb9e0fe14f488e7dfc \ - --hash=sha256:2acfb48634f64996b57f90f39afa692ff362162722581921fe92239a59960f3c \ - --hash=sha256:2efa205e6d98b24d1f3ab395c11aa15cdf10935bca283d0285e0499c284fba21 \ - --hash=sha256:31037c82eccb44b7ea2e9e221d7c01429430e989a1f4b91ea5a855f6017b509a \ - --hash=sha256:3527bb4942d2c14552155406cdedd906567456821848aed1cb4933a391bf5eca \ - --hash=sha256:39617fb0cde9c0e6306dc70e3bfc096f3da793219879f7ae7aa341a69fbdcf6d \ - --hash=sha256:398c521292f4c7fb807001dcd54694d3a1fcafc179a36ad9cc56f98df85930b6 \ - --hash=sha256:3b1e39888c5e0c7d92cea4fc777396c4a90363b05de75d02eb459a4752200808 \ - --hash=sha256:3dd4a3ff360dfb836fecdb93a4598f9d6e2ac81e3e397125145c6221bf58cf4c \ - --hash=sha256:3ddd90103f9e5c471c49c7852ecc1fe27c7e45eb99e977aefe7caa4e779f4f58 \ - --hash=sha256:446ddd671e43ab535810c4b21cff7104945c701d4a14d1e6d1cd6f4e445a8bea \ - --hash=sha256:45375819235558a4ff1c4971dc32881f022613abdb180128f5cb4768c1765a1c \ - --hash=sha256:46f1326ca6e65b0879d23ca302c0f2415aad42ff0309b9c818e7949fe19a41d8 \ - --hash=sha256:48036f6374aaa79eb3b754ec29c61d1c6b1606749d705a13f8854fa2539671f6 \ - --hash=sha256:4ebe8f0b5ec5a5024dc4a4c59f444c4e9afc5f2abdbb8962065b75d27fb971f9 \ - --hash=sha256:4eeb011098fcb77af513dcef521a3dbecbf8849b1e38940759d293b7a93f5026 \ - --hash=sha256:508f56a89ba9cb26e4168cbc37dbd60a28d82430a9e18ad1d25fe0883c314ca2 \ - --hash=sha256:5604dfd046dc37eca90250fc3be938b076c8059fa772ac0ed6f499b0f0fb0415 \ - --hash=sha256:56a33f191f17d8c417f99945ebdc1e691d3af9605d86ec68c7e54a57e3e17af6 \ - --hash=sha256:57e8915c7986aa33d25e4d3629cef711cd2863f2961b10409f0c04cb8b7d9020 \ - --hash=sha256:57eeeb05db7979413dec5438f2db21d7ecbba787cde7a711df1a6f6df672aa06 \ - --hash=sha256:5b73ab8afcf66c622db143d1c6fda4e58e4d537ee4f125229ad47b1ab80f34c0 \ - --hash=sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa \ - --hash=sha256:6351571c8a42b505eb555c0dc47d740d0fb66977dc142919eea6f4325b7c56a0 \ - --hash=sha256:6441cc660d76107934a09c22167200839a0e89604a6297f78a974e66e931d2c0 \ - --hash=sha256:65c8c8c37377794bd5b2f3ebe51919042bf17aec802e23c833d89782ed0c78af \ - --hash=sha256:6ba42b2e7e7f46cf68cc6a5ca36fa07959f9bbd9c6bdcc47b6ee76549a590248 \ - --hash=sha256:71b61c5bfe1c806332defc42ad6c780b3c55f661986d7f40283a3a88274b4c00 \ - --hash=sha256:728d8bfd28a8845c8b6bc5dc7ce010453d206396786c0765c2740cb65f37791e \ - --hash=sha256:7b92817338591505f282cf3864c145244b1edcf5381d237038df955001091538 \ - --hash=sha256:7e30b874d341fac767d7df5a0870540541c2c054b80cfaac116e8d367a8a7ff2 \ - --hash=sha256:7e87577720152d2caae19fe2baaf1f8d5ca12091e9e229f03915c37d1e4b9178 \ - --hash=sha256:83d0ee4a57d1c87cb549e195ec300b8f0ec3a82eba66d835e4e2ed8634fe4499 \ - --hash=sha256:8676474c07469d6f33dd1085ca2cd45f65785f32518f2b20e36d9953ca07f994 \ - --hash=sha256:86f40a5d6444db30a125c9c9177e6b25dad981cbc37451fd838f145e6edac92e \ - --hash=sha256:872acc074bd29ffc9913ecdfedf6ea77502312ca44a4aa0d3779089c6069d8de \ - --hash=sha256:8abd33fef90b2a9efac5557d6033ca82d1195ed3a15fea5af15ba7b463c6a63b \ - --hash=sha256:8c6e4218fbdfbcd4f6c19efca40930d24a621bf4b48cb76bc6640543bd28ef20 \ - --hash=sha256:8e76e8161ad00694cfce6767d5dea860c6391ac5b83e5c3a39661e696f11fc7e \ - --hash=sha256:8f3af7a4903c5c04a11a196a5aa75cdd7dd3f8508132f9fb3259d9f5908e3b88 \ - --hash=sha256:91328f1c23d47595ca3ef0a7557fa129c5a23404b775c770697d2f35b33e0107 \ - --hash=sha256:916714069da19329ef7de197dcbc77bb3104145c7c2c864dbfbe318f46b88b14 \ - --hash=sha256:93a7860539414dddaefba2b40f8771765ae17949d4c7182b876ce429e11a8309 \ - --hash=sha256:954cc214c04663ee6d266fc61739cad83054683048de65c5bd1d640ad28098ac \ - --hash=sha256:96f5f58b54a063d7ea9dca08e1cf57bfe10499c4d579ee672da284f57f5f0070 \ - --hash=sha256:97cf3bc1b7d7d2306772ec07366c80d9df00ff79e79cea32898883a646d2fae2 \ - --hash=sha256:98bd73080e8756255137e1bd3f3f00295bbc5aa383c0e0f973920e9134d7c4ad \ - --hash=sha256:992604d02e6d9c6d786c24a706a71ecffe1020fc1ef264044474cd81fa2c3919 \ - --hash=sha256:a24852d3c29ad9e47593593d8a247c44ccc3d0548ef12c822d6ed0810affe676 \ - --hash=sha256:a6a563446a41adc451393dc6b8e6ad87979efaee3c8738690a8d1b08ebead1b4 \ - --hash=sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270 \ - --hash=sha256:a8820737949116ffff55fe18f9fc644530063ba6ebfcb8314239416e78f1347c \ - --hash=sha256:a9e1328e17c84c1a5d22ec9f785ecef4a967fab9a42b6a8dc3bcbebd0a0c9e44 \ - --hash=sha256:aa0fbdbac82cb3e4450d0ccde7d7a35607f4cb2dd9fba4b8b69bfaf8c9fa6aed \ - --hash=sha256:b310768746dd314ea6e2ff4cc89ef215426813396ff4e94ee8e6f7096c8b6e03 \ - --hash=sha256:b46b0f094dc1d3b90356c85a0bd2c9bafc4a6a190b9d6f8ddd5a033b6e088ed4 \ - --hash=sha256:b4bb445ff3f725f59df8f6014edb547ee928ec7023a774f6a39a3f953038cbb2 \ - --hash=sha256:b6d189041f15691cfa2b6c4290448ec221244d225b3f5fe9e7771b34ffcdf6e2 \ - --hash=sha256:b96350aa424e79d4fd6b567b344dcbe2b2d6bfc48dfe7717587e1fa6d43da6ff \ - --hash=sha256:be3372b9df6ddecff6486d37e19095a7b4973137caf5512407a89f4455361f41 \ - --hash=sha256:bfe1ce50cbfb569d74e1e4337da6468961f31dbea55fd85aa5de59c0947a805a \ - --hash=sha256:c010eb8caca74bdb40c07498d7ece26b4428fd3f04aa8a72c9ac6f79e8faaac6 \ - --hash=sha256:c8b9b9d294cfea3cd19c718ade7cc93492b2c4991abd9a68d0b3477ae6d8e100 \ - --hash=sha256:c9411dd64ca95477225734a93dfc8583b51916b8d5942f99d6cac21e09965451 \ - --hash=sha256:ca518ed29c46eecba6010b15f1b9a479314d2de409536e71b6a13aa04e3b8a77 \ - --hash=sha256:ccf5249114cc3e772ecdd88a98a86eca0fd74c61ce32a94743758c083fc05d48 \ - --hash=sha256:cd2846168eb9ee3c513902bc8225409cb1caab31d04728b145171fa1625d9621 \ - --hash=sha256:d29eebfc9525db68cad3c97eedd7f754fa265aa5cd0cf4f863b2421e1b48fc9f \ - --hash=sha256:d3d7eb5c9a7f6df82ed3cfac9beb93882a5cbcb5b8b157b56cb2b3b276574ac1 \ - --hash=sha256:d626b84406444b165fc0ba981604edea39f0588ff1f92baa23fe50799ea9afdb \ - --hash=sha256:d641a8c9a61618047796d572a39a79b26167b0411d2c3031937b2fe2d081e2cf \ - --hash=sha256:d659eee77986549c9ea45b861c7567e44d6287c3dc9a4565478853f7b9fe2ff6 \ - --hash=sha256:d6b8a143aca6c39b446ea8092cde25cc8fe9304d4f5fecfbc1a9dbb0282703c2 \ - --hash=sha256:d726ca3f0d76969bf1e8e477d160d3d666bbf999f6860bd314889e5345782046 \ - --hash=sha256:d7bdc0ab8f3dd7e1b4f9ab88634e13374669db86bb3c72e8292f07ae313f539f \ - --hash=sha256:daff2bdbaf1d23e52fdff7c0b7bc2048b68f978df6a4d107ac981f94caef2e66 \ - --hash=sha256:dd2810d22146b6d838acc5ec15602cb6b47920aa4e33015df3868eedfd20bab8 \ - --hash=sha256:ddda5340e6c01a293027dd46232fa79eaff1b48058ce7a98f572b6445b088041 \ - --hash=sha256:dea2e88e1cce4522496cce630e11e67b98b7076620bc4336c3f674bc21a375f4 \ - --hash=sha256:debb893095e944091c16e641a6e33c1b0f4cb61ab945ec5afbf53ce7068834d8 \ - --hash=sha256:dfbe4579b9f08036aa7d101d1835437a20783574ac66327e6b29b4018a138081 \ - --hash=sha256:e1d93bf647916292e8edcec150c07ddf3dc50179ccaf770c04a7f9e452155372 \ - --hash=sha256:e82db382b44d0111b22601c509c89f64434816c9e0eef9d1989cda8cc6ff1c04 \ - --hash=sha256:ea9c8ecfa1b73c73b626534d6626e5340d429630943672b8480724f44e84b962 \ - --hash=sha256:ead4b163ac30a29574510cd4b3e2e985ac5290c05fc7095557d6a5f403fc31b5 \ - --hash=sha256:ecd353045824e4477562a2ac718c25799cdaaa41f7aa925a806a8a3e6848a5b9 \ - --hash=sha256:ed2c9e8068b614c574d8d30e543d617cf5379b0535d46f97ef00e904745a08b5 \ - --hash=sha256:ed457d8e98ae812ed7732bef7bf78de78e834eae0372a74e23ca90ef21d910f9 \ - --hash=sha256:ef31cbfe458e21c6122ba8150ff060e0c7789ed0d26eb423f25472584920b555 \ - --hash=sha256:f079e50a0d3cc3cd5091fa9ff45869a2e6b2cd35895731edafb0327901a8d86d \ - --hash=sha256:f3844f134e834076677dd369976e9f5068679fcb8e50102fdf6b7ac96a3ec127 \ - --hash=sha256:f7a7c26137296beba7784de6eba69c6a93a63ccebc385e4962fe67e267a91225 \ - --hash=sha256:fa411799ca8da32a8d38d020a88faa5b6f91657d284761352940ecf9f7c3bbdd \ - --hash=sha256:fd03c4f0e33280d15cae17159b899245d6b7c53d21def19b263b39655061f5ce \ - --hash=sha256:fd190e88a895a8901325fad284a3f74ea52b1da8525b76cc811fa9b1edf0ce2b \ - --hash=sha256:ff8d372ac2acdc048d1c19916f27ee61bc5722728458ba6ca5052f2c72d51763 +regex==2026.2.19 \ + --hash=sha256:015088b8558502f1f0bccd58754835aa154a7a5b0bd9d4c9b7b96ff4ae9ba876 \ + --hash=sha256:02b9e1b8a7ebe2807cd7bbdf662510c8e43053a23262b9f46ad4fc2dfc9d204e \ + --hash=sha256:03d191a9bcf94d31af56d2575210cb0d0c6a054dbcad2ea9e00aa4c42903b919 \ + --hash=sha256:0d0e72703c60d68b18b27cde7cdb65ed2570ae29fb37231aa3076bfb6b1d1c13 \ + --hash=sha256:11c138febb40546ff9e026dbbc41dc9fb8b29e61013fa5848ccfe045f5b23b83 \ + --hash=sha256:127ea69273485348a126ebbf3d6052604d3c7da284f797bba781f364c0947d47 \ + --hash=sha256:17648e1a88e72d88641b12635e70e6c71c5136ba14edba29bf8fc6834005a265 \ + --hash=sha256:1e7a08622f7d51d7a068f7e4052a38739c412a3e74f55817073d2e2418149619 \ + --hash=sha256:2905ff4a97fad42f2d0834d8b1ea3c2f856ec209837e458d71a061a7d05f9f01 \ + --hash=sha256:294c0fb2e87c6bcc5f577c8f609210f5700b993151913352ed6c6af42f30f95f \ + --hash=sha256:2c1693ca6f444d554aa246b592355b5cec030ace5a2729eae1b04ab6e853e768 \ + --hash=sha256:2f914ae8c804c8a8a562fe216100bc156bfb51338c1f8d55fe32cf407774359a \ + --hash=sha256:2fedd459c791da24914ecc474feecd94cf7845efb262ac3134fe27cbd7eda799 \ + --hash=sha256:311fcccb76af31be4c588d5a17f8f1a059ae8f4b097192896ebffc95612f223a \ + --hash=sha256:3aa0944f1dc6e92f91f3b306ba7f851e1009398c84bfd370633182ee4fc26a64 \ + --hash=sha256:4071209fd4376ab5ceec72ad3507e9d3517c59e38a889079b98916477a871868 \ + --hash=sha256:43cdde87006271be6963896ed816733b10967baaf0e271d529c82e93da66675b \ + --hash=sha256:46e69a4bf552e30e74a8aa73f473c87efcb7f6e8c8ece60d9fd7bf13d5c86f02 \ + --hash=sha256:4a02faea614e7fdd6ba8b3bec6c8e79529d356b100381cec76e638f45d12ca04 \ + --hash=sha256:50f1ee9488dd7a9fda850ec7c68cad7a32fa49fd19733f5403a3f92b451dcf73 \ + --hash=sha256:516ee067c6c721d0d0bfb80a2004edbd060fffd07e456d4e1669e38fe82f922e \ + --hash=sha256:5390b130cce14a7d1db226a3896273b7b35be10af35e69f1cca843b6e5d2bb2d \ + --hash=sha256:5a8f28dd32a4ce9c41758d43b5b9115c1c497b4b1f50c457602c1d571fa98ce1 \ + --hash=sha256:5e3a31e94d10e52a896adaa3adf3621bd526ad2b45b8c2d23d1bbe74c7423007 \ + --hash=sha256:5e56c669535ac59cbf96ca1ece0ef26cb66809990cda4fa45e1e32c3b146599e \ + --hash=sha256:5ec1d7c080832fdd4e150c6f5621fe674c70c63b3ae5a4454cebd7796263b175 \ + --hash=sha256:6380f29ff212ec922b6efb56100c089251940e0526a0d05aa7c2d9b571ddf2fe \ + --hash=sha256:64128549b600987e0f335c2365879895f860a9161f283b14207c800a6ed623d3 \ + --hash=sha256:654dc41a5ba9b8cc8432b3f1aa8906d8b45f3e9502442a07c2f27f6c63f85db5 \ + --hash=sha256:655f553a1fa3ab8a7fd570eca793408b8d26a80bfd89ed24d116baaf13a38969 \ + --hash=sha256:6c8fb3b19652e425ff24169dad3ee07f99afa7996caa9dfbb3a9106cd726f49a \ + --hash=sha256:6fb8cb09b10e38f3ae17cc6dc04a1df77762bd0351b6ba9041438e7cc85ec310 \ + --hash=sha256:7187fdee1be0896c1499a991e9bf7c78e4b56b7863e7405d7bb687888ac10c4b \ + --hash=sha256:74ff212aa61532246bb3036b3dfea62233414b0154b8bc3676975da78383cac3 \ + --hash=sha256:77cfd6b5e7c4e8bf7a39d243ea05882acf5e3c7002b0ef4756de6606893b0ecd \ + --hash=sha256:790dbf87b0361606cb0d79b393c3e8f4436a14ee56568a7463014565d97da02a \ + --hash=sha256:80caaa1ddcc942ec7be18427354f9d58a79cee82dea2a6b3d4fd83302e1240d7 \ + --hash=sha256:8457c1bc10ee9b29cdfd897ccda41dce6bde0e9abd514bcfef7bcd05e254d411 \ + --hash=sha256:8497421099b981f67c99eba4154cf0dfd8e47159431427a11cfb6487f7791d9e \ + --hash=sha256:8abe671cf0f15c26b1ad389bf4043b068ce7d3b1c5d9313e12895f57d6738555 \ + --hash=sha256:8df08decd339e8b3f6a2eb5c05c687fe9d963ae91f352bc57beb05f5b2ac6879 \ + --hash=sha256:8e6e77cd92216eb489e21e5652a11b186afe9bdefca8a2db739fd6b205a9e0a4 \ + --hash=sha256:8edda06079bd770f7f0cf7f3bba1a0b447b96b4a543c91fe0c142d034c166161 \ + --hash=sha256:93d881cab5afdc41a005dba1524a40947d6f7a525057aa64aaf16065cf62faa9 \ + --hash=sha256:997862c619994c4a356cb7c3592502cbd50c2ab98da5f61c5c871f10f22de7e5 \ + --hash=sha256:9cbc69eae834afbf634f7c902fc72ff3e993f1c699156dd1af1adab5d06b7fe7 \ + --hash=sha256:9e6693b8567a59459b5dda19104c4a4dbbd4a1c78833eacc758796f2cfef1854 \ + --hash=sha256:9fff45852160960f29e184ec8a5be5ab4063cfd0b168d439d1fc4ac3744bf29e \ + --hash=sha256:a09ae430e94c049dc6957f6baa35ee3418a3a77f3c12b6e02883bd80a2b679b0 \ + --hash=sha256:a178df8ec03011153fbcd2c70cb961bc98cbbd9694b28f706c318bee8927c3db \ + --hash=sha256:ab780092b1424d13200aa5a62996e95f65ee3db8509be366437439cdc0af1a9f \ + --hash=sha256:b5100acb20648d9efd3f4e7e91f51187f95f22a741dcd719548a6cf4e1b34b3f \ + --hash=sha256:b9ab8dec42afefa6314ea9b31b188259ffdd93f433d77cad454cd0b8d235ce1c \ + --hash=sha256:bcf57d30659996ee5c7937999874504c11b5a068edc9515e6a59221cc2744dd1 \ + --hash=sha256:c0761d7ae8d65773e01515ebb0b304df1bf37a0a79546caad9cbe79a42c12af7 \ + --hash=sha256:c0924c64b082d4512b923ac016d6e1dcf647a3560b8a4c7e55cbbd13656cb4ed \ + --hash=sha256:c13228fbecb03eadbfd8f521732c5fda09ef761af02e920a3148e18ad0e09968 \ + --hash=sha256:c227f2922153ee42bbeb355fd6d009f8c81d9d7bdd666e2276ce41f53ed9a743 \ + --hash=sha256:c7e121a918bbee3f12ac300ce0a0d2f2c979cf208fb071ed8df5a6323281915c \ + --hash=sha256:cce8027010d1ffa3eb89a0b19621cdc78ae548ea2b49fea1f7bfb3ea77064c2b \ + --hash=sha256:d00c95a2b6bfeb3ea1cb68d1751b1dfce2b05adc2a72c488d77a780db06ab867 \ + --hash=sha256:d793c5b4d2b4c668524cd1651404cfc798d40694c759aec997e196fe9729ec60 \ + --hash=sha256:d96162140bb819814428800934c7b71b7bffe81fb6da2d6abc1dcca31741eca3 \ + --hash=sha256:e581f75d5c0b15669139ca1c2d3e23a65bb90e3c06ba9d9ea194c377c726a904 \ + --hash=sha256:ea8dfc99689240e61fb21b5fc2828f68b90abf7777d057b62d3166b7c1543c4c # via dateparser requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ @@ -2096,9 +1548,9 @@ requests==2.34.2 \ # google-cloud-storage # ocotilloapi # pygeoapi -rich==14.3.3 \ - --hash=sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d \ - --hash=sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b +rich==14.3.2 \ + --hash=sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69 \ + --hash=sha256:e712f11c1a562a11843306f5ed999475f09ac31ffb64281f73ab29ffdda8b3b8 # via typer rpds-py==0.30.0 \ --hash=sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136 \ @@ -2167,7 +1619,6 @@ rsa==4.9.1 \ --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 # via - # google-auth # ocotilloapi # python-jose scramp==1.4.8 \ @@ -2235,60 +1686,25 @@ sniffio==1.3.1 \ sqlalchemy==2.0.50 \ --hash=sha256:03f4323c980ad0e918cc9e5369b015f759f4e534db5bbaf4dc36832c10d05064 \ --hash=sha256:06a9210bdc5f4298cff0781087e2ff45683922252dacc452846373a58761f093 \ - --hash=sha256:0a31c5963d58d3e3d11c5b97709e248305705de1fdf51ec3bf396674c5898b7e \ - --hash=sha256:0e104e196f457ec608eb8af736c5eb4c6bc58f481b546f485a7f9c628ee532be \ - --hash=sha256:0f5e4ac70e9e757f6b3e87c0491ff034442ecd8dfd36d041a50564c322dafc0e \ - --hash=sha256:0fe7822866f3a9fc5f3db21a290ce8961a53050115f05edf9402b6a5feb92a9f \ - --hash=sha256:0fec460e18cdbb4c7773531122ce9a27e96c6ca17af3933941d94da475ad2c86 \ - --hash=sha256:110fdac56ace278949f00de805edacbd6141e382d992f9ba28238b3a0827a600 \ - --hash=sha256:1208050441471d003b7c8cb4054fb084f185cf35ac3f0ea270803865bca9939a \ - --hash=sha256:13b85b20f9ab714a666df9d8e72e253ec33c16c7e1e375c877e5bf6367a3e917 \ --hash=sha256:15708c613cd5005b7dffe1f66ee6a63ee8f5e46799f71c70ebad74178c676a39 \ --hash=sha256:1918a3cf564d16d95bca7301005f41ab2ad50b07cd3b9da50d3ed986db148d6a \ - --hash=sha256:1aa6e403663a9c43c8fef7ce4bdb4cf48bcd8d352e91deda2a99f963270bd508 \ - --hash=sha256:1c5f858fe79c9f5d8fda065c06186356acb7f8df3cd52dbd5ee3f200e4b144f5 \ --hash=sha256:1fbd55a969d7ac44a98e3dec75016074f809fa08f871585ace58dde110d1bf3e \ - --hash=sha256:23ae23d8b9d344d30d0a92f06d45825024a5790f1c1dd4cf452636a50d3e58cb \ - --hash=sha256:27b7062af702c61994e8806ad87e42d0a2c879e0a8e5c61c7f69d81dabe24fdf \ --hash=sha256:287086e67275a212c4582d166a6fb03a65ccc5551d80866270ce0dd9f34eccd3 \ --hash=sha256:2b9dcc43afef8ac157cd92fce96985d6b8b0cfbd3df4d666f66b4d55a75d202f \ - --hash=sha256:2c1920cde9d741ba3dda9b1aa5acd8c23ea17780ccfb2252d01878d5d0d628d3 \ - --hash=sha256:2dab927761d9108550f0cf8e66ff21af56f907a0ce0a689793db615e2b55f62c \ --hash=sha256:31648fa14460537e768a7303b078e4344d208e0d23e06867c1f376a227ed82db \ --hash=sha256:3699dac4be410e97049a1658e9480da9cde956594aa0f3aebc60b88f21c5ba70 \ - --hash=sha256:3d10700bd519573f6ce5badbabbfe7f5baea84cdf370f2cbbfb4be28dfddbf1d \ - --hash=sha256:409a8121b917116b035bedc5e532ad470c74a2d279f6c302100985b6304e9f9e \ - --hash=sha256:47b71b933e7b4ebad407c8fdfd70d2c4f08b78b3238bb30eebdd6eb32ca51b89 \ - --hash=sha256:4a8e8af330cbb3a1931d3d6c91b239fc2ef135f7dd471dfa34c575028e0b1fa8 \ - --hash=sha256:51b637a84f9fa35ae1f9017e786cb142974a25305085e1b378b3647a67f65ad3 \ - --hash=sha256:545eae198d37bcf837a10ede3684e2af32458d6f35c597c35c2de7502dc38fc4 \ --hash=sha256:60922d6599065ddca2c6f376b9aa2f41a6b85a271725e0909490bbc50b1998a5 \ - --hash=sha256:66e374271ecb7101273f57af1a62446a953d327eec4f8089147de57c591bbacc \ - --hash=sha256:68b154b08088b4ec32bb4d2958bfbb50e57549f91a4cd3e7f928e3553ed69031 \ --hash=sha256:6c206aec519a2e7bd08abbfb33436e325fd22c632d9c21a9047e376ce241646e \ - --hash=sha256:724f3dcbe53dd0151e3cb5e7ec4ba4c620bede579caacd16275dc35ce06e8615 \ - --hash=sha256:7af6eeb84985bf840ba779018ff9424d61ff69b52e66b8789d3c8da7bf5341b2 \ - --hash=sha256:7b1ddb7b5fc60dfa9df6a487f06a143c77def47c0351849da2bcea59b244a56c \ - --hash=sha256:7e36efdcc5493f8024ec873a4ee3855bfd2de0c5b19eba16f920e9d2a0d28622 \ - --hash=sha256:83a9fce296b7e052316d8c6943237b31b9c00f58ca9c253f2d165df52637a293 \ --hash=sha256:8b53784972ade4f8174b9aa661f31a06f8a936d2cfdd602913ff3c6dd40ae873 \ --hash=sha256:8f00e3eb43ba30eb1b238ee03a8a62309486d1321eda3328bb611e0340033ad8 \ --hash=sha256:92064363517a3ff8212b5a93b8c62876579d8dfd1ca5b561335f30152d884fa9 \ - --hash=sha256:9602c07b03e1449747ecb69f9998a7194a589124475788b370adce57c9e9a56e \ --hash=sha256:96fbee6b19c19cd1556c8bf9419447cf2ec149ffcab7ab64348c23e54ef8547f \ - --hash=sha256:9d1af51558029a156a70986b7df88f042b3d158d7c8d8fb5072912d4b32d89c7 \ - --hash=sha256:adc0fe7d38d8c8058f7421c25508fcbc74df38233a42aa8324409844122dce8f \ --hash=sha256:af5607d11ef90fd6a5c0549fe0045dce1663d427426bcfb506dcb5346a85a3b9 \ --hash=sha256:b00098cdbdbd38c7be3d568b0c9c3122b8c0ec62b911b57cd5e6e0254d60a76d \ --hash=sha256:bef4ac756363227ef6402a75fee025a4bc690f92328e825868939b3b3a446a6d \ --hash=sha256:c4e70c46fad30c3bcc6a4708bc0130a3173e11a5b25f0ea4a9d8911b450f1f52 \ --hash=sha256:c5c3cdb753a9004183e1ccb634b41611654c989e61bc68617ce878e46d6f1e51 \ --hash=sha256:c966932507a4d7d0a37314927dbfcd89720e3f37d2a1e3352e7ae7939fa8e8a0 \ - --hash=sha256:e195687f1af431c9515416288373b323b6eb599f774409814e89e9d603a56e39 \ - --hash=sha256:e6e814658818fd165e749e3d8490ef16cc7f379a118c37ada8b0589ffbaaac22 \ - --hash=sha256:e8e1b0f6a4dcd9b4839e2320afb5df37a6981cbc20ff9c423ae11c5537bdbd21 \ - --hash=sha256:ea1a8a2db4b2217d456c8d7a873bfc605f06fe3584d315264ea18c2a17585d0b \ - --hash=sha256:eefd9a03cc0047b14153872d228499d048bd7deaf926109c9ec25b15157b8e23 \ --hash=sha256:f96233858e3df43932ac11589e22520da6e8aeb624b03fedfeebb0e8ea213086 \ --hash=sha256:faffef4bcc20a1892e65e155293d99d60855bbbc79250ab712819cfd56a8e6bb # via @@ -2313,6 +1729,10 @@ sqlalchemy-utils==0.42.1 \ # via # ocotilloapi # sqlalchemy-searchable +sqlparse==0.5.5 \ + --hash=sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba \ + --hash=sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e + # via ocotilloapi starlette==0.52.1 \ --hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ --hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 @@ -2386,116 +1806,65 @@ uvicorn==0.49.0 \ --hash=sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f \ --hash=sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3 # via ocotilloapi -virtualenv==21.4.2 \ - --hash=sha256:38e6ee0a555615c0ea9da2ac7e9998fe8dc3b911dd33ad8eaad2020957653b0c \ - --hash=sha256:854210ca524a1a4d0d744734f4acbc721c3ffe163b85bbf5d56d14d5ae2f0fae +virtualenv==20.32.0 \ + --hash=sha256:2c310aecb62e5aa1b06103ed7c2977b81e042695de2697d01017ff0f1034af56 \ + --hash=sha256:886bf75cadfdc964674e6e33eb74d787dff31ca314ceace03ca5810620f4ecf0 # via pre-commit -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ - --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 +werkzeug==3.1.6 \ + --hash=sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25 \ + --hash=sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 # via flask yarl==1.24.2 \ --hash=sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b \ - --hash=sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30 \ --hash=sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc \ - --hash=sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f \ - --hash=sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae \ --hash=sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8 \ - --hash=sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75 \ - --hash=sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a \ - --hash=sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c \ --hash=sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461 \ --hash=sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44 \ --hash=sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b \ - --hash=sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727 \ --hash=sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9 \ --hash=sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd \ --hash=sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67 \ --hash=sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420 \ - --hash=sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db \ --hash=sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50 \ --hash=sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b \ - --hash=sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50 \ - --hash=sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9 \ - --hash=sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1 \ --hash=sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488 \ - --hash=sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2 \ - --hash=sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f \ - --hash=sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d \ - --hash=sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003 \ --hash=sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536 \ --hash=sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a \ - --hash=sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a \ --hash=sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa \ --hash=sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f \ - --hash=sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e \ - --hash=sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035 \ - --hash=sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12 \ --hash=sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe \ - --hash=sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4 \ - --hash=sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294 \ - --hash=sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7 \ --hash=sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761 \ - --hash=sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643 \ - --hash=sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413 \ --hash=sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57 \ - --hash=sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36 \ --hash=sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14 \ --hash=sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd \ - --hash=sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5 \ --hash=sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656 \ - --hash=sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad \ - --hash=sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c \ - --hash=sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0 \ --hash=sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992 \ - --hash=sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342 \ --hash=sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1 \ --hash=sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf \ --hash=sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024 \ --hash=sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986 \ --hash=sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb \ - --hash=sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d \ --hash=sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543 \ - --hash=sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d \ --hash=sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed \ --hash=sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617 \ - --hash=sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996 \ --hash=sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8 \ - --hash=sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2 \ --hash=sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3 \ --hash=sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535 \ --hash=sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630 \ --hash=sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215 \ --hash=sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592 \ --hash=sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf \ - --hash=sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b \ - --hash=sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac \ --hash=sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0 \ --hash=sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92 \ - --hash=sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122 \ --hash=sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1 \ - --hash=sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8 \ - --hash=sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576 \ --hash=sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8 \ - --hash=sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712 \ --hash=sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1 \ - --hash=sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2 \ - --hash=sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b \ --hash=sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a \ - --hash=sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53 \ - --hash=sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1 \ --hash=sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d \ --hash=sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208 \ --hash=sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0 \ - --hash=sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c \ --hash=sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607 \ - --hash=sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c \ --hash=sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8 \ - --hash=sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39 \ - --hash=sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f \ - --hash=sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8 \ - --hash=sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90 \ - --hash=sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45 \ --hash=sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2 \ --hash=sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056 \ --hash=sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14 From bb9f2cc8e81cb2ccdf552bd7e4d6dedcb179f425 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Fri, 12 Jun 2026 15:24:08 -0700 Subject: [PATCH 21/41] exporting NM_Wells csv files -generate csv files via script -fix some data overflow problems --- pyproject.toml | 1 + transfers/export_nmw_csvs.py | 86 ++++++++++++++++++++++++++++++++ transfers/nmw_mirror_transfer.py | 13 ++++- uv.lock | 26 +++++++++- 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 transfers/export_nmw_csvs.py diff --git a/pyproject.toml b/pyproject.toml index 3feae3128..05e26ce4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ dependencies = [ "utm==0.8.1", "uvicorn==0.49.0", "yarl==1.24.2", + "pymssql>=2.3.13", ] [tool.uv] diff --git a/transfers/export_nmw_csvs.py b/transfers/export_nmw_csvs.py new file mode 100644 index 000000000..e2cf438b3 --- /dev/null +++ b/transfers/export_nmw_csvs.py @@ -0,0 +1,86 @@ +"""Export NM_Wells SQL Server tables to CSV files for the transfer pipeline. + +Connects to the NM_Wells SQL Server database and exports each source table to +transfers/data/nma_csv_cache/
.csv, which is where nmw_mirror_transfer.py +looks for them when NMW_SQL_DUMP is not set. + +Usage: + uv run python -m transfers.export_nmw_csvs + +Required environment variables (add to .env): + NMW_HOST SQL Server hostname or IP + NMW_PORT SQL Server port (default: 1433) + NMW_USER SQL Server username + NMW_PASSWORD SQL Server password + NMW_DATABASE Database name (default: NM_Wells) +""" + +import os +from pathlib import Path + +import pymssql +from dotenv import load_dotenv + +from transfers.nmw_mirror_transfer import NMW_MIRROR_SPECS + +load_dotenv(override=False) + +TABLES = [spec.source_table for spec in NMW_MIRROR_SPECS] + +OUT_DIR = Path(__file__).parent / "data" / "nma_csv_cache" + + +def _get_connection(): + host = os.environ["NMW_HOST"] + port = int(os.environ.get("NMW_PORT", 1433)) + user = os.environ["NMW_USER"] + password = os.environ["NMW_PASSWORD"] + database = os.environ.get("NMW_DATABASE", "NM_Wells") + return pymssql.connect( + server=host, + port=port, + user=user, + password=password, + database=database, + ) + + +def export_table(cursor, table: str, out_path: Path) -> int: + cursor.execute(f"SELECT * FROM dbo.{table}") + columns = [desc[0] for desc in cursor.description] + rows = cursor.fetchall() + + import csv + + with out_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(columns) + writer.writerows(rows) + + return len(rows) + + +def main(): + OUT_DIR.mkdir(parents=True, exist_ok=True) + print( + f"Connecting to {os.environ.get('NMW_HOST')} / {os.environ.get('NMW_DATABASE', 'NM_Wells')}" + ) + conn = _get_connection() + cursor = conn.cursor() + + for table in TABLES: + out_path = OUT_DIR / f"{table}.csv" + print(f" Exporting {table}...", end=" ", flush=True) + try: + n = export_table(cursor, table, out_path) + print(f"{n} rows -> {out_path.name}") + except Exception as e: + print(f"FAILED: {e}") + + cursor.close() + conn.close() + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index d541d0f0e..18f930e13 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -84,6 +84,9 @@ # Optional output dir for the per-table CSVs written from the dump (COPY path). # Defaults to a fresh temp dir. _CSV_DIR_ENV = "NMW_CSV_DIR" +# pg8000 encodes the parameter count as an unsigned short (max 65535). +# Keep chunk size below that ceiling based on actual column count per table. +_MAX_PG8000_PARAMS = 65535 _CHUNK_SIZE = 2000 # Materialized OGC views over the geothermal mirror that need a REFRESH after a @@ -248,6 +251,7 @@ def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: if limit and limit > 0: rows_iter = itertools.islice(rows_iter, limit) + chunk_size = min(_CHUNK_SIZE, _MAX_PG8000_PARAMS // max(len(cols), 1)) total = 0 inserted = 0 batch: list[dict] = [] @@ -266,7 +270,7 @@ def _load_table(session: Session, spec: MirrorSpec, limit: int = 0) -> dict: if any(row.get(pk) is None for pk in pk_cols): continue # cannot upsert without a PK value batch.append(row) - if len(batch) >= _CHUNK_SIZE: + if len(batch) >= chunk_size: inserted += _flush(session, spec.model, batch, pk_cols) batch = [] inserted += _flush(session, spec.model, batch, pk_cols) @@ -347,7 +351,12 @@ def refresh_materialized_views(session: Session) -> list[str]: refreshed = [] for view in _MATERIALIZED_VIEWS: exists = session.execute( - text("SELECT to_regclass(:n)"), {"n": f"public.{view}"} + text( + "SELECT EXISTS(" + "SELECT 1 FROM pg_matviews WHERE schemaname='public' AND matviewname=:n" + ")" + ), + {"n": view}, ).scalar() if not exists: logger.warning("Skip refresh; materialized view missing: %s", view) diff --git a/uv.lock b/uv.lock index a56136a1a..129add870 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.13" [[package]] @@ -1560,6 +1560,7 @@ dependencies = [ { name = "pygeoapi" }, { name = "pygments" }, { name = "pyjwt" }, + { name = "pymssql" }, { name = "pyproj" }, { name = "pyshp" }, { name = "python-dateutil" }, @@ -1674,6 +1675,7 @@ requires-dist = [ { name = "pygeoapi", specifier = "==0.23.4" }, { name = "pygments", specifier = "==2.20.0" }, { name = "pyjwt", specifier = "==2.13.0" }, + { name = "pymssql", specifier = ">=2.3.13" }, { name = "pyproj", specifier = "==3.7.2" }, { name = "pyshp", specifier = "==2.3.1" }, { name = "python-dateutil", specifier = "==2.9.0.post0" }, @@ -2313,6 +2315,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] +[[package]] +name = "pymssql" +version = "2.3.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/cc/843c044b7f71ee329436b7327c578383e2f2499313899f88ad267cdf1f33/pymssql-2.3.13.tar.gz", hash = "sha256:2137e904b1a65546be4ccb96730a391fcd5a85aab8a0632721feb5d7e39cfbce", size = 203153, upload-time = "2026-02-14T05:00:36.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/4f/ee15b1f6b11e7c3accdc7da7840a019b63f12ba09eaa008acc601182f516/pymssql-2.3.13-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:30918bb044242865c01838909777ef5e0f1b9ecd7f5882346aefa57f4414b29c", size = 3156333, upload-time = "2026-02-14T05:00:01.21Z" }, + { url = "https://files.pythonhosted.org/packages/79/03/aea5c77bad4a52649a1d9f786a1d9ce1c83d50f1a75df288e292737b6d80/pymssql-2.3.13-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1c6d0b2d7961f159a07e4f0d8cc81f70ceab83f5e7fd1e832a2d069e1d67ee4e", size = 2957990, upload-time = "2026-02-14T05:00:03.11Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f8/30ac16fba32ff066b05f12c392d7b812fe11f06cb62d1d86ca5177c50a8b/pymssql-2.3.13-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16c5957a3c9e51a03276bfd76a22431e2bc4c565e2e95f2cbb3559312edda230", size = 3065264, upload-time = "2026-02-14T05:00:05.377Z" }, + { url = "https://files.pythonhosted.org/packages/a9/98/7568447bf85921d21453fd56e19b6c9591d595fde0546c5a569f3ae937a8/pymssql-2.3.13-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fddd24efe9d18bbf174fab7c6745b0927773718387f5517cf8082241f721a68", size = 3190039, upload-time = "2026-02-14T05:00:06.925Z" }, + { url = "https://files.pythonhosted.org/packages/35/f1/4d9d275ebaac42cdd49d40d504ccb648f27710660c8b60cc427752438c09/pymssql-2.3.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:123c55ee41bc7a82c76db12e2eb189b50d0d7a11222b4f8789206d1cda3b33b9", size = 3710151, upload-time = "2026-02-14T05:00:08.424Z" }, + { url = "https://files.pythonhosted.org/packages/6f/bd/a5cc6244fd27d3ea0cc82f12a7d38a24d7fd90b0022afd250014e8bfba15/pymssql-2.3.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e053b443e842f9e1698fcb2b23a4bff1ff3d410894d880064e754ad823d541e5", size = 3453156, upload-time = "2026-02-14T05:00:09.978Z" }, + { url = "https://files.pythonhosted.org/packages/26/d0/c20ff0bbffd18db528bcc7b0c68b25c12ad563ed67c56ceca87c58f7399e/pymssql-2.3.13-cp313-cp313-win_amd64.whl", hash = "sha256:5c045c0f1977a679cc30d5acd9da3f8aeb2dc6e744895b26444b4a2f20dad9a0", size = 1995236, upload-time = "2026-02-14T05:00:11.495Z" }, + { url = "https://files.pythonhosted.org/packages/ec/5f/6b64f78181d680f655ab40ba7b34cb68c045a2f4e04a10a70d768cd383b7/pymssql-2.3.13-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:fc5482969c813b0a45ce51c41844ae5bfa8044ad5ef8b4820ef6de7d4545b7f2", size = 3158377, upload-time = "2026-02-14T05:00:13.581Z" }, + { url = "https://files.pythonhosted.org/packages/ff/24/155dbb0992c431496d440f47fb9d587cd0059ee20baf65e3d891794d862a/pymssql-2.3.13-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:ff5be7ab1d643dbce2ee3424d2ef9ae8e4146cf75bd20946bc7a6108e3ad1e47", size = 2959039, upload-time = "2026-02-14T05:00:15.883Z" }, + { url = "https://files.pythonhosted.org/packages/c9/89/b453dd1b1188779621fb974ac715ab2e738f4a0b69f7291ab014298bd80d/pymssql-2.3.13-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d66ce0a249d2e3b57369048d71e1f00d08dfb90a758d134da0250ae7bc739c1", size = 3063862, upload-time = "2026-02-14T05:00:17.537Z" }, + { url = "https://files.pythonhosted.org/packages/02/e5/96f57c78162013678ecc3f3f7e5fb52c83ee07beef26906d0870770c3ef6/pymssql-2.3.13-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d663c908414a6a032f04d17628138b1782af916afc0df9fefac4751fa394c3ac", size = 3188155, upload-time = "2026-02-14T05:00:19.011Z" }, + { url = "https://files.pythonhosted.org/packages/cd/a2/4bee9484734ae0c55d10a2f6ff82dd4e416f52420755161b8760c817ad64/pymssql-2.3.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aa5e07eff7e6e8bd4ba22c30e4cb8dd073e138cd272090603609a15cc5dbc75b", size = 3709344, upload-time = "2026-02-14T05:00:21.139Z" }, + { url = "https://files.pythonhosted.org/packages/37/cf/3520d96afa213c88db4f4a1988199db476d869a62afdd5d9c4635c184631/pymssql-2.3.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:db77da1a3fc9b5b5c5400639d79d7658ba7ad620957100c5b025be608b562193", size = 3451799, upload-time = "2026-02-14T05:00:22.504Z" }, + { url = "https://files.pythonhosted.org/packages/25/50/4be9bd9cf4b43208a7175117a533ece200cfe4131a39f9909bdc7560ddeb/pymssql-2.3.13-cp314-cp314-win_amd64.whl", hash = "sha256:7d7037d2b5b907acc7906d0479924db2935a70c720450c41339146a4ada2b93d", size = 2049139, upload-time = "2026-02-14T05:00:23.951Z" }, +] + [[package]] name = "pyparsing" version = "3.3.2" From 1bf1977ec78b8c30ee77c9c68b329e4d50716581 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Tue, 16 Jun 2026 12:47:58 -0700 Subject: [PATCH 22/41] feat: add geothermal OGC collections and fix Docker pygeoapi config Add geothermal_wells_bht and geothermal_wells_temperature_profile to the pygeoapi-config.yml template, exposing NM_Wells geothermal views as OGC API - Features collections. Fix _resolve() priority in pygeoapi.py so that shell PYGEOAPI_* env vars (e.g. PYGEOAPI_POSTGRES_HOST=db set in docker-compose) take precedence over generic POSTGRES_* values from .env, preventing localhost from shadowing the Docker service hostname. Add INSTALL_DEV=true build arg to docker-compose so faker and other dev dependencies are present when the container runs in development mode. Co-Authored-By: Claude Sonnet 4.6 --- core/pygeoapi-config.yml | 46 ++++++++++++++++++++++++ core/pygeoapi.py | 77 +++++++++++++++++++++++----------------- docker-compose.yml | 4 ++- 3 files changed, 93 insertions(+), 34 deletions(-) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 1bae81d90..7464e6a08 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -288,3 +288,49 @@ resources: geom_field: project_area {thing_collections_block} + + geothermal_wells_bht: + type: collection + title: Geothermal Wells — Bottom-Hole Temperature + description: Geothermal wells with bottom-hole temperature (BHT) measurements from the NM_Wells database. + keywords: [geothermal, wells, bottom-hole-temperature, bht] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_geothermal_wells_bht + geom_field: geom + + geothermal_wells_temperature_profile: + type: collection + title: Geothermal Wells — Temperature-Depth Profile + description: Geothermal wells with downhole temperature-vs-depth series from the NM_Wells database. + keywords: [geothermal, wells, temperature, depth, profile] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_geothermal_wells_temperature_profile + geom_field: geom diff --git a/core/pygeoapi.py b/core/pygeoapi.py index 7783af100..b40489f85 100644 --- a/core/pygeoapi.py +++ b/core/pygeoapi.py @@ -206,39 +206,55 @@ def _thing_collections_block( def _pygeoapi_db_settings() -> tuple[str, str, str, str, str]: - host = ( - (os.environ.get("PYGEOAPI_POSTGRES_HOST") or "").strip() - or (os.environ.get("POSTGRES_HOST") or "").strip() - or "127.0.0.1" - ) - port = ( - (os.environ.get("PYGEOAPI_POSTGRES_PORT") or "").strip() - or (os.environ.get("POSTGRES_PORT") or "").strip() - or "5432" - ) - dbname = ( - (os.environ.get("PYGEOAPI_POSTGRES_DB") or "").strip() - or (os.environ.get("POSTGRES_DB") or "").strip() - or "postgres" - ) - user = (os.environ.get("PYGEOAPI_POSTGRES_USER") or "").strip() or ( - os.environ.get("POSTGRES_USER") or "" - ).strip() + from dotenv import dotenv_values + + # Read .env directly so stale shell vars (e.g. from conda) can't override + # PYGEOAPI_POSTGRES_* values. Shell env takes precedence only for the + # PYGEOAPI_-prefixed keys (explicit per-service override), while the + # generic POSTGRES_* fallback always comes from the file. + env_file = Path(__file__).resolve().parents[1] / ".env" + dotenv = dotenv_values(env_file) if env_file.exists() else {} + + def _resolve(pygeoapi_key: str, fallback_key: str, default: str = "") -> str: + # Priority: .env PYGEOAPI_* > shell PYGEOAPI_* > .env POSTGRES_* > + # shell POSTGRES_* > hard default. + # Shell PYGEOAPI_* comes before .env POSTGRES_* so that explicit + # per-service overrides in Docker (e.g. PYGEOAPI_POSTGRES_HOST=db) + # beat the generic localhost values in .env. + return ( + (dotenv.get(pygeoapi_key) or "").strip() + or (os.environ.get(pygeoapi_key) or "").strip() + or (dotenv.get(fallback_key) or "").strip() + or (os.environ.get(fallback_key) or "").strip() + or default + ) + + host = _resolve("PYGEOAPI_POSTGRES_HOST", "POSTGRES_HOST", "127.0.0.1") + port = _resolve("PYGEOAPI_POSTGRES_PORT", "POSTGRES_PORT", "5432") + dbname = _resolve("PYGEOAPI_POSTGRES_DB", "POSTGRES_DB", "postgres") + user = _resolve("PYGEOAPI_POSTGRES_USER", "POSTGRES_USER") if not user: raise RuntimeError( "PYGEOAPI_POSTGRES_USER or POSTGRES_USER must be set and " - "non-empty to generate the pygeoapi configuration." + "non-empty in the environment or .env file." ) - if os.environ.get("PYGEOAPI_POSTGRES_PASSWORD") is None: + # Resolve the actual password at config-write time and embed it directly + # in the generated config file (which is already chmod 0600). This avoids + # stale shell env vars corrupting the ${VAR} expansion that pygeoapi's + # yaml_load would otherwise perform at request time. + password = _resolve("PYGEOAPI_POSTGRES_PASSWORD", "POSTGRES_PASSWORD") + if not password: raise RuntimeError( - "PYGEOAPI_POSTGRES_PASSWORD must be set to " - "generate the pygeoapi configuration." + "PYGEOAPI_POSTGRES_PASSWORD or POSTGRES_PASSWORD must be set " + "and non-empty in the environment or .env file." ) - return host, port, dbname, user, "${PYGEOAPI_POSTGRES_PASSWORD}" + return host, port, dbname, user, password def _write_config(path: Path) -> None: - host, port, dbname, user, password_placeholder = _pygeoapi_db_settings() + host, port, dbname, user, password = _pygeoapi_db_settings() + # Escape braces so str.format() doesn't misinterpret them in the password. + password_for_format = password.replace("{", "{{").replace("}", "}}") template = _template_path().read_text(encoding="utf-8") config = template.format( server_url=_server_url(), @@ -246,24 +262,19 @@ def _write_config(path: Path) -> None: postgres_port=port, postgres_db=dbname, postgres_user=user, - postgres_password_env=password_placeholder, + postgres_password_env=password_for_format, thing_collections_block=_thing_collections_block( host=host, port=port, dbname=dbname, user=user, - password_placeholder=password_placeholder, + password_placeholder=password, ), ) - # NOTE: The generated runtime config file at - # `${PYGEOAPI_RUNTIME_DIR}/pygeoapi-config.yml` (default: - # `/tmp/pygeoapi/pygeoapi-config.yml`) contains database connection details - # (host, port, dbname, user). Although the password is expected to be - # provided via environment variables at runtime by pygeoapi, this file - # should still be treated as sensitive configuration: + # NOTE: The generated runtime config file contains database credentials + # including the plaintext password. It is protected by chmod 0600. # * Do not commit it to version control. # * Do not expose it in logs, error messages, or diagnostics. - # * Ensure filesystem permissions restrict access appropriately. path.write_text(config, encoding="utf-8") path.chmod(0o600) diff --git a/docker-compose.yml b/docker-compose.yml index 78120d761..94991fb99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,9 +23,11 @@ services: retries: 20 app: - build: + build: context: . dockerfile: ./docker/app/Dockerfile + args: + INSTALL_DEV: "true" environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} From b0400d4de5c539c89e1fe35e99316314623b2256 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Tue, 16 Jun 2026 12:51:35 -0700 Subject: [PATCH 23/41] Migrations adding migrations and views --- ...ure_profile_view_duplicate_well_data_id.py | 132 +++++++++++ ...data_id_to_text_in_geothermal_ogc_views.py | 217 ++++++++++++++++++ ..._add_integer_id_to_geothermal_ogc_views.py | 215 +++++++++++++++++ ...add_fk_constraints_to_nmw_mirror_tables.py | 210 +++++++++++++++++ 4 files changed, 774 insertions(+) create mode 100644 alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py create mode 100644 alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py create mode 100644 alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py create mode 100644 alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py diff --git a/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py b/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py new file mode 100644 index 000000000..dc2cd4642 --- /dev/null +++ b/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py @@ -0,0 +1,132 @@ +"""fix temperature profile view duplicate well_data_id + +Revision ID: a3b4c5d6e7f8 +Revises: z2a3b4c5d6e7 +Create Date: 2026-06-15 + +NMW_WellLocations has multiple rows per WellDataID (OBJECTID is its PK, not +WellDataID). The prior view grouped by WellDataID + Lat_dd83 + Long_dd83, +producing one row per (well, location) pair. When a well has more than one +location row the unique index on well_data_id fails at REFRESH time. + +Fix: deduplicate NMW_WellLocations to one row per WellDataID via DISTINCT ON +before joining, so the GROUP BY always yields exactly one row per well. +""" + +from alembic import op +from sqlalchemy import text + +# revision identifiers, used by Alembic. +revision = "a3b4c5d6e7f8" +down_revision = "z2a3b4c5d6e7" +branch_labels = None +depends_on = None + +_VIEW = "ogc_geothermal_wells_temperature_profile" + + +def upgrade() -> None: + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_VIEW}"')) + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_VIEW}_well_data_id " + f'ON "{_VIEW}" (well_data_id)' + ) + ) + op.execute(text(f'CREATE INDEX ix_{_VIEW}_geom ON "{_VIEW}" USING GIST (geom)')) + + +def downgrade() -> None: + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_VIEW}"')) + # Restore the original view (without the DISTINCT ON deduplication). + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_VIEW}" AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + AND td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_VIEW}_well_data_id " + f'ON "{_VIEW}" (well_data_id)' + ) + ) + op.execute(text(f'CREATE INDEX ix_{_VIEW}_geom ON "{_VIEW}" USING GIST (geom)')) diff --git a/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py b/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py new file mode 100644 index 000000000..1f68a4fd2 --- /dev/null +++ b/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py @@ -0,0 +1,217 @@ +"""cast well_data_id to text in geothermal OGC views + +Revision ID: b4c5d6e7f8a9 +Revises: a3b4c5d6e7f8 +Create Date: 2026-06-15 + +pygeoapi does not support UUID id_field columns. Cast well_data_id to text in +both geothermal OGC views so pygeoapi can use it as the feature identifier. +""" + +from alembic import op +from sqlalchemy import text + +revision = "b4c5d6e7f8a9" +down_revision = "a3b4c5d6e7f8" +branch_labels = None +depends_on = None + +_BHT_VIEW = "ogc_geothermal_wells_bht" +_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" + + +def upgrade() -> None: + # BHT view — plain view, just DROP and recreate + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_BHT_VIEW}" AS + SELECT + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """ + ) + ) + + # Temperature profile — materialized view, DROP indexes first + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) + SELECT + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " + f'ON "{_PROFILE_VIEW}" (well_data_id)' + ) + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) + + +def downgrade() -> None: + # Restore UUID (non-text) versions + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_BHT_VIEW}" AS + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """ + ) + ) + + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) + SELECT + r."WellDataID" AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " + f'ON "{_PROFILE_VIEW}" (well_data_id)' + ) + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) diff --git a/alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py b/alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py new file mode 100644 index 000000000..ca1bdf2b8 --- /dev/null +++ b/alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py @@ -0,0 +1,215 @@ +"""add integer id to geothermal OGC views + +Revision ID: c5d6e7f8a9b0 +Revises: b4c5d6e7f8a9 +Create Date: 2026-06-15 + +All other OGC views use an integer id_field (from thing.id). pygeoapi's +PostgreSQL provider is tested against integer PKs. Replace well_data_id as the +id_field with row_number() OVER () AS id to match the convention, and keep +well_data_id as a regular attribute column. +""" + +from alembic import op +from sqlalchemy import text + +revision = "c5d6e7f8a9b0" +down_revision = "b4c5d6e7f8a9" +branch_labels = None +depends_on = None + +_BHT_VIEW = "ogc_geothermal_wells_bht" +_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" + + +def upgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_BHT_VIEW}" AS + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """ + ) + ) + + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text(f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id " f'ON "{_PROFILE_VIEW}" (id)') + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_BHT_VIEW}" AS + SELECT + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """ + ) + ) + + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute( + text( + f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) + SELECT + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """ + ) + ) + op.execute( + text( + f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " + f'ON "{_PROFILE_VIEW}" (well_data_id)' + ) + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) diff --git a/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py b/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py new file mode 100644 index 000000000..b42dfd5a4 --- /dev/null +++ b/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py @@ -0,0 +1,210 @@ +"""add FK constraints to NMW staging mirror tables + +Revision ID: z2a3b4c5d6e7 +Revises: y1z2a3b4c5d6 +Create Date: 2026-06-15 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "z2a3b4c5d6e7" +down_revision = "y1z2a3b4c5d6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # WellLocations -> WellHeaders + op.create_foreign_key( + "fk_nmw_welllocations_welldataid", + "NMW_WellLocations", + "NMW_WellHeaders", + ["WellDataID"], + ["WellDataID"], + ) + # WellRecords -> WellHeaders + op.create_foreign_key( + "fk_nmw_wellrecords_welldataid", + "NMW_WellRecords", + "NMW_WellHeaders", + ["WellDataID"], + ["WellDataID"], + ) + # WellZDatum -> WellRecords + op.create_foreign_key( + "fk_nmw_wellzdatum_recrdsetid", + "NMW_WellZDatum", + "NMW_WellRecords", + ["RecrdsetID"], + ["RecrdSetID"], + ) + # WellSamples -> WellRecords + op.create_foreign_key( + "fk_nmw_wellsamples_recrdsetid", + "NMW_WellSamples", + "NMW_WellRecords", + ["RecrdsetID"], + ["RecrdSetID"], + ) + # GtBhtHeaders -> WellSamples + op.create_foreign_key( + "fk_nmw_gtbhtheaders_samplsetid", + "NMW_GtBhtHeaders", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + # GtBhtData -> GtBhtHeaders + op.create_foreign_key( + "fk_nmw_gtbhtdata_bhtguid", + "NMW_GtBhtData", + "NMW_GtBhtHeaders", + ["BHTGUID"], + ["BHTGUID"], + ) + # WsIntervals -> WellSamples + op.create_foreign_key( + "fk_nmw_wsintervals_samplsetid", + "NMW_WsIntervals", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + # GtConductivity -> WsIntervals + op.create_foreign_key( + "fk_nmw_gtconductivity_intrvlguid", + "NMW_GtConductivity", + "NMW_WsIntervals", + ["IntrvlGUID"], + ["IntrvlGUID"], + ) + # GtHeatFlow -> WsIntervals + op.create_foreign_key( + "fk_nmw_gtheatflow_intrvlguid", + "NMW_GtHeatFlow", + "NMW_WsIntervals", + ["IntrvlGUID"], + ["IntrvlGUID"], + ) + # GtSumHeatFlow -> WellRecords + op.create_foreign_key( + "fk_nmw_gtsumheatflow_recrdsetid", + "NMW_GtSumHeatFlow", + "NMW_WellRecords", + ["RecrdSetID"], + ["RecrdSetID"], + ) + # GtSumHeatFlow -> WellSamples + op.create_foreign_key( + "fk_nmw_gtsumheatflow_samplsetid", + "NMW_GtSumHeatFlow", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + # GtTempDepths -> WellSamples + op.create_foreign_key( + "fk_nmw_gttempdepths_samplsetid", + "NMW_GtTempDepths", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + # WsDstHeaders -> WellSamples + op.create_foreign_key( + "fk_nmw_wsdstheaders_samplsetid", + "NMW_WsDstHeaders", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + # WsDstIntervals -> WsDstHeaders + op.create_foreign_key( + "fk_nmw_wsdstintervals_dstguid", + "NMW_WsDstIntervals", + "NMW_WsDstHeaders", + ["DSTGUID"], + ["DSTGUID"], + ) + # WsDstFlowHistory -> WsDstIntervals + op.create_foreign_key( + "fk_nmw_wsdstflowhistory_dstinterval", + "NMW_WsDstFlowHistory", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + # WsDstFluidProperties -> WsDstIntervals + op.create_foreign_key( + "fk_nmw_wsdstfluidproperties_dstinterval", + "NMW_WsDstFluidProperties", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + # WsDstPressure -> WsDstIntervals + op.create_foreign_key( + "fk_nmw_wsdstpressure_dstinterval", + "NMW_WsDstPressure", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + + +def downgrade() -> None: + op.drop_constraint( + "fk_nmw_wsdstpressure_dstinterval", "NMW_WsDstPressure", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsdstfluidproperties_dstinterval", + "NMW_WsDstFluidProperties", + type_="foreignkey", + ) + op.drop_constraint( + "fk_nmw_wsdstflowhistory_dstinterval", + "NMW_WsDstFlowHistory", + type_="foreignkey", + ) + op.drop_constraint( + "fk_nmw_wsdstintervals_dstguid", "NMW_WsDstIntervals", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsdstheaders_samplsetid", "NMW_WsDstHeaders", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gttempdepths_samplsetid", "NMW_GtTempDepths", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtsumheatflow_samplsetid", "NMW_GtSumHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtsumheatflow_recrdsetid", "NMW_GtSumHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtheatflow_intrvlguid", "NMW_GtHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtconductivity_intrvlguid", "NMW_GtConductivity", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsintervals_samplsetid", "NMW_WsIntervals", type_="foreignkey" + ) + op.drop_constraint("fk_nmw_gtbhtdata_bhtguid", "NMW_GtBhtData", type_="foreignkey") + op.drop_constraint( + "fk_nmw_gtbhtheaders_samplsetid", "NMW_GtBhtHeaders", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellsamples_recrdsetid", "NMW_WellSamples", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellzdatum_recrdsetid", "NMW_WellZDatum", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellrecords_welldataid", "NMW_WellRecords", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_welllocations_welldataid", "NMW_WellLocations", type_="foreignkey" + ) From 04b8ecb9dbd3d22fea952ba379625b796997c926 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Wed, 17 Jun 2026 11:17:10 -0700 Subject: [PATCH 24/41] feat: add ogc_bht_measurements OGC collection Translates the legacy MSSQL BHT query to a PostgreSQL view returning one row per individual BHT measurement (not aggregated per well). Joins NMW_GtBhtData through headers, samples, records, Z-datum filter, well headers, and locations. Exposes 5 063 features via /ogcapi/collections/bht_measurements. Co-Authored-By: Claude Sonnet 4.6 --- ...7f8a9b0c1_add_ogc_bht_measurements_view.py | 65 +++++++++++++++++++ core/pygeoapi-config.yml | 23 +++++++ 2 files changed, 88 insertions(+) create mode 100644 alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py diff --git a/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py b/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py new file mode 100644 index 000000000..ef5c037c1 --- /dev/null +++ b/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py @@ -0,0 +1,65 @@ +"""add ogc_bht_measurements view + +Revision ID: d6e7f8a9b0c1 +Revises: c5d6e7f8a9b0 +Create Date: 2026-06-17 + +Individual BHT measurement rows with well header, location, and Z-datum +filter — translated from the legacy MSSQL query against NM_Aquifer. +One row per measurement (not aggregated per well). +""" + +from alembic import op +from sqlalchemy import text + +revision = "d6e7f8a9b0c1" +down_revision = "c5d6e7f8a9b0" +branch_labels = None +depends_on = None + +_VIEW = "ogc_bht_measurements" + + +def upgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_VIEW}" AS + SELECT + d."OBJECTID" AS id, + hdr."API" AS api, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."CurOperatr" AS operator, + hdr."WellType" AS well_type, + hdr."Well_TVD" AS well_tvd, + hdr."ComplDate" AS completion_date, + hdr."CurStatus" AS current_status, + hdr."TotalDepth" AS total_depth, + hdr."Cuttings" AS cuttings, + hdr."CoreExists" AS core_exists, + loc."County" AS county, + d."Depth" AS bht_depth, + d."BHT" AS bht, + d."HrsSnceCir" AS hours_since_circulation, + d."DateMeasrd" AS date_measured, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS bh ON bh."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = bh."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 7464e6a08..66fcb7d83 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -334,3 +334,26 @@ resources: id_field: id table: ogc_geothermal_wells_temperature_profile geom_field: geom + + bht_measurements: + type: collection + title: BHT Measurements + description: Individual bottom-hole temperature measurements with well header and location data from the NM_Wells database. + keywords: [geothermal, bht, bottom-hole-temperature, measurements] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_bht_measurements + geom_field: geom From e0a2491d86e144a5e783f3993587854ff5048143 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Thu, 18 Jun 2026 10:43:13 -0700 Subject: [PATCH 25/41] feat: add ogc_temp_depth_measurements OGC collection Translates the legacy MSSQL TempDepth2_SortedWellName query to a PostgreSQL view returning one row per individual temperature-depth reading. Includes both NAD27 and NAD83 coordinates, elevation datums (GL, unspc, KB), and filters out excluded locations. Exposes 363 858 features via /ogcapi/collections/temp_depth_measurements. Co-Authored-By: Claude Sonnet 4.6 --- ...d2_add_ogc_temp_depth_measurements_view.py | 72 +++++++++++++++++++ core/pygeoapi-config.yml | 23 ++++++ 2 files changed, 95 insertions(+) create mode 100644 alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py diff --git a/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py b/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py new file mode 100644 index 000000000..74834da16 --- /dev/null +++ b/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py @@ -0,0 +1,72 @@ +"""add ogc_temp_depth_measurements view + +Revision ID: e7f8a9b0c1d2 +Revises: d6e7f8a9b0c1 +Create Date: 2026-06-18 + +Individual temperature-depth readings with well header, location, and +elevation data. Translated from the legacy MSSQL TempDepth2_SortedWellName +query against NM_Aquifer. One row per reading; locations with Exclude=1 +are filtered out. +""" + +from alembic import op +from sqlalchemy import text + +revision = "e7f8a9b0c1d2" +down_revision = "d6e7f8a9b0c1" +branch_labels = None +depends_on = None + +_VIEW = "ogc_temp_depth_measurements" + + +def upgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_VIEW}" AS + SELECT + td."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + r."SourceID" AS source_id, + s."SampleFm" AS sample_fm, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd27" AS lat_dd27, + loc."Long_dd27" AS long_dd27, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + loc."LocAccVal" AS loc_acc_val, + s."EnteredBy" AS entered_by, + s."EntryDate" AS entry_date, + td."Depth" AS depth, + s."SmpDpUnt" AS depth_unit, + td."Temp" AS temp, + td."TempUnit" AS temp_unit, + z."Elev_GL" AS elev_gl, + z."Elev_unspc" AS elev_unspc, + z."Elev_KB" AS elev_kb, + s."SampleDate" AS sample_date, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + WHERE loc."Exclude" = 0 + AND loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 66fcb7d83..cceff876f 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -357,3 +357,26 @@ resources: id_field: id table: ogc_bht_measurements geom_field: geom + + temp_depth_measurements: + type: collection + title: Temperature-Depth Measurements + description: Individual downhole temperature readings with well header, location, and elevation data from the NM_Wells database. + keywords: [geothermal, temperature, depth, measurements] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_temp_depth_measurements + geom_field: geom From 7435e6fb83e6dbb8baebf2abd0f7a63eff5db85a Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Thu, 18 Jun 2026 11:07:02 -0700 Subject: [PATCH 26/41] feat: add NMW_Sources mirror table and transfer spec Adds a 1:1 staging mirror of the NM_Wells tbl_sources publication registry. Wires it into the nmw_mirror_transfer MirrorSpec list so it loads with the rest of the NMW tables. Data loads once tbl_sources.csv is exported from the legacy SQL Server and placed in transfers/data/nma_csv_cache/. Co-Authored-By: Claude Sonnet 4.6 --- ...a9b0c1d2e3_add_nmw_sources_mirror_table.py | 45 +++++++++++++++++++ db/nmw_legacy.py | 30 ++++++++++++- transfers/nmw_mirror_transfer.py | 3 ++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py diff --git a/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py b/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py new file mode 100644 index 000000000..1a8c8d5f6 --- /dev/null +++ b/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py @@ -0,0 +1,45 @@ +"""add NMW_Sources mirror table + +Revision ID: f8a9b0c1d2e3 +Revises: e7f8a9b0c1d2 +Create Date: 2026-06-18 + +1:1 mirror of the NM_Wells tbl_sources publication/data-source registry. +Keyed by the free-text SourceID string that appears in NMW_WellRecords.SourceID. +Needed to join publication attribution (FirstAuth, PubYear, Title, etc.) +into the ogc_heat_flow view. +""" + +from alembic import op +import sqlalchemy as sa + +revision = "f8a9b0c1d2e3" +down_revision = "e7f8a9b0c1d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "NMW_Sources", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("FirstAuth", sa.String(), nullable=True), + sa.Column("PubYear", sa.String(), nullable=True), + sa.Column("Title", sa.String(), nullable=True), + sa.Column("Journal", sa.String(), nullable=True), + sa.Column("Volume", sa.String(), nullable=True), + sa.Column("PageNo", sa.String(), nullable=True), + sa.Column("ReportNo", sa.String(), nullable=True), + sa.Column("Publisher", sa.String(), nullable=True), + sa.Column("City", sa.String(), nullable=True), + sa.Column("URL", sa.String(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_Sources_SourceID", "NMW_Sources", ["SourceID"]) + + +def downgrade() -> None: + op.drop_index("ix_NMW_Sources_SourceID", table_name="NMW_Sources") + op.drop_table("NMW_Sources") diff --git a/db/nmw_legacy.py b/db/nmw_legacy.py index 4c53c32c9..c20d0cb7d 100644 --- a/db/nmw_legacy.py +++ b/db/nmw_legacy.py @@ -723,10 +723,38 @@ class NMW_WsDstPressure(Base): global_id = mapped_column("GlobalID", UUID(as_uuid=True)) # Drop +# ============================================================================= +# PUBLICATIONS +# ============================================================================= + + +class NMW_Sources(Base): + """1:1 mirror of NM_Wells ``tbl_sources`` (publication / data source registry). + + Transform target: ``publication``. Each row is a citable source keyed by + the free-text ``SourceID`` string that appears in ``NMW_WellRecords.SourceID``. + """ + + __tablename__ = "NMW_Sources" + + object_id = mapped_column("OBJECTID", Integer, primary_key=True) # identity PK + source_id = mapped_column("SourceID", String, index=True) # join key (text FK) + first_auth = mapped_column("FirstAuth", String) + pub_year = mapped_column("PubYear", String) + title = mapped_column("Title", String) + journal = mapped_column("Journal", String) + volume = mapped_column("Volume", String) + page_no = mapped_column("PageNo", String) + report_no = mapped_column("ReportNo", String) + publisher = mapped_column("Publisher", String) + city = mapped_column("City", String) + url = mapped_column("URL", String) + comments = mapped_column("Comments", String) + + # ============================================================================= # TODO(remaining "Migrate First" tables, no DDL/mapping yet) # ----------------------------------------------------------------------------- -# Publications: tbl_sources # Subsurface Library: dst_scan, log_scanned, Well_Header, well_operators # See docs/nm_wells-migration.md for the full inventory + recommendations. # ============================================================================= diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 18f930e13..89b9ee1ea 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -62,6 +62,7 @@ NMW_GtHeatFlow, NMW_GtSumHeatFlow, NMW_GtTempDepths, + NMW_Sources, NMW_WellHeaders, NMW_WellLocations, NMW_WellRecords, @@ -111,6 +112,8 @@ class MirrorSpec: MirrorSpec(NMW_WellRecords, "tbl_well_records"), MirrorSpec(NMW_WellZDatum, "tbl_well_z_datum"), MirrorSpec(NMW_WellSamples, "tbl_well_samples"), + # Publications + MirrorSpec(NMW_Sources, "tbl_sources"), # Geothermal MirrorSpec(NMW_GtBhtHeaders, "tbl_gt_bht_headers"), MirrorSpec(NMW_GtBhtData, "tbl_gt_bht_data"), From 88bd152ed3bb4b1dd5b8d705b91193b42b51f24b Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Thu, 18 Jun 2026 11:18:08 -0700 Subject: [PATCH 27/41] feat: add ogc_heat_flow OGC collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Translates the legacy MSSQL HeatFlow query to a PostgreSQL view. Includes CASE WHEN unit conversions (ft->m, TCU->SI, HFU->mW/m²), publication attribution from NMW_Sources, and a LEFT JOIN on Well_Z_Datum for elevation. County WHERE filter removed in favour of API-level filtering. Exposes 1 522 features via /ogcapi/collections/heat_flow. Co-Authored-By: Claude Sonnet 4.6 --- .../a9b0c1d2e3f4_add_ogc_heat_flow_view.py | 109 ++++++++++++++++++ core/pygeoapi-config.yml | 23 ++++ 2 files changed, 132 insertions(+) create mode 100644 alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py diff --git a/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py b/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py new file mode 100644 index 000000000..b011a7928 --- /dev/null +++ b/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py @@ -0,0 +1,109 @@ +"""add ogc_heat_flow view + +Revision ID: a9b0c1d2e3f4 +Revises: f8a9b0c1d2e3 +Create Date: 2026-06-18 + +Summary heat-flow records with well header, location, elevation, and +publication attribution. Translated from the legacy MSSQL HeatFlow query +against NM_Aquifer. IIf() unit-conversion expressions translated to +CASE WHEN. County WHERE filter removed — filter via API instead. +One row per GT_SumHeatFlow record. +""" + +from alembic import op +from sqlalchemy import text + +revision = "a9b0c1d2e3f4" +down_revision = "f8a9b0c1d2e3" +branch_labels = None +depends_on = None + +_VIEW = "ogc_heat_flow" + + +def upgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_VIEW}" AS + SELECT + shf."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd27" AS lat_dd27, + loc."Long_dd27" AS long_dd27, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + r."SourceID" AS source_id, + z."Elev_GL" AS elev_gl, + z."Elev_KB" AS elev_kb, + z."Elev_unspc" AS elev_unspc, + CASE WHEN z."DepthUnits" = 'ft' + THEN 0.3048 * z."Elev_unspc" + ELSE z."Elev_unspc" + END AS elevation_m, + z."DepthUnits" AS depth_units, + hdr."TotalDepth" AS total_depth, + CASE WHEN z."DepthUnits" = 'ft' + THEN 0.3048 * hdr."TotalDepth" + ELSE hdr."TotalDepth" + END AS total_depth_m, + shf."FromDepth" AS from_depth, + shf."ToDepth" AS to_depth, + shf."ThermlCond" AS therml_cond, + shf."TCondRange" AS tcond_range, + shf."TCondError" AS tcond_error, + shf."TCondUnit" AS tcond_unit, + CASE WHEN shf."TCondUnit" = 'TCU' + THEN 0.4184 * shf."ThermlCond" + ELSE shf."ThermlCond" + END AS tc_si, + shf."SampleType" AS sample_type, + shf."NumSamples" AS num_samples, + shf."ThermlGrad" AS therml_grad, + shf."TGradRange" AS tgrad_range, + shf."TGError" AS tg_error, + shf."GradUnit" AS grad_unit, + shf."HeatFlow" AS heat_flow, + shf."HtFlowUnit" AS ht_flow_unit, + CASE WHEN shf."HtFlowUnit" = 'HFU' + THEN 41.84 * shf."HeatFlow" + ELSE shf."HeatFlow" + END AS heat_flow_si, + shf."Quality" AS quality, + src."FirstAuth" AS first_auth, + src."PubYear" AS pub_year, + src."Title" AS title, + src."Journal" AS journal, + src."Volume" AS volume, + src."PageNo" AS page_no, + shf."HtFlowEst" AS ht_flow_est, + r."EntryDate" AS entry_date, + CASE WHEN shf."HtFlowUnit" = 'HFU' + THEN 41.84 * shf."HtFlowEst" + ELSE shf."HtFlowEst" + END AS ht_flow_est_si, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtSumHeatFlow" AS shf + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_Sources" AS src ON src."SourceID" = r."SourceID" + WHERE loc."Exclude" = 0 + AND loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index cceff876f..1723aaff4 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -380,3 +380,26 @@ resources: id_field: id table: ogc_temp_depth_measurements geom_field: geom + + heat_flow: + type: collection + title: Heat Flow + description: Summary heat-flow records with thermal conductivity, gradient, and publication attribution from the NM_Wells database. + keywords: [geothermal, heat-flow, thermal-conductivity, gradient] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_heat_flow + geom_field: geom From 87c4c7f7e36b61cb0180ec3bf387e7b2a8908f5d Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Thu, 18 Jun 2026 11:23:13 -0700 Subject: [PATCH 28/41] feat: add ogc_dst OGC collection Translates the legacy MSSQL DST query (which never executed in Access due to a broken DST_flwHstryConcat saved query). Replaces the broken cross-join with a string_agg() CTE over NMW_WsDstFlowHistory, concatenating operation descriptions per interval. GROUP BY with no aggregates translated to SELECT DISTINCT. Exposes 1 798 features via /ogcapi/collections/dst. Co-Authored-By: Claude Sonnet 4.6 --- .../versions/b0c1d2e3f4a5_add_ogc_dst_view.py | 102 ++++++++++++++++++ core/pygeoapi-config.yml | 23 ++++ 2 files changed, 125 insertions(+) create mode 100644 alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py diff --git a/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py b/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py new file mode 100644 index 000000000..376f7c106 --- /dev/null +++ b/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py @@ -0,0 +1,102 @@ +"""add ogc_dst view + +Revision ID: b0c1d2e3f4a5 +Revises: a9b0c1d2e3f4 +Create Date: 2026-06-18 + +Drill Stem Test records with well header, location, interval, and pressure +data. Translated from the legacy MSSQL DST query against NM_Aquifer. + +The original Access query referenced DST_flwHstryConcat, a broken saved +query that never executed. We replace it with a string_agg() CTE over +NMW_WsDstFlowHistory that concatenates operation descriptions per interval. + +The original GROUP BY with no aggregate functions is equivalent to +SELECT DISTINCT, implemented that way here. +""" + +from alembic import op +from sqlalchemy import text + +revision = "b0c1d2e3f4a5" +down_revision = "a9b0c1d2e3f4" +branch_labels = None +depends_on = None + +_VIEW = "ogc_dst" + + +def upgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_VIEW}" AS + WITH flow_history AS ( + SELECT + "DSTInterval", + string_agg("Operation", '; ' ORDER BY "OBJECTID") AS flow_history + FROM "NMW_WsDstFlowHistory" + GROUP BY "DSTInterval" + ) + SELECT DISTINCT + i."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + i."DSTName" AS dst_name, + dh."DSTOprator" AS dst_operator, + i."DSTNumber" AS dst_number, + i."DSTDate" AS dst_date, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + s."From_Depth" AS from_depth, + s."To_Depth" AS to_depth, + i."TargetFm" AS target_fm, + i."PackrFrom" AS packer_from, + i."PackerTo" AS packer_to, + i."SrfChokeSz" AS srf_choke_sz, + i."BotChokeSz" AS bot_choke_sz, + s."SmpDpUnt" AS depth_unit, + z."Elev_GL" AS elev_gl, + z."Elev_unspc" AS elev_unspc, + p."PrsGageDpt" AS prs_gage_dpt, + i."PipeDia" AS pipe_dia, + i."PipeLength" AS pipe_length, + fh.flow_history AS flow_history, + p."PrsInShtIn" AS init_flow, + p."FlwPrsInMin" AS flw_prs_in_min, + p."PrsFnShtIn" AS fin_flow, + p."FlwPrsFinMin" AS flw_prs_fin_min, + p."PrsInitClsdIn" AS prs_init_clsd_in, + p."InShtInMin" AS in_sht_in_min, + p."EquilPress" AS fin_shut_in, + p."FnShtInMin" AS fn_sht_in_min, + p."HydrostPrsIn" AS hydrost_prs_in, + p."HydStPrsFl" AS hyd_st_prs_fl, + dh."PressUnits" AS press_units, + p."BlankedOff" AS blanked_off, + p."FmTemp" AS fm_temp, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_WsDstIntervals" AS i + JOIN "NMW_WsDstHeaders" AS dh ON dh."DSTGUID" = i."DSTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = dh."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + LEFT JOIN "NMW_WsDstPressure" AS p ON p."DSTInterval" = i."DSTInterval" + LEFT JOIN flow_history AS fh ON fh."DSTInterval" = i."DSTInterval" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 1723aaff4..45f3bac17 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -403,3 +403,26 @@ resources: id_field: id table: ogc_heat_flow geom_field: geom + + dst: + type: collection + title: Drill Stem Tests + description: Drill stem test intervals with pressure, flow history, and well header data from the NM_Wells database. + keywords: [geothermal, dst, drill-stem-test, pressure, formation] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: PostgreSQL + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_dst + geom_field: geom From c352eded013d4f0f3a84f7fa6d2f3f575d185ef8 Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Mon, 22 Jun 2026 11:39:48 -0700 Subject: [PATCH 29/41] Update pygeoapi.py revert changes to pygeoapi --- core/pygeoapi.py | 77 +++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 44 deletions(-) diff --git a/core/pygeoapi.py b/core/pygeoapi.py index b40489f85..7783af100 100644 --- a/core/pygeoapi.py +++ b/core/pygeoapi.py @@ -206,55 +206,39 @@ def _thing_collections_block( def _pygeoapi_db_settings() -> tuple[str, str, str, str, str]: - from dotenv import dotenv_values - - # Read .env directly so stale shell vars (e.g. from conda) can't override - # PYGEOAPI_POSTGRES_* values. Shell env takes precedence only for the - # PYGEOAPI_-prefixed keys (explicit per-service override), while the - # generic POSTGRES_* fallback always comes from the file. - env_file = Path(__file__).resolve().parents[1] / ".env" - dotenv = dotenv_values(env_file) if env_file.exists() else {} - - def _resolve(pygeoapi_key: str, fallback_key: str, default: str = "") -> str: - # Priority: .env PYGEOAPI_* > shell PYGEOAPI_* > .env POSTGRES_* > - # shell POSTGRES_* > hard default. - # Shell PYGEOAPI_* comes before .env POSTGRES_* so that explicit - # per-service overrides in Docker (e.g. PYGEOAPI_POSTGRES_HOST=db) - # beat the generic localhost values in .env. - return ( - (dotenv.get(pygeoapi_key) or "").strip() - or (os.environ.get(pygeoapi_key) or "").strip() - or (dotenv.get(fallback_key) or "").strip() - or (os.environ.get(fallback_key) or "").strip() - or default - ) - - host = _resolve("PYGEOAPI_POSTGRES_HOST", "POSTGRES_HOST", "127.0.0.1") - port = _resolve("PYGEOAPI_POSTGRES_PORT", "POSTGRES_PORT", "5432") - dbname = _resolve("PYGEOAPI_POSTGRES_DB", "POSTGRES_DB", "postgres") - user = _resolve("PYGEOAPI_POSTGRES_USER", "POSTGRES_USER") + host = ( + (os.environ.get("PYGEOAPI_POSTGRES_HOST") or "").strip() + or (os.environ.get("POSTGRES_HOST") or "").strip() + or "127.0.0.1" + ) + port = ( + (os.environ.get("PYGEOAPI_POSTGRES_PORT") or "").strip() + or (os.environ.get("POSTGRES_PORT") or "").strip() + or "5432" + ) + dbname = ( + (os.environ.get("PYGEOAPI_POSTGRES_DB") or "").strip() + or (os.environ.get("POSTGRES_DB") or "").strip() + or "postgres" + ) + user = (os.environ.get("PYGEOAPI_POSTGRES_USER") or "").strip() or ( + os.environ.get("POSTGRES_USER") or "" + ).strip() if not user: raise RuntimeError( "PYGEOAPI_POSTGRES_USER or POSTGRES_USER must be set and " - "non-empty in the environment or .env file." + "non-empty to generate the pygeoapi configuration." ) - # Resolve the actual password at config-write time and embed it directly - # in the generated config file (which is already chmod 0600). This avoids - # stale shell env vars corrupting the ${VAR} expansion that pygeoapi's - # yaml_load would otherwise perform at request time. - password = _resolve("PYGEOAPI_POSTGRES_PASSWORD", "POSTGRES_PASSWORD") - if not password: + if os.environ.get("PYGEOAPI_POSTGRES_PASSWORD") is None: raise RuntimeError( - "PYGEOAPI_POSTGRES_PASSWORD or POSTGRES_PASSWORD must be set " - "and non-empty in the environment or .env file." + "PYGEOAPI_POSTGRES_PASSWORD must be set to " + "generate the pygeoapi configuration." ) - return host, port, dbname, user, password + return host, port, dbname, user, "${PYGEOAPI_POSTGRES_PASSWORD}" def _write_config(path: Path) -> None: - host, port, dbname, user, password = _pygeoapi_db_settings() - # Escape braces so str.format() doesn't misinterpret them in the password. - password_for_format = password.replace("{", "{{").replace("}", "}}") + host, port, dbname, user, password_placeholder = _pygeoapi_db_settings() template = _template_path().read_text(encoding="utf-8") config = template.format( server_url=_server_url(), @@ -262,19 +246,24 @@ def _write_config(path: Path) -> None: postgres_port=port, postgres_db=dbname, postgres_user=user, - postgres_password_env=password_for_format, + postgres_password_env=password_placeholder, thing_collections_block=_thing_collections_block( host=host, port=port, dbname=dbname, user=user, - password_placeholder=password, + password_placeholder=password_placeholder, ), ) - # NOTE: The generated runtime config file contains database credentials - # including the plaintext password. It is protected by chmod 0600. + # NOTE: The generated runtime config file at + # `${PYGEOAPI_RUNTIME_DIR}/pygeoapi-config.yml` (default: + # `/tmp/pygeoapi/pygeoapi-config.yml`) contains database connection details + # (host, port, dbname, user). Although the password is expected to be + # provided via environment variables at runtime by pygeoapi, this file + # should still be treated as sensitive configuration: # * Do not commit it to version control. # * Do not expose it in logs, error messages, or diagnostics. + # * Ensure filesystem permissions restrict access appropriately. path.write_text(config, encoding="utf-8") path.chmod(0o600) From 451ebb7aef6ac16567f7a6994f7a8b6b0f6d638c Mon Sep 17 00:00:00 2001 From: Peter Rowland Date: Mon, 22 Jun 2026 13:09:47 -0700 Subject: [PATCH 30/41] simplify migrations simplified migrations and change order of mirror transfers to start with well headers --- ...ure_profile_view_duplicate_well_data_id.py | 132 ---- .../a9b0c1d2e3f4_add_ogc_heat_flow_view.py | 109 --- .../versions/b0c1d2e3f4a5_add_ogc_dst_view.py | 102 --- ...data_id_to_text_in_geothermal_ogc_views.py | 217 ------ .../c0d1e2f3a4b5_nmw_mirror_tables.py | 723 ++++++++++++++++++ ...b5c6_nmw_per_well_geothermal_ogc_views.py} | 169 ++-- ...7f8a9b0c1_add_ogc_bht_measurements_view.py | 65 -- .../e2f3a4b5c6d7_nmw_measurement_ogc_views.py | 295 +++++++ ...d2_add_ogc_temp_depth_measurements_view.py | 72 -- ...a9b0c1d2e3_add_nmw_sources_mirror_table.py | 45 -- ...x0y1z2_nmw_legacy_staging_mirror_tables.py | 207 ----- ...y1z2a3_nmw_geothermal_dst_mirror_tables.py | 330 -------- .../w9x0y1z2a3b4_add_geothermal_ogc_views.py | 173 ----- ...3b4c5_add_geothermal_heat_flow_ogc_view.py | 109 --- ..._geothermal_interval_heat_flow_ogc_view.py | 112 --- ...add_fk_constraints_to_nmw_mirror_tables.py | 210 ----- transfers/nmw_mirror_transfer.py | 35 +- 17 files changed, 1146 insertions(+), 1959 deletions(-) delete mode 100644 alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py delete mode 100644 alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py delete mode 100644 alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py delete mode 100644 alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py create mode 100644 alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py rename alembic/versions/{c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py => d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py} (52%) delete mode 100644 alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py create mode 100644 alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py delete mode 100644 alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py delete mode 100644 alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py delete mode 100644 alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py delete mode 100644 alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py delete mode 100644 alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py delete mode 100644 alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py delete mode 100644 alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py delete mode 100644 alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py diff --git a/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py b/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py deleted file mode 100644 index dc2cd4642..000000000 --- a/alembic/versions/a3b4c5d6e7f8_fix_temperature_profile_view_duplicate_well_data_id.py +++ /dev/null @@ -1,132 +0,0 @@ -"""fix temperature profile view duplicate well_data_id - -Revision ID: a3b4c5d6e7f8 -Revises: z2a3b4c5d6e7 -Create Date: 2026-06-15 - -NMW_WellLocations has multiple rows per WellDataID (OBJECTID is its PK, not -WellDataID). The prior view grouped by WellDataID + Lat_dd83 + Long_dd83, -producing one row per (well, location) pair. When a well has more than one -location row the unique index on well_data_id fails at REFRESH time. - -Fix: deduplicate NMW_WellLocations to one row per WellDataID via DISTINCT ON -before joining, so the GROUP BY always yields exactly one row per well. -""" - -from alembic import op -from sqlalchemy import text - -# revision identifiers, used by Alembic. -revision = "a3b4c5d6e7f8" -down_revision = "z2a3b4c5d6e7" -branch_labels = None -depends_on = None - -_VIEW = "ogc_geothermal_wells_temperature_profile" - - -def upgrade() -> None: - op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_VIEW}"')) - op.execute( - text( - f""" - CREATE MATERIALIZED VIEW "{_VIEW}" AS - WITH loc AS ( - SELECT DISTINCT ON ("WellDataID") - "WellDataID", "Lat_dd83", "Long_dd83" - FROM "NMW_WellLocations" - WHERE "Lat_dd83" IS NOT NULL - AND "Long_dd83" IS NOT NULL - ORDER BY "WellDataID", "OBJECTID" - ) - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, - json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - ) - ) - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_VIEW}_well_data_id " - f'ON "{_VIEW}" (well_data_id)' - ) - ) - op.execute(text(f'CREATE INDEX ix_{_VIEW}_geom ON "{_VIEW}" USING GIST (geom)')) - - -def downgrade() -> None: - op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_VIEW}"')) - # Restore the original view (without the DISTINCT ON deduplication). - op.execute( - text( - f""" - CREATE MATERIALIZED VIEW "{_VIEW}" AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, - json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - AND td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - ) - ) - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_VIEW}_well_data_id " - f'ON "{_VIEW}" (well_data_id)' - ) - ) - op.execute(text(f'CREATE INDEX ix_{_VIEW}_geom ON "{_VIEW}" USING GIST (geom)')) diff --git a/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py b/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py deleted file mode 100644 index b011a7928..000000000 --- a/alembic/versions/a9b0c1d2e3f4_add_ogc_heat_flow_view.py +++ /dev/null @@ -1,109 +0,0 @@ -"""add ogc_heat_flow view - -Revision ID: a9b0c1d2e3f4 -Revises: f8a9b0c1d2e3 -Create Date: 2026-06-18 - -Summary heat-flow records with well header, location, elevation, and -publication attribution. Translated from the legacy MSSQL HeatFlow query -against NM_Aquifer. IIf() unit-conversion expressions translated to -CASE WHEN. County WHERE filter removed — filter via API instead. -One row per GT_SumHeatFlow record. -""" - -from alembic import op -from sqlalchemy import text - -revision = "a9b0c1d2e3f4" -down_revision = "f8a9b0c1d2e3" -branch_labels = None -depends_on = None - -_VIEW = "ogc_heat_flow" - - -def upgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_VIEW}" AS - SELECT - shf."OBJECTID" AS id, - hdr."CurWellNam" AS well_name, - hdr."CurWellNum" AS well_num, - hdr."API" AS api, - loc."County" AS county, - loc."State" AS state, - loc."Lat_dd27" AS lat_dd27, - loc."Long_dd27" AS long_dd27, - loc."Lat_dd83" AS lat_dd83, - loc."Long_dd83" AS long_dd83, - r."SourceID" AS source_id, - z."Elev_GL" AS elev_gl, - z."Elev_KB" AS elev_kb, - z."Elev_unspc" AS elev_unspc, - CASE WHEN z."DepthUnits" = 'ft' - THEN 0.3048 * z."Elev_unspc" - ELSE z."Elev_unspc" - END AS elevation_m, - z."DepthUnits" AS depth_units, - hdr."TotalDepth" AS total_depth, - CASE WHEN z."DepthUnits" = 'ft' - THEN 0.3048 * hdr."TotalDepth" - ELSE hdr."TotalDepth" - END AS total_depth_m, - shf."FromDepth" AS from_depth, - shf."ToDepth" AS to_depth, - shf."ThermlCond" AS therml_cond, - shf."TCondRange" AS tcond_range, - shf."TCondError" AS tcond_error, - shf."TCondUnit" AS tcond_unit, - CASE WHEN shf."TCondUnit" = 'TCU' - THEN 0.4184 * shf."ThermlCond" - ELSE shf."ThermlCond" - END AS tc_si, - shf."SampleType" AS sample_type, - shf."NumSamples" AS num_samples, - shf."ThermlGrad" AS therml_grad, - shf."TGradRange" AS tgrad_range, - shf."TGError" AS tg_error, - shf."GradUnit" AS grad_unit, - shf."HeatFlow" AS heat_flow, - shf."HtFlowUnit" AS ht_flow_unit, - CASE WHEN shf."HtFlowUnit" = 'HFU' - THEN 41.84 * shf."HeatFlow" - ELSE shf."HeatFlow" - END AS heat_flow_si, - shf."Quality" AS quality, - src."FirstAuth" AS first_auth, - src."PubYear" AS pub_year, - src."Title" AS title, - src."Journal" AS journal, - src."Volume" AS volume, - src."PageNo" AS page_no, - shf."HtFlowEst" AS ht_flow_est, - r."EntryDate" AS entry_date, - CASE WHEN shf."HtFlowUnit" = 'HFU' - THEN 41.84 * shf."HtFlowEst" - ELSE shf."HtFlowEst" - END AS ht_flow_est_si, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtSumHeatFlow" AS shf - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" - JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" - JOIN "NMW_Sources" AS src ON src."SourceID" = r."SourceID" - WHERE loc."Exclude" = 0 - AND loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - """ - ) - ) - - -def downgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py b/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py deleted file mode 100644 index 376f7c106..000000000 --- a/alembic/versions/b0c1d2e3f4a5_add_ogc_dst_view.py +++ /dev/null @@ -1,102 +0,0 @@ -"""add ogc_dst view - -Revision ID: b0c1d2e3f4a5 -Revises: a9b0c1d2e3f4 -Create Date: 2026-06-18 - -Drill Stem Test records with well header, location, interval, and pressure -data. Translated from the legacy MSSQL DST query against NM_Aquifer. - -The original Access query referenced DST_flwHstryConcat, a broken saved -query that never executed. We replace it with a string_agg() CTE over -NMW_WsDstFlowHistory that concatenates operation descriptions per interval. - -The original GROUP BY with no aggregate functions is equivalent to -SELECT DISTINCT, implemented that way here. -""" - -from alembic import op -from sqlalchemy import text - -revision = "b0c1d2e3f4a5" -down_revision = "a9b0c1d2e3f4" -branch_labels = None -depends_on = None - -_VIEW = "ogc_dst" - - -def upgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_VIEW}" AS - WITH flow_history AS ( - SELECT - "DSTInterval", - string_agg("Operation", '; ' ORDER BY "OBJECTID") AS flow_history - FROM "NMW_WsDstFlowHistory" - GROUP BY "DSTInterval" - ) - SELECT DISTINCT - i."OBJECTID" AS id, - hdr."CurWellNam" AS well_name, - hdr."CurWellNum" AS well_num, - hdr."API" AS api, - i."DSTName" AS dst_name, - dh."DSTOprator" AS dst_operator, - i."DSTNumber" AS dst_number, - i."DSTDate" AS dst_date, - loc."County" AS county, - loc."State" AS state, - loc."Lat_dd83" AS lat_dd83, - loc."Long_dd83" AS long_dd83, - s."From_Depth" AS from_depth, - s."To_Depth" AS to_depth, - i."TargetFm" AS target_fm, - i."PackrFrom" AS packer_from, - i."PackerTo" AS packer_to, - i."SrfChokeSz" AS srf_choke_sz, - i."BotChokeSz" AS bot_choke_sz, - s."SmpDpUnt" AS depth_unit, - z."Elev_GL" AS elev_gl, - z."Elev_unspc" AS elev_unspc, - p."PrsGageDpt" AS prs_gage_dpt, - i."PipeDia" AS pipe_dia, - i."PipeLength" AS pipe_length, - fh.flow_history AS flow_history, - p."PrsInShtIn" AS init_flow, - p."FlwPrsInMin" AS flw_prs_in_min, - p."PrsFnShtIn" AS fin_flow, - p."FlwPrsFinMin" AS flw_prs_fin_min, - p."PrsInitClsdIn" AS prs_init_clsd_in, - p."InShtInMin" AS in_sht_in_min, - p."EquilPress" AS fin_shut_in, - p."FnShtInMin" AS fn_sht_in_min, - p."HydrostPrsIn" AS hydrost_prs_in, - p."HydStPrsFl" AS hyd_st_prs_fl, - dh."PressUnits" AS press_units, - p."BlankedOff" AS blanked_off, - p."FmTemp" AS fm_temp, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_WsDstIntervals" AS i - JOIN "NMW_WsDstHeaders" AS dh ON dh."DSTGUID" = i."DSTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = dh."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" - LEFT JOIN "NMW_WsDstPressure" AS p ON p."DSTInterval" = i."DSTInterval" - LEFT JOIN flow_history AS fh ON fh."DSTInterval" = i."DSTInterval" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - """ - ) - ) - - -def downgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py b/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py deleted file mode 100644 index 1f68a4fd2..000000000 --- a/alembic/versions/b4c5d6e7f8a9_cast_well_data_id_to_text_in_geothermal_ogc_views.py +++ /dev/null @@ -1,217 +0,0 @@ -"""cast well_data_id to text in geothermal OGC views - -Revision ID: b4c5d6e7f8a9 -Revises: a3b4c5d6e7f8 -Create Date: 2026-06-15 - -pygeoapi does not support UUID id_field columns. Cast well_data_id to text in -both geothermal OGC views so pygeoapi can use it as the feature identifier. -""" - -from alembic import op -from sqlalchemy import text - -revision = "b4c5d6e7f8a9" -down_revision = "a3b4c5d6e7f8" -branch_labels = None -depends_on = None - -_BHT_VIEW = "ogc_geothermal_wells_bht" -_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" - - -def upgrade() -> None: - # BHT view — plain view, just DROP and recreate - op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_BHT_VIEW}" AS - SELECT - r."WellDataID"::text AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - hdr."TotalDepth" AS total_depth, - count(d.*) AS bht_count, - max(d."BHT") AS max_bht, - min(d."BHT") AS min_bht, - max(d."Depth") AS max_bht_depth, - max(d."TempUnit") AS temp_unit, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtBhtData" AS d - JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API", - hdr."TotalDepth" - """ - ) - ) - - # Temperature profile — materialized view, DROP indexes first - op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) - op.execute( - text( - f""" - CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS - WITH loc AS ( - SELECT DISTINCT ON ("WellDataID") - "WellDataID", "Lat_dd83", "Long_dd83" - FROM "NMW_WellLocations" - WHERE "Lat_dd83" IS NOT NULL - AND "Long_dd83" IS NOT NULL - ORDER BY "WellDataID", "OBJECTID" - ) - SELECT - r."WellDataID"::text AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, - json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - ) - ) - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " - f'ON "{_PROFILE_VIEW}" (well_data_id)' - ) - ) - op.execute( - text( - f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' - ) - ) - - -def downgrade() -> None: - # Restore UUID (non-text) versions - op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_BHT_VIEW}" AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - hdr."TotalDepth" AS total_depth, - count(d.*) AS bht_count, - max(d."BHT") AS max_bht, - min(d."BHT") AS min_bht, - max(d."Depth") AS max_bht_depth, - max(d."TempUnit") AS temp_unit, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtBhtData" AS d - JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API", - hdr."TotalDepth" - """ - ) - ) - - op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) - op.execute( - text( - f""" - CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS - WITH loc AS ( - SELECT DISTINCT ON ("WellDataID") - "WellDataID", "Lat_dd83", "Long_dd83" - FROM "NMW_WellLocations" - WHERE "Lat_dd83" IS NOT NULL - AND "Long_dd83" IS NOT NULL - ORDER BY "WellDataID", "OBJECTID" - ) - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, - json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - ) - ) - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " - f'ON "{_PROFILE_VIEW}" (well_data_id)' - ) - ) - op.execute( - text( - f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' - ) - ) diff --git a/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py b/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py new file mode 100644 index 000000000..42a8cc8ba --- /dev/null +++ b/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py @@ -0,0 +1,723 @@ +"""NMW staging mirror tables and FK constraints + +Revision ID: c0d1e2f3a4b5 +Revises: t6u7v8w9x0y1 +Create Date: 2026-06-22 + +1:1 staging mirror of the legacy NM_Wells SQL Server tables needed for the +geothermal OGC collections (see db/nmw_legacy.py and docs/nm_wells-migration.md). +Faithful, column-for-column copies; the transform into the Ocotillo data model +is a later phase. + +Core well tables: + tbl_well_locations -> NMW_WellLocations + tbl_well_headers -> NMW_WellHeaders + tbl_well_records -> NMW_WellRecords + tbl_well_z_datum -> NMW_WellZDatum + tbl_well_samples -> NMW_WellSamples + +Geothermal tables: + tbl_gt_bht_headers -> NMW_GtBhtHeaders + tbl_gt_bht_data -> NMW_GtBhtData + tbl_ws_intervals -> NMW_WsIntervals + tbl_gt_conductivity -> NMW_GtConductivity + tbl_gt_heat_flow -> NMW_GtHeatFlow + tbl_gt_sum_heat_flow -> NMW_GtSumHeatFlow + tbl_gt_temp_depths -> NMW_GtTempDepths + +Drill Stem Test tables: + tbl_ws_dst_headers -> NMW_WsDstHeaders + tbl_ws_dst_intervals -> NMW_WsDstIntervals + tbl_ws_dst_flow_history -> NMW_WsDstFlowHistory + tbl_ws_dst_fluid_properties -> NMW_WsDstFluidProperties + tbl_ws_dst_pressure -> NMW_WsDstPressure + +Publication/source registry: + tbl_sources -> NMW_Sources + (Keyed by free-text SourceID; joined into ogc_heat_flow for attribution.) +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "c0d1e2f3a4b5" +down_revision: Union[str, Sequence[str], None] = "t6u7v8w9x0y1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ------------------------------------------------------------------ + # Core well tables + # ------------------------------------------------------------------ + op.create_table( + "NMW_WellLocations", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Well_ID", sa.String(), nullable=True), + sa.Column("Import_ID", sa.Integer(), nullable=True), + sa.Column("Township", sa.Float(), nullable=True), + sa.Column("NorS_TDir", sa.String(), nullable=True), + sa.Column("Range", sa.Float(), nullable=True), + sa.Column("EorW_RDir", sa.String(), nullable=True), + sa.Column("Sectn", sa.SmallInteger(), nullable=True), + sa.Column("SectnPart", sa.String(), nullable=True), + sa.Column("UnitLetter", sa.String(), nullable=True), + sa.Column("UTM_zone", sa.String(), nullable=True), + sa.Column("State", sa.String(), nullable=True), + sa.Column("County", sa.String(), nullable=True), + sa.Column("Basin", sa.String(), nullable=True), + sa.Column("Footage_NS", sa.Float(), nullable=True), + sa.Column("NorS_FDir", sa.String(), nullable=True), + sa.Column("Footage_EW", sa.Float(), nullable=True), + sa.Column("EorW_FDir", sa.String(), nullable=True), + sa.Column("Lat_min", sa.SmallInteger(), nullable=True), + sa.Column("Lat_sec", sa.Float(), nullable=True), + sa.Column("Long_deg", sa.SmallInteger(), nullable=True), + sa.Column("Long_min", sa.SmallInteger(), nullable=True), + sa.Column("Long_sec", sa.Float(), nullable=True), + sa.Column("Lat_dd27", sa.Float(), nullable=True), + sa.Column("Long_dd27", sa.Float(), nullable=True), + sa.Column("Lat_dd83", sa.Float(), nullable=True), + sa.Column("Long_dd83", sa.Float(), nullable=True), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("SourceDatum", sa.String(), nullable=True), + sa.Column("SourceUnits", sa.String(), nullable=True), + sa.Column("LocAccType", sa.String(), nullable=True), + sa.Column("LocAccMeas", sa.String(), nullable=True), + sa.Column("LocAccVal", sa.Float(), nullable=True), + sa.Column("Duplicated", sa.SmallInteger(), nullable=True), + sa.Column("Exclude", sa.SmallInteger(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("API", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WellLocations_WellDataID", "NMW_WellLocations", ["WellDataID"] + ) + + op.create_table( + "NMW_WellHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("WellSpotID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("API", sa.String(), nullable=True), + sa.Column("WellClass", sa.String(), nullable=True), + sa.Column("WellType", sa.String(), nullable=True), + sa.Column("WellOrient", sa.String(), nullable=True), + sa.Column("CurWellNam", sa.String(), nullable=True), + sa.Column("CurWellNum", sa.String(), nullable=True), + sa.Column("CurStatus", sa.String(), nullable=True), + sa.Column("PrdPoolCnt", sa.SmallInteger(), nullable=True), + sa.Column("CurOperatr", sa.String(), nullable=True), + sa.Column("CurOwner", sa.String(), nullable=True), + sa.Column("TotalDepth", sa.Float(), nullable=True), + sa.Column("Well_TVD", sa.Float(), nullable=True), + sa.Column("Fm_TD", sa.String(), nullable=True), + sa.Column("Age_TD", sa.String(), nullable=True), + sa.Column("SpudDate", sa.DateTime(), nullable=True), + sa.Column("ComplDate", sa.DateTime(), nullable=True), + sa.Column("PlugDate", sa.DateTime(), nullable=True), + sa.Column("PlugBack", sa.Float(), nullable=True), + sa.Column("BridgePlug", sa.String(), nullable=True), + sa.Column("ScoutTickt", sa.SmallInteger(), nullable=True), + sa.Column("DwnHoleSur", sa.SmallInteger(), nullable=True), + sa.Column("GeolLog", sa.SmallInteger(), nullable=True), + sa.Column("Geophyslog", sa.SmallInteger(), nullable=True), + sa.Column("GthrmExist", sa.SmallInteger(), nullable=True), + sa.Column("PetroData", sa.SmallInteger(), nullable=True), + sa.Column("CoreExists", sa.SmallInteger(), nullable=True), + sa.Column("Cuttings", sa.SmallInteger(), nullable=True), + sa.Column("SampleData", sa.SmallInteger(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("Import_ID", sa.String(), nullable=True), + sa.Column("Import_DB", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("WellDataID"), + ) + + op.create_table( + "NMW_WellRecords", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("RecrdClass", sa.String(), nullable=True), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("ActionDate", sa.DateTime(), nullable=True), + sa.Column("WellName", sa.String(), nullable=True), + sa.Column("WellNumber", sa.String(), nullable=True), + sa.Column("API_suffix", sa.String(), nullable=True), + sa.Column("EnteredBy", sa.String(), nullable=True), + sa.Column("EntryDate", sa.DateTime(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("RecrdSetID"), + ) + op.create_index("ix_NMW_WellRecords_WellDataID", "NMW_WellRecords", ["WellDataID"]) + + op.create_table( + "NMW_WellZDatum", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Elev_GL", sa.Float(), nullable=True), + sa.Column("Elev_DF", sa.Float(), nullable=True), + sa.Column("Elev_KB", sa.Float(), nullable=True), + sa.Column("Elev_unspc", sa.Float(), nullable=True), + sa.Column("DatumElev", sa.Float(), nullable=True), + sa.Column("DepthDatum", sa.String(), nullable=True), + sa.Column("DepthUnits", sa.String(), nullable=True), + sa.Column("Z_datum", sa.String(), nullable=True), + sa.Column("Z_units", sa.String(), nullable=True), + sa.Column("ElevSource", sa.String(), nullable=True), + sa.Column("ElvAccType", sa.String(), nullable=True), + sa.Column("ElvAccMeas", sa.String(), nullable=True), + sa.Column("ElvAccVal", sa.Float(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_WellZDatum_RecrdsetID", "NMW_WellZDatum", ["RecrdsetID"]) + + op.create_table( + "NMW_WellSamples", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SmpSetName", sa.String(), nullable=True), + sa.Column("SamplClass", sa.String(), nullable=True), + sa.Column("SampleType", sa.String(), nullable=True), + sa.Column("SampleFm", sa.String(), nullable=True), + sa.Column("SampleLoc", sa.String(), nullable=True), + sa.Column("SampleDate", sa.DateTime(), nullable=True), + sa.Column("From_Depth", sa.Float(), nullable=True), + sa.Column("To_Depth", sa.Float(), nullable=True), + sa.Column("SmpDpUnt", sa.String(), nullable=True), + sa.Column("From_TVD", sa.Float(), nullable=True), + sa.Column("To_TVD", sa.Float(), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("Porosity", sa.SmallInteger(), nullable=True), + sa.Column("Permeablty", sa.SmallInteger(), nullable=True), + sa.Column("Density", sa.SmallInteger(), nullable=True), + sa.Column("DST_Tests", sa.SmallInteger(), nullable=True), + sa.Column("ThinSect", sa.SmallInteger(), nullable=True), + sa.Column("Geochron", sa.SmallInteger(), nullable=True), + sa.Column("Geochem", sa.SmallInteger(), nullable=True), + sa.Column("Geothermal", sa.SmallInteger(), nullable=True), + sa.Column("WholeRock", sa.SmallInteger(), nullable=True), + sa.Column("Paleontlgy", sa.SmallInteger(), nullable=True), + sa.Column("EnteredBy", sa.String(), nullable=True), + sa.Column("EntryDate", sa.DateTime(), nullable=True), + sa.Column("Notes", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("SamplSetID"), + ) + op.create_index("ix_NMW_WellSamples_RecrdsetID", "NMW_WellSamples", ["RecrdsetID"]) + + # ------------------------------------------------------------------ + # Geothermal tables + # ------------------------------------------------------------------ + op.create_table( + "NMW_GtBhtHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("BoreDia", sa.Float(), nullable=True), + sa.Column("BoreUnits", sa.String(length=16), nullable=True), + sa.Column("DrillFluid", sa.String(length=16), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("FldSalinity", sa.Float(), nullable=True), + sa.Column("FldRstvity", sa.Float(), nullable=True), + sa.Column("Fluid_pH", sa.Float(), nullable=True), + sa.Column("FldDensity", sa.Float(), nullable=True), + sa.Column("FldLevel", sa.Float(), nullable=True), + sa.Column("FldViscsty", sa.Float(), nullable=True), + sa.Column("FluidLoss", sa.String(length=50), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("BHTGUID"), + ) + op.create_index( + "ix_NMW_GtBhtHeaders_SamplSetID", "NMW_GtBhtHeaders", ["SamplSetID"] + ) + + op.create_table( + "NMW_GtBhtData", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Depth", sa.Float(), nullable=True), + sa.Column("BHT", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=5), nullable=True), + sa.Column("HrsSnceCir", sa.Float(), nullable=True), + sa.Column("DateMeasrd", sa.DateTime(), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_GtBhtData_BHTGUID", "NMW_GtBhtData", ["BHTGUID"]) + + op.create_table( + "NMW_WsIntervals", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SampleID", sa.String(length=128), nullable=True), + sa.Column("From_Depth", sa.Float(), nullable=True), + sa.Column("To_Depth", sa.Float(), nullable=True), + sa.Column("From_TVD", sa.Float(), nullable=True), + sa.Column("To_TVD", sa.Float(), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("Intv_Notes", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("IntrvlGUID"), + ) + op.create_index("ix_NMW_WsIntervals_SamplSetID", "NMW_WsIntervals", ["SamplSetID"]) + + op.create_table( + "NMW_GtConductivity", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Cnductvity", sa.Float(), nullable=True), + sa.Column("CnductUnit", sa.String(length=3), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtConductivity_IntrvlGUID", "NMW_GtConductivity", ["IntrvlGUID"] + ) + + op.create_table( + "NMW_GtHeatFlow", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Gradient", sa.Float(), nullable=True), + sa.Column("Ka", sa.Float(), nullable=True), + sa.Column("Ka_unit", sa.String(length=3), nullable=True), + sa.Column("Pm", sa.Float(), nullable=True), + sa.Column("Kpr", sa.Float(), nullable=True), + sa.Column("Kpr_unit", sa.String(length=3), nullable=True), + sa.Column("Q", sa.Float(), nullable=True), + sa.Column("Q_unit", sa.String(length=3), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_GtHeatFlow_IntrvlGUID", "NMW_GtHeatFlow", ["IntrvlGUID"]) + + op.create_table( + "NMW_GtSumHeatFlow", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("LithClass", sa.String(length=50), nullable=True), + sa.Column("UnitBasis", sa.String(length=16), nullable=True), + sa.Column("UnitName", sa.String(length=128), nullable=True), + sa.Column("GeoID", sa.String(length=16), nullable=True), + sa.Column("FromDepth", sa.Float(), nullable=True), + sa.Column("ToDepth", sa.Float(), nullable=True), + sa.Column("DepthUnit", sa.String(length=8), nullable=True), + sa.Column("From_Elev", sa.Float(), nullable=True), + sa.Column("To_Elev", sa.Float(), nullable=True), + sa.Column("ThermlGrad", sa.Float(), nullable=True), + sa.Column("TGError", sa.Float(), nullable=True), + sa.Column("GradUnit", sa.String(length=3), nullable=True), + sa.Column("TGradRange", sa.String(length=15), nullable=True), + sa.Column("SampleType", sa.String(length=50), nullable=True), + sa.Column("NumSamples", sa.SmallInteger(), nullable=True), + sa.Column("ThermlCond", sa.Float(), nullable=True), + sa.Column("TCondError", sa.Float(), nullable=True), + sa.Column("TCondUnit", sa.String(length=3), nullable=True), + sa.Column("TCondRange", sa.String(length=15), nullable=True), + sa.Column("HeatFlow", sa.Float(), nullable=True), + sa.Column("HtFlowErr", sa.Float(), nullable=True), + sa.Column("HtFlowUnit", sa.String(length=3), nullable=True), + sa.Column("HtFlowEst", sa.Float(), nullable=True), + sa.Column("Quality", sa.String(length=50), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtSumHeatFlow_RecrdSetID", "NMW_GtSumHeatFlow", ["RecrdSetID"] + ) + op.create_index( + "ix_NMW_GtSumHeatFlow_SamplSetID", "NMW_GtSumHeatFlow", ["SamplSetID"] + ) + + op.create_table( + "NMW_GtTempDepths", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Depth", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("IntrvlGrad", sa.Float(), nullable=True), + sa.Column("Comments", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_GtTempDepths_SamplSetID", "NMW_GtTempDepths", ["SamplSetID"] + ) + + # ------------------------------------------------------------------ + # Drill Stem Test tables + # ------------------------------------------------------------------ + op.create_table( + "NMW_WsDstHeaders", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("TestType", sa.String(length=50), nullable=True), + sa.Column("DSTOprator", sa.String(length=50), nullable=True), + sa.Column("PressUnits", sa.String(length=8), nullable=True), + sa.Column("TempUnit", sa.String(length=1), nullable=True), + sa.Column("PipeDiaUnt", sa.String(length=8), nullable=True), + sa.Column("PipeLenUnt", sa.String(length=8), nullable=True), + sa.Column("ChokeSizUn", sa.String(length=8), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("DSTGUID"), + ) + op.create_index( + "ix_NMW_WsDstHeaders_SamplSetID", "NMW_WsDstHeaders", ["SamplSetID"] + ) + + op.create_table( + "NMW_WsDstIntervals", + sa.Column("OBJECTID", sa.Integer(), nullable=True), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("DSTName", sa.String(length=128), nullable=True), + sa.Column("TargetFm", sa.String(length=16), nullable=True), + sa.Column("DSTDate", sa.DateTime(), nullable=True), + sa.Column("DSTNumber", sa.SmallInteger(), nullable=True), + sa.Column("Status", sa.String(length=255), nullable=True), + sa.Column("StatusDate", sa.DateTime(), nullable=True), + sa.Column("PackrFrom", sa.Float(), nullable=True), + sa.Column("PackerTo", sa.Float(), nullable=True), + sa.Column("SrfChokeSz", sa.Float(), nullable=True), + sa.Column("BotChokeSz", sa.Float(), nullable=True), + sa.Column("PipeDia", sa.Float(), nullable=True), + sa.Column("PipeLength", sa.Float(), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("DSTInterval"), + ) + op.create_index("ix_NMW_WsDstIntervals_DSTGUID", "NMW_WsDstIntervals", ["DSTGUID"]) + + op.create_table( + "NMW_WsDstFlowHistory", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("Operation", sa.String(length=255), nullable=True), + sa.Column("StartTime", sa.DateTime(), nullable=True), + sa.Column("EndTime", sa.DateTime(), nullable=True), + sa.Column("Duration", sa.Float(), nullable=True), + sa.Column("Pressure", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("RecovColmn", sa.Float(), nullable=True), + sa.Column("RecovType", sa.String(length=255), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstFlowHistory_DSTInterval", "NMW_WsDstFlowHistory", ["DSTInterval"] + ) + + op.create_table( + "NMW_WsDstFluidProperties", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("SourceLoc", sa.String(length=255), nullable=True), + sa.Column("Resistivty", sa.Float(), nullable=True), + sa.Column("Temp", sa.Float(), nullable=True), + sa.Column("Chlorides", sa.Float(), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstFluidProperties_DSTInterval", + "NMW_WsDstFluidProperties", + ["DSTInterval"], + ) + + op.create_table( + "NMW_WsDstPressure", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("PrsGageDpt", sa.Float(), nullable=True), + sa.Column("BlankedOff", sa.SmallInteger(), nullable=True), + sa.Column("InShtInMin", sa.Float(), nullable=True), + sa.Column("FlwPrsInMin", sa.Float(), nullable=True), + sa.Column("PrsInShtIn", sa.Float(), nullable=True), + sa.Column("PrsInitClsdIn", sa.Float(), nullable=True), + sa.Column("FnShtInMin", sa.Float(), nullable=True), + sa.Column("FlwPrsFinMin", sa.Float(), nullable=True), + sa.Column("PrsFnShtIn", sa.Float(), nullable=True), + sa.Column("ShtInPrMth", sa.String(length=255), nullable=True), + sa.Column("HydrostPrsIn", sa.Float(), nullable=True), + sa.Column("HydStPrsFl", sa.Float(), nullable=True), + sa.Column("HydstPrMth", sa.String(length=255), nullable=True), + sa.Column("EquilPress", sa.Float(), nullable=True), + sa.Column("EqlPrsMth", sa.String(length=255), nullable=True), + sa.Column("FlowPrsMin", sa.Float(), nullable=True), + sa.Column("FlowPrsMax", sa.Float(), nullable=True), + sa.Column("FlowPrsMth", sa.String(length=255), nullable=True), + sa.Column("DSTFluid", sa.String(length=128), nullable=True), + sa.Column("FmTemp", sa.Float(), nullable=True), + sa.Column("TempCorrtn", sa.Float(), nullable=True), + sa.Column("TempFlowng", sa.Float(), nullable=True), + sa.Column("TempUnit", sa.String(length=5), nullable=True), + sa.Column("Notes", sa.String(length=255), nullable=True), + sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index( + "ix_NMW_WsDstPressure_DSTInterval", "NMW_WsDstPressure", ["DSTInterval"] + ) + + # ------------------------------------------------------------------ + # Publication/source registry + # ------------------------------------------------------------------ + op.create_table( + "NMW_Sources", + sa.Column("OBJECTID", sa.Integer(), nullable=False), + sa.Column("SourceID", sa.String(), nullable=True), + sa.Column("FirstAuth", sa.String(), nullable=True), + sa.Column("PubYear", sa.String(), nullable=True), + sa.Column("Title", sa.String(), nullable=True), + sa.Column("Journal", sa.String(), nullable=True), + sa.Column("Volume", sa.String(), nullable=True), + sa.Column("PageNo", sa.String(), nullable=True), + sa.Column("ReportNo", sa.String(), nullable=True), + sa.Column("Publisher", sa.String(), nullable=True), + sa.Column("City", sa.String(), nullable=True), + sa.Column("URL", sa.String(), nullable=True), + sa.Column("Comments", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("OBJECTID"), + ) + op.create_index("ix_NMW_Sources_SourceID", "NMW_Sources", ["SourceID"]) + + # ------------------------------------------------------------------ + # FK constraints + # ------------------------------------------------------------------ + op.create_foreign_key( + "fk_nmw_welllocations_welldataid", + "NMW_WellLocations", + "NMW_WellHeaders", + ["WellDataID"], + ["WellDataID"], + ) + op.create_foreign_key( + "fk_nmw_wellrecords_welldataid", + "NMW_WellRecords", + "NMW_WellHeaders", + ["WellDataID"], + ["WellDataID"], + ) + op.create_foreign_key( + "fk_nmw_wellzdatum_recrdsetid", + "NMW_WellZDatum", + "NMW_WellRecords", + ["RecrdsetID"], + ["RecrdSetID"], + ) + op.create_foreign_key( + "fk_nmw_wellsamples_recrdsetid", + "NMW_WellSamples", + "NMW_WellRecords", + ["RecrdsetID"], + ["RecrdSetID"], + ) + op.create_foreign_key( + "fk_nmw_gtbhtheaders_samplsetid", + "NMW_GtBhtHeaders", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + op.create_foreign_key( + "fk_nmw_gtbhtdata_bhtguid", + "NMW_GtBhtData", + "NMW_GtBhtHeaders", + ["BHTGUID"], + ["BHTGUID"], + ) + op.create_foreign_key( + "fk_nmw_wsintervals_samplsetid", + "NMW_WsIntervals", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + op.create_foreign_key( + "fk_nmw_gtconductivity_intrvlguid", + "NMW_GtConductivity", + "NMW_WsIntervals", + ["IntrvlGUID"], + ["IntrvlGUID"], + ) + op.create_foreign_key( + "fk_nmw_gtheatflow_intrvlguid", + "NMW_GtHeatFlow", + "NMW_WsIntervals", + ["IntrvlGUID"], + ["IntrvlGUID"], + ) + op.create_foreign_key( + "fk_nmw_gtsumheatflow_recrdsetid", + "NMW_GtSumHeatFlow", + "NMW_WellRecords", + ["RecrdSetID"], + ["RecrdSetID"], + ) + op.create_foreign_key( + "fk_nmw_gtsumheatflow_samplsetid", + "NMW_GtSumHeatFlow", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + op.create_foreign_key( + "fk_nmw_gttempdepths_samplsetid", + "NMW_GtTempDepths", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + op.create_foreign_key( + "fk_nmw_wsdstheaders_samplsetid", + "NMW_WsDstHeaders", + "NMW_WellSamples", + ["SamplSetID"], + ["SamplSetID"], + ) + op.create_foreign_key( + "fk_nmw_wsdstintervals_dstguid", + "NMW_WsDstIntervals", + "NMW_WsDstHeaders", + ["DSTGUID"], + ["DSTGUID"], + ) + op.create_foreign_key( + "fk_nmw_wsdstflowhistory_dstinterval", + "NMW_WsDstFlowHistory", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + op.create_foreign_key( + "fk_nmw_wsdstfluidproperties_dstinterval", + "NMW_WsDstFluidProperties", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + op.create_foreign_key( + "fk_nmw_wsdstpressure_dstinterval", + "NMW_WsDstPressure", + "NMW_WsDstIntervals", + ["DSTInterval"], + ["DSTInterval"], + ) + + +def downgrade() -> None: + # Drop FKs before tables (reverse order of creation) + op.drop_constraint( + "fk_nmw_wsdstpressure_dstinterval", "NMW_WsDstPressure", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsdstfluidproperties_dstinterval", + "NMW_WsDstFluidProperties", + type_="foreignkey", + ) + op.drop_constraint( + "fk_nmw_wsdstflowhistory_dstinterval", + "NMW_WsDstFlowHistory", + type_="foreignkey", + ) + op.drop_constraint( + "fk_nmw_wsdstintervals_dstguid", "NMW_WsDstIntervals", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsdstheaders_samplsetid", "NMW_WsDstHeaders", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gttempdepths_samplsetid", "NMW_GtTempDepths", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtsumheatflow_samplsetid", "NMW_GtSumHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtsumheatflow_recrdsetid", "NMW_GtSumHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtheatflow_intrvlguid", "NMW_GtHeatFlow", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_gtconductivity_intrvlguid", "NMW_GtConductivity", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wsintervals_samplsetid", "NMW_WsIntervals", type_="foreignkey" + ) + op.drop_constraint("fk_nmw_gtbhtdata_bhtguid", "NMW_GtBhtData", type_="foreignkey") + op.drop_constraint( + "fk_nmw_gtbhtheaders_samplsetid", "NMW_GtBhtHeaders", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellsamples_recrdsetid", "NMW_WellSamples", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellzdatum_recrdsetid", "NMW_WellZDatum", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_wellrecords_welldataid", "NMW_WellRecords", type_="foreignkey" + ) + op.drop_constraint( + "fk_nmw_welllocations_welldataid", "NMW_WellLocations", type_="foreignkey" + ) + + # Drop tables in child-first order + op.drop_index("ix_NMW_Sources_SourceID", table_name="NMW_Sources") + op.drop_table("NMW_Sources") + op.drop_index("ix_NMW_WsDstPressure_DSTInterval", table_name="NMW_WsDstPressure") + op.drop_table("NMW_WsDstPressure") + op.drop_index( + "ix_NMW_WsDstFluidProperties_DSTInterval", table_name="NMW_WsDstFluidProperties" + ) + op.drop_table("NMW_WsDstFluidProperties") + op.drop_index( + "ix_NMW_WsDstFlowHistory_DSTInterval", table_name="NMW_WsDstFlowHistory" + ) + op.drop_table("NMW_WsDstFlowHistory") + op.drop_index("ix_NMW_WsDstIntervals_DSTGUID", table_name="NMW_WsDstIntervals") + op.drop_table("NMW_WsDstIntervals") + op.drop_index("ix_NMW_WsDstHeaders_SamplSetID", table_name="NMW_WsDstHeaders") + op.drop_table("NMW_WsDstHeaders") + op.drop_index("ix_NMW_GtConductivity_IntrvlGUID", table_name="NMW_GtConductivity") + op.drop_table("NMW_GtConductivity") + op.drop_index("ix_NMW_GtHeatFlow_IntrvlGUID", table_name="NMW_GtHeatFlow") + op.drop_table("NMW_GtHeatFlow") + op.drop_index("ix_NMW_WsIntervals_SamplSetID", table_name="NMW_WsIntervals") + op.drop_table("NMW_WsIntervals") + op.drop_index("ix_NMW_GtBhtData_BHTGUID", table_name="NMW_GtBhtData") + op.drop_table("NMW_GtBhtData") + op.drop_index("ix_NMW_GtBhtHeaders_SamplSetID", table_name="NMW_GtBhtHeaders") + op.drop_table("NMW_GtBhtHeaders") + op.drop_index("ix_NMW_GtSumHeatFlow_SamplSetID", table_name="NMW_GtSumHeatFlow") + op.drop_index("ix_NMW_GtSumHeatFlow_RecrdSetID", table_name="NMW_GtSumHeatFlow") + op.drop_table("NMW_GtSumHeatFlow") + op.drop_index("ix_NMW_GtTempDepths_SamplSetID", table_name="NMW_GtTempDepths") + op.drop_table("NMW_GtTempDepths") + op.drop_index("ix_NMW_WellSamples_RecrdsetID", table_name="NMW_WellSamples") + op.drop_table("NMW_WellSamples") + op.drop_index("ix_NMW_WellZDatum_RecrdsetID", table_name="NMW_WellZDatum") + op.drop_table("NMW_WellZDatum") + op.drop_index("ix_NMW_WellRecords_WellDataID", table_name="NMW_WellRecords") + op.drop_table("NMW_WellRecords") + op.drop_index("ix_NMW_WellLocations_WellDataID", table_name="NMW_WellLocations") + op.drop_table("NMW_WellLocations") + op.drop_table("NMW_WellHeaders") diff --git a/alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py similarity index 52% rename from alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py rename to alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py index ca1bdf2b8..607c9ccd3 100644 --- a/alembic/versions/c5d6e7f8a9b0_add_integer_id_to_geothermal_ogc_views.py +++ b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py @@ -1,28 +1,52 @@ -"""add integer id to geothermal OGC views +"""NMW per-well geothermal OGC views -Revision ID: c5d6e7f8a9b0 -Revises: b4c5d6e7f8a9 -Create Date: 2026-06-15 +Revision ID: d1e2f3a4b5c6 +Revises: c0d1e2f3a4b5 +Create Date: 2026-06-22 -All other OGC views use an integer id_field (from thing.id). pygeoapi's -PostgreSQL provider is tested against integer PKs. Replace well_data_id as the -id_field with row_number() OVER () AS id to match the convention, and keep -well_data_id as a regular attribute column. +Four pygeoapi point-layer views that aggregate geothermal data to one +feature per well. All geometry is built from NMW_WellLocations Lat/Long_dd83 +(WGS84). All views use integer id (row_number() OVER ()) as the pygeoapi +id_field; pygeoapi's PostgreSQL provider requires an integer PK column. + + ogc_geothermal_wells_bht (VIEW) + One feature per well with BHT aggregate stats. Links via: + NMW_GtBhtData → NMW_GtBhtHeaders.BHTGUID → NMW_WellSamples.SamplSetID + → NMW_WellRecords.RecrdSetID → NMW_WellLocations/Headers.WellDataID + + ogc_geothermal_wells_temperature_profile (MATERIALIZED VIEW) + One feature per well with temperature-depth JSON series. Materialized + because the source NMW_GtTempDepths is large (~370k rows) and the + json_agg is too heavy to recompute per request. NMW_WellLocations is + deduped via DISTINCT ON before joining because a well can have multiple + location rows (OBJECTID is the PK, not WellDataID). REFRESH after a + data reload. + + ogc_geothermal_wells_summary_heat_flow (VIEW) + One feature per well with summary heat-flow stats and a measurements + JSON series. Links via NMW_GtSumHeatFlow.RecrdSetID. + + ogc_geothermal_wells_interval_heat_flow (VIEW) + One feature per well with per-interval heat-flow stats. Links via + NMW_GtHeatFlow.IntrvlGUID → NMW_WsIntervals → NMW_WellSamples. """ from alembic import op from sqlalchemy import text -revision = "c5d6e7f8a9b0" -down_revision = "b4c5d6e7f8a9" +revision = "d1e2f3a4b5c6" +down_revision = "c0d1e2f3a4b5" branch_labels = None depends_on = None _BHT_VIEW = "ogc_geothermal_wells_bht" _PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" +_SUM_HF_VIEW = "ogc_geothermal_wells_summary_heat_flow" +_INT_HF_VIEW = "ogc_geothermal_wells_interval_heat_flow" def upgrade() -> None: + # ogc_geothermal_wells_bht op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) op.execute( text( @@ -61,6 +85,7 @@ def upgrade() -> None: ) ) + # ogc_geothermal_wells_temperature_profile (materialized) op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) op.execute( text( @@ -109,7 +134,7 @@ def upgrade() -> None: ) ) op.execute( - text(f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id " f'ON "{_PROFILE_VIEW}" (id)') + text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') ) op.execute( text( @@ -117,30 +142,47 @@ def upgrade() -> None: ) ) - -def downgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + # ogc_geothermal_wells_summary_heat_flow + op.execute(text(f'DROP VIEW IF EXISTS "{_SUM_HF_VIEW}"')) op.execute( text( f""" - CREATE VIEW "{_BHT_VIEW}" AS + CREATE VIEW "{_SUM_HF_VIEW}" AS SELECT + row_number() OVER () AS id, r."WellDataID"::text AS well_data_id, hdr."CurWellNam" AS well_name, hdr."API" AS api, - hdr."TotalDepth" AS total_depth, - count(d.*) AS bht_count, - max(d."BHT") AS max_bht, - min(d."BHT") AS min_bht, - max(d."Depth") AS max_bht_depth, - max(d."TempUnit") AS temp_unit, + count(shf.*) AS heat_flow_count, + max(shf."HeatFlow") AS max_heat_flow, + avg(shf."HeatFlow") AS avg_heat_flow, + max(shf."HtFlowUnit") AS heat_flow_unit, + max(shf."ThermlGrad") AS max_thermal_gradient, + max(shf."GradUnit") AS gradient_unit, + max(shf."ThermlCond") AS max_thermal_conductivity, + max(shf."TCondUnit") AS conductivity_unit, + max(shf."Quality") AS quality, + json_agg( + json_build_object( + 'from_depth', shf."FromDepth", + 'to_depth', shf."ToDepth", + 'depth_unit', shf."DepthUnit", + 'heat_flow', shf."HeatFlow", + 'heat_flow_error', shf."HtFlowErr", + 'heat_flow_unit', shf."HtFlowUnit", + 'thermal_gradient', shf."ThermlGrad", + 'gradient_unit', shf."GradUnit", + 'thermal_conductivity', shf."ThermlCond", + 'conductivity_unit', shf."TCondUnit", + 'quality', shf."Quality" + ) + ORDER BY shf."FromDepth" + ) AS measurements, ST_SetSRID( ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 ) AS geom - FROM "NMW_GtBhtData" AS d - JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + FROM "NMW_GtSumHeatFlow" AS shf + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" WHERE loc."Lat_dd83" IS NOT NULL @@ -150,49 +192,56 @@ def downgrade() -> None: loc."Lat_dd83", loc."Long_dd83", hdr."CurWellNam", - hdr."API", - hdr."TotalDepth" + hdr."API" """ ) ) - op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + # ogc_geothermal_wells_interval_heat_flow + op.execute(text(f'DROP VIEW IF EXISTS "{_INT_HF_VIEW}"')) op.execute( text( f""" - CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS - WITH loc AS ( - SELECT DISTINCT ON ("WellDataID") - "WellDataID", "Lat_dd83", "Long_dd83" - FROM "NMW_WellLocations" - WHERE "Lat_dd83" IS NOT NULL - AND "Long_dd83" IS NOT NULL - ORDER BY "WellDataID", "OBJECTID" - ) + CREATE VIEW "{_INT_HF_VIEW}" AS SELECT + row_number() OVER () AS id, r."WellDataID"::text AS well_data_id, hdr."CurWellNam" AS well_name, hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, + count(hf.*) AS interval_count, + max(hf."Q") AS max_heat_flow, + avg(hf."Q") AS avg_heat_flow, + max(hf."Q_unit") AS heat_flow_unit, + max(hf."Gradient") AS max_gradient, + max(hf."Kpr") AS max_thermal_conductivity, + max(hf."Kpr_unit") AS conductivity_unit, + max(hf."Ka") AS max_diffusivity, + max(hf."Ka_unit") AS diffusivity_unit, json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, + json_build_object( + 'from_depth', i."From_Depth", + 'to_depth', i."To_Depth", + 'heat_flow', hf."Q", + 'heat_flow_unit', hf."Q_unit", + 'gradient', hf."Gradient", + 'thermal_conductivity', hf."Kpr", + 'conductivity_unit', hf."Kpr_unit", + 'diffusivity', hf."Ka", + 'diffusivity_unit', hf."Ka_unit" + ) + ORDER BY i."From_Depth" + ) AS measurements, ST_SetSRID( ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + FROM "NMW_GtHeatFlow" AS hf + JOIN "NMW_WsIntervals" AS i ON i."IntrvlGUID" = hf."IntrvlGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = i."SamplSetID" JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN loc ON loc."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL GROUP BY r."WellDataID", loc."Lat_dd83", @@ -202,14 +251,10 @@ def downgrade() -> None: """ ) ) - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " - f'ON "{_PROFILE_VIEW}" (well_data_id)' - ) - ) - op.execute( - text( - f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' - ) - ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_INT_HF_VIEW}"')) + op.execute(text(f'DROP VIEW IF EXISTS "{_SUM_HF_VIEW}"')) + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) diff --git a/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py b/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py deleted file mode 100644 index ef5c037c1..000000000 --- a/alembic/versions/d6e7f8a9b0c1_add_ogc_bht_measurements_view.py +++ /dev/null @@ -1,65 +0,0 @@ -"""add ogc_bht_measurements view - -Revision ID: d6e7f8a9b0c1 -Revises: c5d6e7f8a9b0 -Create Date: 2026-06-17 - -Individual BHT measurement rows with well header, location, and Z-datum -filter — translated from the legacy MSSQL query against NM_Aquifer. -One row per measurement (not aggregated per well). -""" - -from alembic import op -from sqlalchemy import text - -revision = "d6e7f8a9b0c1" -down_revision = "c5d6e7f8a9b0" -branch_labels = None -depends_on = None - -_VIEW = "ogc_bht_measurements" - - -def upgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_VIEW}" AS - SELECT - d."OBJECTID" AS id, - hdr."API" AS api, - hdr."CurWellNam" AS well_name, - hdr."CurWellNum" AS well_num, - hdr."CurOperatr" AS operator, - hdr."WellType" AS well_type, - hdr."Well_TVD" AS well_tvd, - hdr."ComplDate" AS completion_date, - hdr."CurStatus" AS current_status, - hdr."TotalDepth" AS total_depth, - hdr."Cuttings" AS cuttings, - hdr."CoreExists" AS core_exists, - loc."County" AS county, - d."Depth" AS bht_depth, - d."BHT" AS bht, - d."HrsSnceCir" AS hours_since_circulation, - d."DateMeasrd" AS date_measured, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtBhtData" AS d - JOIN "NMW_GtBhtHeaders" AS bh ON bh."BHTGUID" = d."BHTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = bh."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" - JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - """ - ) - ) - - -def downgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py b/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py new file mode 100644 index 000000000..54958d794 --- /dev/null +++ b/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py @@ -0,0 +1,295 @@ +"""NMW individual-measurement and analytical OGC views + +Revision ID: e2f3a4b5c6d7 +Revises: d1e2f3a4b5c6 +Create Date: 2026-06-22 + +Four pygeoapi point-layer views exposing individual measurement rows (not +aggregated per well) and analytical summaries from the NMW staging mirror. +All translated from legacy MSSQL queries against NM_Aquifer. Integer OBJECTID +columns are used as the pygeoapi id_field directly. + + ogc_bht_measurements (VIEW) + One row per BHT measurement with well header and location. + Translated from the legacy MSSQL BHT query. + + ogc_temp_depth_measurements (VIEW) + One row per downhole temperature reading with well header, location, + and elevation. Translated from TempDepth2_SortedWellName query. + Locations with Exclude=1 are filtered out. + + ogc_heat_flow (VIEW) + One row per NMW_GtSumHeatFlow record with well header, location, + elevation, and publication attribution from NMW_Sources. Unit + conversions (ft→m, HFU→mW/m², TCU→W/m·K) applied inline via + CASE WHEN. Translated from the legacy MSSQL HeatFlow query. + + ogc_dst (VIEW) + One row per DST interval with pressure, flow history, and well + header. The original Access query referenced DST_flwHstryConcat + (a broken saved query); replaced with a string_agg() CTE over + NMW_WsDstFlowHistory. DISTINCT used in place of the original + GROUP BY with no aggregate functions. Translated from the legacy + MSSQL DST query. +""" + +from alembic import op +from sqlalchemy import text + +revision = "e2f3a4b5c6d7" +down_revision = "d1e2f3a4b5c6" +branch_labels = None +depends_on = None + +_BHT_MEAS_VIEW = "ogc_bht_measurements" +_TEMP_DEPTH_VIEW = "ogc_temp_depth_measurements" +_HEAT_FLOW_VIEW = "ogc_heat_flow" +_DST_VIEW = "ogc_dst" + + +def upgrade() -> None: + # ogc_bht_measurements + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_MEAS_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_BHT_MEAS_VIEW}" AS + SELECT + d."OBJECTID" AS id, + hdr."API" AS api, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."CurOperatr" AS operator, + hdr."WellType" AS well_type, + hdr."Well_TVD" AS well_tvd, + hdr."ComplDate" AS completion_date, + hdr."CurStatus" AS current_status, + hdr."TotalDepth" AS total_depth, + hdr."Cuttings" AS cuttings, + hdr."CoreExists" AS core_exists, + loc."County" AS county, + d."Depth" AS bht_depth, + d."BHT" AS bht, + d."HrsSnceCir" AS hours_since_circulation, + d."DateMeasrd" AS date_measured, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS bh ON bh."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = bh."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + # ogc_temp_depth_measurements + op.execute(text(f'DROP VIEW IF EXISTS "{_TEMP_DEPTH_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_TEMP_DEPTH_VIEW}" AS + SELECT + td."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + r."SourceID" AS source_id, + s."SampleFm" AS sample_fm, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd27" AS lat_dd27, + loc."Long_dd27" AS long_dd27, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + loc."LocAccVal" AS loc_acc_val, + s."EnteredBy" AS entered_by, + s."EntryDate" AS entry_date, + td."Depth" AS depth, + s."SmpDpUnt" AS depth_unit, + td."Temp" AS temp, + td."TempUnit" AS temp_unit, + z."Elev_GL" AS elev_gl, + z."Elev_unspc" AS elev_unspc, + z."Elev_KB" AS elev_kb, + s."SampleDate" AS sample_date, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + WHERE loc."Exclude" = 0 + AND loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + # ogc_heat_flow + op.execute(text(f'DROP VIEW IF EXISTS "{_HEAT_FLOW_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_HEAT_FLOW_VIEW}" AS + SELECT + shf."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd27" AS lat_dd27, + loc."Long_dd27" AS long_dd27, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + r."SourceID" AS source_id, + z."Elev_GL" AS elev_gl, + z."Elev_KB" AS elev_kb, + z."Elev_unspc" AS elev_unspc, + CASE WHEN z."DepthUnits" = 'ft' + THEN 0.3048 * z."Elev_unspc" + ELSE z."Elev_unspc" + END AS elevation_m, + z."DepthUnits" AS depth_units, + hdr."TotalDepth" AS total_depth, + CASE WHEN z."DepthUnits" = 'ft' + THEN 0.3048 * hdr."TotalDepth" + ELSE hdr."TotalDepth" + END AS total_depth_m, + shf."FromDepth" AS from_depth, + shf."ToDepth" AS to_depth, + shf."ThermlCond" AS therml_cond, + shf."TCondRange" AS tcond_range, + shf."TCondError" AS tcond_error, + shf."TCondUnit" AS tcond_unit, + CASE WHEN shf."TCondUnit" = 'TCU' + THEN 0.4184 * shf."ThermlCond" + ELSE shf."ThermlCond" + END AS tc_si, + shf."SampleType" AS sample_type, + shf."NumSamples" AS num_samples, + shf."ThermlGrad" AS therml_grad, + shf."TGradRange" AS tgrad_range, + shf."TGError" AS tg_error, + shf."GradUnit" AS grad_unit, + shf."HeatFlow" AS heat_flow, + shf."HtFlowUnit" AS ht_flow_unit, + CASE WHEN shf."HtFlowUnit" = 'HFU' + THEN 41.84 * shf."HeatFlow" + ELSE shf."HeatFlow" + END AS heat_flow_si, + shf."Quality" AS quality, + src."FirstAuth" AS first_auth, + src."PubYear" AS pub_year, + src."Title" AS title, + src."Journal" AS journal, + src."Volume" AS volume, + src."PageNo" AS page_no, + shf."HtFlowEst" AS ht_flow_est, + r."EntryDate" AS entry_date, + CASE WHEN shf."HtFlowUnit" = 'HFU' + THEN 41.84 * shf."HtFlowEst" + ELSE shf."HtFlowEst" + END AS ht_flow_est_si, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtSumHeatFlow" AS shf + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + JOIN "NMW_Sources" AS src ON src."SourceID" = r."SourceID" + WHERE loc."Exclude" = 0 + AND loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + # ogc_dst + op.execute(text(f'DROP VIEW IF EXISTS "{_DST_VIEW}"')) + op.execute( + text( + f""" + CREATE VIEW "{_DST_VIEW}" AS + WITH flow_history AS ( + SELECT + "DSTInterval", + string_agg("Operation", '; ' ORDER BY "OBJECTID") AS flow_history + FROM "NMW_WsDstFlowHistory" + GROUP BY "DSTInterval" + ) + SELECT DISTINCT + i."OBJECTID" AS id, + hdr."CurWellNam" AS well_name, + hdr."CurWellNum" AS well_num, + hdr."API" AS api, + i."DSTName" AS dst_name, + dh."DSTOprator" AS dst_operator, + i."DSTNumber" AS dst_number, + i."DSTDate" AS dst_date, + loc."County" AS county, + loc."State" AS state, + loc."Lat_dd83" AS lat_dd83, + loc."Long_dd83" AS long_dd83, + s."From_Depth" AS from_depth, + s."To_Depth" AS to_depth, + i."TargetFm" AS target_fm, + i."PackrFrom" AS packer_from, + i."PackerTo" AS packer_to, + i."SrfChokeSz" AS srf_choke_sz, + i."BotChokeSz" AS bot_choke_sz, + s."SmpDpUnt" AS depth_unit, + z."Elev_GL" AS elev_gl, + z."Elev_unspc" AS elev_unspc, + p."PrsGageDpt" AS prs_gage_dpt, + i."PipeDia" AS pipe_dia, + i."PipeLength" AS pipe_length, + fh.flow_history AS flow_history, + p."PrsInShtIn" AS init_flow, + p."FlwPrsInMin" AS flw_prs_in_min, + p."PrsFnShtIn" AS fin_flow, + p."FlwPrsFinMin" AS flw_prs_fin_min, + p."PrsInitClsdIn" AS prs_init_clsd_in, + p."InShtInMin" AS in_sht_in_min, + p."EquilPress" AS fin_shut_in, + p."FnShtInMin" AS fn_sht_in_min, + p."HydrostPrsIn" AS hydrost_prs_in, + p."HydStPrsFl" AS hyd_st_prs_fl, + dh."PressUnits" AS press_units, + p."BlankedOff" AS blanked_off, + p."FmTemp" AS fm_temp, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_WsDstIntervals" AS i + JOIN "NMW_WsDstHeaders" AS dh ON dh."DSTGUID" = i."DSTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = dh."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" + LEFT JOIN "NMW_WsDstPressure" AS p ON p."DSTInterval" = i."DSTInterval" + LEFT JOIN flow_history AS fh ON fh."DSTInterval" = i."DSTInterval" + WHERE loc."Lat_dd83" IS NOT NULL + AND loc."Long_dd83" IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.execute(text(f'DROP VIEW IF EXISTS "{_DST_VIEW}"')) + op.execute(text(f'DROP VIEW IF EXISTS "{_HEAT_FLOW_VIEW}"')) + op.execute(text(f'DROP VIEW IF EXISTS "{_TEMP_DEPTH_VIEW}"')) + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_MEAS_VIEW}"')) diff --git a/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py b/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py deleted file mode 100644 index 74834da16..000000000 --- a/alembic/versions/e7f8a9b0c1d2_add_ogc_temp_depth_measurements_view.py +++ /dev/null @@ -1,72 +0,0 @@ -"""add ogc_temp_depth_measurements view - -Revision ID: e7f8a9b0c1d2 -Revises: d6e7f8a9b0c1 -Create Date: 2026-06-18 - -Individual temperature-depth readings with well header, location, and -elevation data. Translated from the legacy MSSQL TempDepth2_SortedWellName -query against NM_Aquifer. One row per reading; locations with Exclude=1 -are filtered out. -""" - -from alembic import op -from sqlalchemy import text - -revision = "e7f8a9b0c1d2" -down_revision = "d6e7f8a9b0c1" -branch_labels = None -depends_on = None - -_VIEW = "ogc_temp_depth_measurements" - - -def upgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) - op.execute( - text( - f""" - CREATE VIEW "{_VIEW}" AS - SELECT - td."OBJECTID" AS id, - hdr."CurWellNam" AS well_name, - hdr."CurWellNum" AS well_num, - hdr."API" AS api, - r."SourceID" AS source_id, - s."SampleFm" AS sample_fm, - loc."County" AS county, - loc."State" AS state, - loc."Lat_dd27" AS lat_dd27, - loc."Long_dd27" AS long_dd27, - loc."Lat_dd83" AS lat_dd83, - loc."Long_dd83" AS long_dd83, - loc."LocAccVal" AS loc_acc_val, - s."EnteredBy" AS entered_by, - s."EntryDate" AS entry_date, - td."Depth" AS depth, - s."SmpDpUnt" AS depth_unit, - td."Temp" AS temp, - td."TempUnit" AS temp_unit, - z."Elev_GL" AS elev_gl, - z."Elev_unspc" AS elev_unspc, - z."Elev_KB" AS elev_kb, - s."SampleDate" AS sample_date, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellZDatum" AS z ON z."RecrdsetID" = r."RecrdSetID" - JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - WHERE loc."Exclude" = 0 - AND loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - """ - ) - ) - - -def downgrade() -> None: - op.execute(text(f'DROP VIEW IF EXISTS "{_VIEW}"')) diff --git a/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py b/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py deleted file mode 100644 index 1a8c8d5f6..000000000 --- a/alembic/versions/f8a9b0c1d2e3_add_nmw_sources_mirror_table.py +++ /dev/null @@ -1,45 +0,0 @@ -"""add NMW_Sources mirror table - -Revision ID: f8a9b0c1d2e3 -Revises: e7f8a9b0c1d2 -Create Date: 2026-06-18 - -1:1 mirror of the NM_Wells tbl_sources publication/data-source registry. -Keyed by the free-text SourceID string that appears in NMW_WellRecords.SourceID. -Needed to join publication attribution (FirstAuth, PubYear, Title, etc.) -into the ogc_heat_flow view. -""" - -from alembic import op -import sqlalchemy as sa - -revision = "f8a9b0c1d2e3" -down_revision = "e7f8a9b0c1d2" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "NMW_Sources", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("SourceID", sa.String(), nullable=True), - sa.Column("FirstAuth", sa.String(), nullable=True), - sa.Column("PubYear", sa.String(), nullable=True), - sa.Column("Title", sa.String(), nullable=True), - sa.Column("Journal", sa.String(), nullable=True), - sa.Column("Volume", sa.String(), nullable=True), - sa.Column("PageNo", sa.String(), nullable=True), - sa.Column("ReportNo", sa.String(), nullable=True), - sa.Column("Publisher", sa.String(), nullable=True), - sa.Column("City", sa.String(), nullable=True), - sa.Column("URL", sa.String(), nullable=True), - sa.Column("Comments", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index("ix_NMW_Sources_SourceID", "NMW_Sources", ["SourceID"]) - - -def downgrade() -> None: - op.drop_index("ix_NMW_Sources_SourceID", table_name="NMW_Sources") - op.drop_table("NMW_Sources") diff --git a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py b/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py deleted file mode 100644 index b413554db..000000000 --- a/alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py +++ /dev/null @@ -1,207 +0,0 @@ -"""NM_Wells 1:1 staging mirror tables - -Revision ID: u7v8w9x0y1z2 -Revises: t6u7v8w9x0y1 -Create Date: 2026-06-06 00:00:00.000000 - -1:1 staging mirror of the legacy NM_Wells SQL Server "Migrate First / Main" -tables (see db/nmw_legacy.py and docs/nm_wells-migration.md). Faithful, -column-for-column copies; the transform into the Ocotillo data model is a -later phase. - - tbl_well_locations -> NMW_WellLocations - tbl_well_headers -> NMW_WellHeaders - tbl_well_records -> NMW_WellRecords - tbl_well_z_datum -> NMW_WellZDatum - tbl_well_samples -> NMW_WellSamples -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = "u7v8w9x0y1z2" -down_revision: Union[str, Sequence[str], None] = "t6u7v8w9x0y1" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - op.create_table( - "NMW_WellLocations", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Well_ID", sa.String(), nullable=True), - sa.Column("Import_ID", sa.Integer(), nullable=True), - sa.Column("Township", sa.Float(), nullable=True), - sa.Column("NorS_TDir", sa.String(), nullable=True), - sa.Column("Range", sa.Float(), nullable=True), - sa.Column("EorW_RDir", sa.String(), nullable=True), - sa.Column("Sectn", sa.SmallInteger(), nullable=True), - sa.Column("SectnPart", sa.String(), nullable=True), - sa.Column("UnitLetter", sa.String(), nullable=True), - sa.Column("UTM_zone", sa.String(), nullable=True), - sa.Column("State", sa.String(), nullable=True), - sa.Column("County", sa.String(), nullable=True), - sa.Column("Basin", sa.String(), nullable=True), - sa.Column("Footage_NS", sa.Float(), nullable=True), - sa.Column("NorS_FDir", sa.String(), nullable=True), - sa.Column("Footage_EW", sa.Float(), nullable=True), - sa.Column("EorW_FDir", sa.String(), nullable=True), - sa.Column("Lat_min", sa.SmallInteger(), nullable=True), - sa.Column("Lat_sec", sa.Float(), nullable=True), - sa.Column("Long_deg", sa.SmallInteger(), nullable=True), - sa.Column("Long_min", sa.SmallInteger(), nullable=True), - sa.Column("Long_sec", sa.Float(), nullable=True), - sa.Column("Lat_dd27", sa.Float(), nullable=True), - sa.Column("Long_dd27", sa.Float(), nullable=True), - sa.Column("Lat_dd83", sa.Float(), nullable=True), - sa.Column("Long_dd83", sa.Float(), nullable=True), - sa.Column("SourceID", sa.String(), nullable=True), - sa.Column("SourceDatum", sa.String(), nullable=True), - sa.Column("SourceUnits", sa.String(), nullable=True), - sa.Column("LocAccType", sa.String(), nullable=True), - sa.Column("LocAccMeas", sa.String(), nullable=True), - sa.Column("LocAccVal", sa.Float(), nullable=True), - sa.Column("Duplicated", sa.SmallInteger(), nullable=True), - sa.Column("Exclude", sa.SmallInteger(), nullable=True), - sa.Column("Comments", sa.String(), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("API", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_WellLocations_WellDataID", "NMW_WellLocations", ["WellDataID"] - ) - - op.create_table( - "NMW_WellHeaders", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("WellSpotID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("API", sa.String(), nullable=True), - sa.Column("WellClass", sa.String(), nullable=True), - sa.Column("WellType", sa.String(), nullable=True), - sa.Column("WellOrient", sa.String(), nullable=True), - sa.Column("CurWellNam", sa.String(), nullable=True), - sa.Column("CurWellNum", sa.String(), nullable=True), - sa.Column("CurStatus", sa.String(), nullable=True), - sa.Column("PrdPoolCnt", sa.SmallInteger(), nullable=True), - sa.Column("CurOperatr", sa.String(), nullable=True), - sa.Column("CurOwner", sa.String(), nullable=True), - sa.Column("TotalDepth", sa.Float(), nullable=True), - sa.Column("Well_TVD", sa.Float(), nullable=True), - sa.Column("Fm_TD", sa.String(), nullable=True), - sa.Column("Age_TD", sa.String(), nullable=True), - sa.Column("SpudDate", sa.DateTime(), nullable=True), - sa.Column("ComplDate", sa.DateTime(), nullable=True), - sa.Column("PlugDate", sa.DateTime(), nullable=True), - sa.Column("PlugBack", sa.Float(), nullable=True), - sa.Column("BridgePlug", sa.String(), nullable=True), - sa.Column("ScoutTickt", sa.SmallInteger(), nullable=True), - sa.Column("DwnHoleSur", sa.SmallInteger(), nullable=True), - sa.Column("GeolLog", sa.SmallInteger(), nullable=True), - sa.Column("Geophyslog", sa.SmallInteger(), nullable=True), - sa.Column("GthrmExist", sa.SmallInteger(), nullable=True), - sa.Column("PetroData", sa.SmallInteger(), nullable=True), - sa.Column("CoreExists", sa.SmallInteger(), nullable=True), - sa.Column("Cuttings", sa.SmallInteger(), nullable=True), - sa.Column("SampleData", sa.SmallInteger(), nullable=True), - sa.Column("Comments", sa.String(), nullable=True), - sa.Column("Import_ID", sa.String(), nullable=True), - sa.Column("Import_DB", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("WellDataID"), - ) - - op.create_table( - "NMW_WellRecords", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("WellDataID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("RecrdClass", sa.String(), nullable=True), - sa.Column("SourceID", sa.String(), nullable=True), - sa.Column("ActionDate", sa.DateTime(), nullable=True), - sa.Column("WellName", sa.String(), nullable=True), - sa.Column("WellNumber", sa.String(), nullable=True), - sa.Column("API_suffix", sa.String(), nullable=True), - sa.Column("EnteredBy", sa.String(), nullable=True), - sa.Column("EntryDate", sa.DateTime(), nullable=True), - sa.Column("Comments", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("RecrdSetID"), - ) - op.create_index("ix_NMW_WellRecords_WellDataID", "NMW_WellRecords", ["WellDataID"]) - - op.create_table( - "NMW_WellZDatum", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Elev_GL", sa.Float(), nullable=True), - sa.Column("Elev_DF", sa.Float(), nullable=True), - sa.Column("Elev_KB", sa.Float(), nullable=True), - sa.Column("Elev_unspc", sa.Float(), nullable=True), - sa.Column("DatumElev", sa.Float(), nullable=True), - sa.Column("DepthDatum", sa.String(), nullable=True), - sa.Column("DepthUnits", sa.String(), nullable=True), - sa.Column("Z_datum", sa.String(), nullable=True), - sa.Column("Z_units", sa.String(), nullable=True), - sa.Column("ElevSource", sa.String(), nullable=True), - sa.Column("ElvAccType", sa.String(), nullable=True), - sa.Column("ElvAccMeas", sa.String(), nullable=True), - sa.Column("ElvAccVal", sa.Float(), nullable=True), - sa.Column("Comments", sa.String(), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index("ix_NMW_WellZDatum_RecrdsetID", "NMW_WellZDatum", ["RecrdsetID"]) - - op.create_table( - "NMW_WellSamples", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("RecrdsetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SmpSetName", sa.String(), nullable=True), - sa.Column("SamplClass", sa.String(), nullable=True), - sa.Column("SampleType", sa.String(), nullable=True), - sa.Column("SampleFm", sa.String(), nullable=True), - sa.Column("SampleLoc", sa.String(), nullable=True), - sa.Column("SampleDate", sa.DateTime(), nullable=True), - sa.Column("From_Depth", sa.Float(), nullable=True), - sa.Column("To_Depth", sa.Float(), nullable=True), - sa.Column("SmpDpUnt", sa.String(), nullable=True), - sa.Column("From_TVD", sa.Float(), nullable=True), - sa.Column("To_TVD", sa.Float(), nullable=True), - sa.Column("From_Elev", sa.Float(), nullable=True), - sa.Column("To_Elev", sa.Float(), nullable=True), - sa.Column("Porosity", sa.SmallInteger(), nullable=True), - sa.Column("Permeablty", sa.SmallInteger(), nullable=True), - sa.Column("Density", sa.SmallInteger(), nullable=True), - sa.Column("DST_Tests", sa.SmallInteger(), nullable=True), - sa.Column("ThinSect", sa.SmallInteger(), nullable=True), - sa.Column("Geochron", sa.SmallInteger(), nullable=True), - sa.Column("Geochem", sa.SmallInteger(), nullable=True), - sa.Column("Geothermal", sa.SmallInteger(), nullable=True), - sa.Column("WholeRock", sa.SmallInteger(), nullable=True), - sa.Column("Paleontlgy", sa.SmallInteger(), nullable=True), - sa.Column("EnteredBy", sa.String(), nullable=True), - sa.Column("EntryDate", sa.DateTime(), nullable=True), - sa.Column("Notes", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("SamplSetID"), - ) - op.create_index("ix_NMW_WellSamples_RecrdsetID", "NMW_WellSamples", ["RecrdsetID"]) - - -def downgrade() -> None: - """Downgrade schema.""" - op.drop_index("ix_NMW_WellSamples_RecrdsetID", table_name="NMW_WellSamples") - op.drop_table("NMW_WellSamples") - op.drop_index("ix_NMW_WellZDatum_RecrdsetID", table_name="NMW_WellZDatum") - op.drop_table("NMW_WellZDatum") - op.drop_index("ix_NMW_WellRecords_WellDataID", table_name="NMW_WellRecords") - op.drop_table("NMW_WellRecords") - op.drop_table("NMW_WellHeaders") - op.drop_index("ix_NMW_WellLocations_WellDataID", table_name="NMW_WellLocations") - op.drop_table("NMW_WellLocations") diff --git a/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py b/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py deleted file mode 100644 index 2a0bd1a97..000000000 --- a/alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py +++ /dev/null @@ -1,330 +0,0 @@ -"""NM_Wells geothermal + drill-stem-test 1:1 staging mirror tables - -Revision ID: v8w9x0y1z2a3 -Revises: u7v8w9x0y1z2 -Create Date: 2026-06-06 00:00:01.000000 - -1:1 staging mirror of the NM_Wells "Migrate First" Geothermal and Drill Stem -Test tables (see db/nmw_legacy.py and docs/nm_wells-migration.md). Columns and -lengths taken directly from the NM_Wells SQL dump DDL. - - Geothermal: - tbl_gt_bht_headers -> NMW_GtBhtHeaders - tbl_gt_bht_data -> NMW_GtBhtData - tbl_ws_intervals -> NMW_WsIntervals - tbl_gt_conductivity -> NMW_GtConductivity - tbl_gt_heat_flow -> NMW_GtHeatFlow - tbl_gt_sum_heat_flow -> NMW_GtSumHeatFlow - tbl_gt_temp_depths -> NMW_GtTempDepths - Drill Stem Tests: - tbl_ws_dst_headers -> NMW_WsDstHeaders - tbl_ws_dst_intervals -> NMW_WsDstIntervals - tbl_ws_dst_flow_history-> NMW_WsDstFlowHistory - tbl_ws_dst_fluid_properties -> NMW_WsDstFluidProperties - tbl_ws_dst_pressure -> NMW_WsDstPressure -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = "v8w9x0y1z2a3" -down_revision: Union[str, Sequence[str], None] = "u7v8w9x0y1z2" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - op.create_table( - "NMW_GtBhtHeaders", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("BoreDia", sa.Float(), nullable=True), - sa.Column("BoreUnits", sa.String(length=16), nullable=True), - sa.Column("DrillFluid", sa.String(length=16), nullable=True), - sa.Column("TempUnit", sa.String(length=1), nullable=True), - sa.Column("FldSalinity", sa.Float(), nullable=True), - sa.Column("FldRstvity", sa.Float(), nullable=True), - sa.Column("Fluid_pH", sa.Float(), nullable=True), - sa.Column("FldDensity", sa.Float(), nullable=True), - sa.Column("FldLevel", sa.Float(), nullable=True), - sa.Column("FldViscsty", sa.Float(), nullable=True), - sa.Column("FluidLoss", sa.String(length=50), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("BHTGUID"), - ) - op.create_index( - "ix_NMW_GtBhtHeaders_SamplSetID", "NMW_GtBhtHeaders", ["SamplSetID"] - ) - - op.create_table( - "NMW_GtBhtData", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("BHTGUID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Depth", sa.Float(), nullable=True), - sa.Column("BHT", sa.Float(), nullable=True), - sa.Column("TempUnit", sa.String(length=5), nullable=True), - sa.Column("HrsSnceCir", sa.Float(), nullable=True), - sa.Column("DateMeasrd", sa.DateTime(), nullable=True), - sa.Column("Comments", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index("ix_NMW_GtBhtData_BHTGUID", "NMW_GtBhtData", ["BHTGUID"]) - - op.create_table( - "NMW_WsIntervals", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SampleID", sa.String(length=128), nullable=True), - sa.Column("From_Depth", sa.Float(), nullable=True), - sa.Column("To_Depth", sa.Float(), nullable=True), - sa.Column("From_TVD", sa.Float(), nullable=True), - sa.Column("To_TVD", sa.Float(), nullable=True), - sa.Column("From_Elev", sa.Float(), nullable=True), - sa.Column("To_Elev", sa.Float(), nullable=True), - sa.Column("Intv_Notes", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("IntrvlGUID"), - ) - op.create_index("ix_NMW_WsIntervals_SamplSetID", "NMW_WsIntervals", ["SamplSetID"]) - - op.create_table( - "NMW_GtConductivity", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Cnductvity", sa.Float(), nullable=True), - sa.Column("CnductUnit", sa.String(length=3), nullable=True), - sa.Column("Comments", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_GtConductivity_IntrvlGUID", "NMW_GtConductivity", ["IntrvlGUID"] - ) - - op.create_table( - "NMW_GtHeatFlow", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("IntrvlGUID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Gradient", sa.Float(), nullable=True), - sa.Column("Ka", sa.Float(), nullable=True), - sa.Column("Ka_unit", sa.String(length=3), nullable=True), - sa.Column("Pm", sa.Float(), nullable=True), - sa.Column("Kpr", sa.Float(), nullable=True), - sa.Column("Kpr_unit", sa.String(length=3), nullable=True), - sa.Column("Q", sa.Float(), nullable=True), - sa.Column("Q_unit", sa.String(length=3), nullable=True), - sa.Column("Comments", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index("ix_NMW_GtHeatFlow_IntrvlGUID", "NMW_GtHeatFlow", ["IntrvlGUID"]) - - op.create_table( - "NMW_GtSumHeatFlow", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("RecrdSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("LithClass", sa.String(length=50), nullable=True), - sa.Column("UnitBasis", sa.String(length=16), nullable=True), - sa.Column("UnitName", sa.String(length=128), nullable=True), - sa.Column("GeoID", sa.String(length=16), nullable=True), - sa.Column("FromDepth", sa.Float(), nullable=True), - sa.Column("ToDepth", sa.Float(), nullable=True), - sa.Column("DepthUnit", sa.String(length=8), nullable=True), - sa.Column("From_Elev", sa.Float(), nullable=True), - sa.Column("To_Elev", sa.Float(), nullable=True), - sa.Column("ThermlGrad", sa.Float(), nullable=True), - sa.Column("TGError", sa.Float(), nullable=True), - sa.Column("GradUnit", sa.String(length=3), nullable=True), - sa.Column("TGradRange", sa.String(length=15), nullable=True), - sa.Column("SampleType", sa.String(length=50), nullable=True), - sa.Column("NumSamples", sa.SmallInteger(), nullable=True), - sa.Column("ThermlCond", sa.Float(), nullable=True), - sa.Column("TCondError", sa.Float(), nullable=True), - sa.Column("TCondUnit", sa.String(length=3), nullable=True), - sa.Column("TCondRange", sa.String(length=15), nullable=True), - sa.Column("HeatFlow", sa.Float(), nullable=True), - sa.Column("HtFlowErr", sa.Float(), nullable=True), - sa.Column("HtFlowUnit", sa.String(length=3), nullable=True), - sa.Column("HtFlowEst", sa.Float(), nullable=True), - sa.Column("Quality", sa.String(length=50), nullable=True), - sa.Column("Comments", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_GtSumHeatFlow_RecrdSetID", "NMW_GtSumHeatFlow", ["RecrdSetID"] - ) - op.create_index( - "ix_NMW_GtSumHeatFlow_SamplSetID", "NMW_GtSumHeatFlow", ["SamplSetID"] - ) - - op.create_table( - "NMW_GtTempDepths", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Depth", sa.Float(), nullable=True), - sa.Column("Temp", sa.Float(), nullable=True), - sa.Column("TempUnit", sa.String(length=1), nullable=True), - sa.Column("IntrvlGrad", sa.Float(), nullable=True), - sa.Column("Comments", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_GtTempDepths_SamplSetID", "NMW_GtTempDepths", ["SamplSetID"] - ) - - op.create_table( - "NMW_WsDstHeaders", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("SamplSetID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("TestType", sa.String(length=50), nullable=True), - sa.Column("DSTOprator", sa.String(length=50), nullable=True), - sa.Column("PressUnits", sa.String(length=8), nullable=True), - sa.Column("TempUnit", sa.String(length=1), nullable=True), - sa.Column("PipeDiaUnt", sa.String(length=8), nullable=True), - sa.Column("PipeLenUnt", sa.String(length=8), nullable=True), - sa.Column("ChokeSizUn", sa.String(length=8), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("DSTGUID"), - ) - op.create_index( - "ix_NMW_WsDstHeaders_SamplSetID", "NMW_WsDstHeaders", ["SamplSetID"] - ) - - op.create_table( - "NMW_WsDstIntervals", - sa.Column("OBJECTID", sa.Integer(), nullable=True), - sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("DSTGUID", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("DSTName", sa.String(length=128), nullable=True), - sa.Column("TargetFm", sa.String(length=16), nullable=True), - sa.Column("DSTDate", sa.DateTime(), nullable=True), - sa.Column("DSTNumber", sa.SmallInteger(), nullable=True), - sa.Column("Status", sa.String(length=255), nullable=True), - sa.Column("StatusDate", sa.DateTime(), nullable=True), - sa.Column("PackrFrom", sa.Float(), nullable=True), - sa.Column("PackerTo", sa.Float(), nullable=True), - sa.Column("SrfChokeSz", sa.Float(), nullable=True), - sa.Column("BotChokeSz", sa.Float(), nullable=True), - sa.Column("PipeDia", sa.Float(), nullable=True), - sa.Column("PipeLength", sa.Float(), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("DSTInterval"), - ) - op.create_index("ix_NMW_WsDstIntervals_DSTGUID", "NMW_WsDstIntervals", ["DSTGUID"]) - - op.create_table( - "NMW_WsDstFlowHistory", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("Operation", sa.String(length=255), nullable=True), - sa.Column("StartTime", sa.DateTime(), nullable=True), - sa.Column("EndTime", sa.DateTime(), nullable=True), - sa.Column("Duration", sa.Float(), nullable=True), - sa.Column("Pressure", sa.Float(), nullable=True), - sa.Column("Temp", sa.Float(), nullable=True), - sa.Column("RecovColmn", sa.Float(), nullable=True), - sa.Column("RecovType", sa.String(length=255), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_WsDstFlowHistory_DSTInterval", "NMW_WsDstFlowHistory", ["DSTInterval"] - ) - - op.create_table( - "NMW_WsDstFluidProperties", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("SourceLoc", sa.String(length=255), nullable=True), - sa.Column("Resistivty", sa.Float(), nullable=True), - sa.Column("Temp", sa.Float(), nullable=True), - sa.Column("Chlorides", sa.Float(), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_WsDstFluidProperties_DSTInterval", - "NMW_WsDstFluidProperties", - ["DSTInterval"], - ) - - op.create_table( - "NMW_WsDstPressure", - sa.Column("OBJECTID", sa.Integer(), nullable=False), - sa.Column("DSTInterval", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("PrsGageDpt", sa.Float(), nullable=True), - sa.Column("BlankedOff", sa.SmallInteger(), nullable=True), - sa.Column("InShtInMin", sa.Float(), nullable=True), - sa.Column("FlwPrsInMin", sa.Float(), nullable=True), - sa.Column("PrsInShtIn", sa.Float(), nullable=True), - sa.Column("PrsInitClsdIn", sa.Float(), nullable=True), - sa.Column("FnShtInMin", sa.Float(), nullable=True), - sa.Column("FlwPrsFinMin", sa.Float(), nullable=True), - sa.Column("PrsFnShtIn", sa.Float(), nullable=True), - sa.Column("ShtInPrMth", sa.String(length=255), nullable=True), - sa.Column("HydrostPrsIn", sa.Float(), nullable=True), - sa.Column("HydStPrsFl", sa.Float(), nullable=True), - sa.Column("HydstPrMth", sa.String(length=255), nullable=True), - sa.Column("EquilPress", sa.Float(), nullable=True), - sa.Column("EqlPrsMth", sa.String(length=255), nullable=True), - sa.Column("FlowPrsMin", sa.Float(), nullable=True), - sa.Column("FlowPrsMax", sa.Float(), nullable=True), - sa.Column("FlowPrsMth", sa.String(length=255), nullable=True), - sa.Column("DSTFluid", sa.String(length=128), nullable=True), - sa.Column("FmTemp", sa.Float(), nullable=True), - sa.Column("TempCorrtn", sa.Float(), nullable=True), - sa.Column("TempFlowng", sa.Float(), nullable=True), - sa.Column("TempUnit", sa.String(length=5), nullable=True), - sa.Column("Notes", sa.String(length=255), nullable=True), - sa.Column("GlobalID", postgresql.UUID(as_uuid=True), nullable=True), - sa.PrimaryKeyConstraint("OBJECTID"), - ) - op.create_index( - "ix_NMW_WsDstPressure_DSTInterval", "NMW_WsDstPressure", ["DSTInterval"] - ) - - -def downgrade() -> None: - """Downgrade schema.""" - op.drop_index("ix_NMW_WsDstPressure_DSTInterval", table_name="NMW_WsDstPressure") - op.drop_table("NMW_WsDstPressure") - op.drop_index( - "ix_NMW_WsDstFluidProperties_DSTInterval", table_name="NMW_WsDstFluidProperties" - ) - op.drop_table("NMW_WsDstFluidProperties") - op.drop_index( - "ix_NMW_WsDstFlowHistory_DSTInterval", table_name="NMW_WsDstFlowHistory" - ) - op.drop_table("NMW_WsDstFlowHistory") - op.drop_index("ix_NMW_WsDstIntervals_DSTGUID", table_name="NMW_WsDstIntervals") - op.drop_table("NMW_WsDstIntervals") - op.drop_index("ix_NMW_WsDstHeaders_SamplSetID", table_name="NMW_WsDstHeaders") - op.drop_table("NMW_WsDstHeaders") - op.drop_index("ix_NMW_GtTempDepths_SamplSetID", table_name="NMW_GtTempDepths") - op.drop_table("NMW_GtTempDepths") - op.drop_index("ix_NMW_GtSumHeatFlow_RecrdSetID", table_name="NMW_GtSumHeatFlow") - op.drop_index("ix_NMW_GtSumHeatFlow_SamplSetID", table_name="NMW_GtSumHeatFlow") - op.drop_table("NMW_GtSumHeatFlow") - op.drop_index("ix_NMW_GtHeatFlow_IntrvlGUID", table_name="NMW_GtHeatFlow") - op.drop_table("NMW_GtHeatFlow") - op.drop_index("ix_NMW_GtConductivity_IntrvlGUID", table_name="NMW_GtConductivity") - op.drop_table("NMW_GtConductivity") - op.drop_index("ix_NMW_WsIntervals_SamplSetID", table_name="NMW_WsIntervals") - op.drop_table("NMW_WsIntervals") - op.drop_index("ix_NMW_GtBhtData_BHTGUID", table_name="NMW_GtBhtData") - op.drop_table("NMW_GtBhtData") - op.drop_index("ix_NMW_GtBhtHeaders_SamplSetID", table_name="NMW_GtBhtHeaders") - op.drop_table("NMW_GtBhtHeaders") diff --git a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py b/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py deleted file mode 100644 index 6e89bff9b..000000000 --- a/alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py +++ /dev/null @@ -1,173 +0,0 @@ -"""add geothermal OGC views (bottom-hole temps + temperature-depth profile) - -Revision ID: w9x0y1z2a3b4 -Revises: v8w9x0y1z2a3 -Create Date: 2026-06-07 00:00:00.000000 - -Two point layers over the NM_Wells staging mirror (db/nmw_legacy.py): - - ogc_geothermal_wells_bht - One feature per geothermal well that has bottom-hole-temperature data - (NMW_GtBhtData), with aggregate BHT stats. - - ogc_geothermal_wells_temperature_profile (MATERIALIZED) - One feature per geothermal well that has a downhole temperature-vs-depth - series (NMW_GtTempDepths, ~370k source rows), with the ordered series as - a JSON array. Materialized + indexed (unique well_data_id, GiST geom); - REFRESH MATERIALIZED VIEW after a data reload. - -Well geometry is built from NMW_WellLocations Lat/Long_dd83 (WGS84). Geothermal -data links to a well via: - gt_*.SamplSetID -> NMW_WellSamples.SamplSetID - NMW_WellSamples.RecrdsetID -> NMW_WellRecords.RecrdSetID - NMW_WellRecords.WellDataID -> NMW_WellLocations/Headers.WellDataID -""" - -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import inspect, text - -# revision identifiers, used by Alembic. -revision: str = "w9x0y1z2a3b4" -down_revision: Union[str, Sequence[str], None] = "v8w9x0y1z2a3" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_BHT_VIEW = "ogc_geothermal_wells_bht" -_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" - -_REQUIRED_TABLES = ( - "NMW_WellLocations", - "NMW_WellHeaders", - "NMW_WellRecords", - "NMW_WellSamples", - "NMW_GtBhtData", - "NMW_GtBhtHeaders", - "NMW_GtTempDepths", -) - - -def _create_bht_view() -> str: - return """ - CREATE VIEW ogc_geothermal_wells_bht AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - hdr."TotalDepth" AS total_depth, - count(d.*) AS bht_count, - max(d."BHT") AS max_bht, - min(d."BHT") AS min_bht, - max(d."Depth") AS max_bht_depth, - max(d."TempUnit") AS temp_unit, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtBhtData" AS d - JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API", - hdr."TotalDepth" - """ - - -def _create_profile_view() -> str: - # Materialized: the source NMW_GtTempDepths is large (~370k source rows) and - # this groups + builds a JSON series per well, too heavy to recompute per - # pygeoapi request. Staging data loads once, so staleness is a non-issue; - # REFRESH MATERIALIZED VIEW after a reload. - return """ - CREATE MATERIALIZED VIEW ogc_geothermal_wells_temperature_profile AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(td.*) AS reading_count, - min(td."Depth") AS min_depth, - max(td."Depth") AS max_depth, - min(td."Temp") AS min_temp, - max(td."Temp") AS max_temp, - max(td."TempUnit") AS temp_unit, - json_agg( - json_build_object('depth', td."Depth", 'temp', td."Temp") - ORDER BY td."Depth" - ) AS series, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtTempDepths" AS td - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - AND td."Depth" IS NOT NULL - AND td."Temp" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - - -def upgrade() -> None: - bind = op.get_bind() - inspector = inspect(bind) - existing = set(inspector.get_table_names(schema="public")) - missing = [t for t in _REQUIRED_TABLES if t not in existing] - if missing: - raise RuntimeError( - "Cannot create geothermal OGC views. Missing required tables: " - + ", ".join(missing) - ) - - op.execute(text(f"DROP VIEW IF EXISTS {_BHT_VIEW}")) - op.execute(text(_create_bht_view())) - op.execute( - text( - f"COMMENT ON VIEW {_BHT_VIEW} IS " - "'Geothermal wells with bottom-hole-temperature data.'" - ) - ) - - op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {_PROFILE_VIEW}")) - op.execute(text(f"DROP VIEW IF EXISTS {_PROFILE_VIEW}")) - op.execute(text(_create_profile_view())) - op.execute( - text( - f"COMMENT ON MATERIALIZED VIEW {_PROFILE_VIEW} IS " - "'Geothermal wells with downhole temperature-vs-depth series.'" - ) - ) - # Unique index on the feature id enables REFRESH ... CONCURRENTLY; GiST on - # the geometry for fast pygeoapi bbox queries. - op.execute( - text( - f"CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_well_data_id " - f"ON {_PROFILE_VIEW} (well_data_id)" - ) - ) - op.execute( - text( - f"CREATE INDEX ix_{_PROFILE_VIEW}_geom ON {_PROFILE_VIEW} USING gist (geom)" - ) - ) - - -def downgrade() -> None: - op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {_PROFILE_VIEW}")) - op.execute(text(f"DROP VIEW IF EXISTS {_BHT_VIEW}")) diff --git a/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py b/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py deleted file mode 100644 index 5422102ae..000000000 --- a/alembic/versions/x0y1z2a3b4c5_add_geothermal_heat_flow_ogc_view.py +++ /dev/null @@ -1,109 +0,0 @@ -"""add geothermal heat-flow OGC view - -Revision ID: x0y1z2a3b4c5 -Revises: w9x0y1z2a3b4 -Create Date: 2026-06-07 00:00:01.000000 - -pygeoapi point layer ogc_geothermal_wells_summary_heat_flow: geothermal wells -with summary heat-flow determinations (NMW_GtSumHeatFlow), one feature per well -with aggregate stats plus a `measurements` JSON series (one element per -determination, ordered by depth). Geometry from NMW_WellLocations Lat/Long_dd83. - -Link: NMW_GtSumHeatFlow.RecrdSetID -> NMW_WellRecords.RecrdSetID -> -NMW_WellLocations/Headers.WellDataID. -""" - -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import inspect, text - -# revision identifiers, used by Alembic. -revision: str = "x0y1z2a3b4c5" -down_revision: Union[str, Sequence[str], None] = "w9x0y1z2a3b4" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_VIEW = "ogc_geothermal_wells_summary_heat_flow" - -_REQUIRED_TABLES = ( - "NMW_WellLocations", - "NMW_WellHeaders", - "NMW_WellRecords", - "NMW_GtSumHeatFlow", -) - - -def _create_view() -> str: - return """ - CREATE VIEW ogc_geothermal_wells_summary_heat_flow AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(shf.*) AS heat_flow_count, - max(shf."HeatFlow") AS max_heat_flow, - avg(shf."HeatFlow") AS avg_heat_flow, - max(shf."HtFlowUnit") AS heat_flow_unit, - max(shf."ThermlGrad") AS max_thermal_gradient, - max(shf."GradUnit") AS gradient_unit, - max(shf."ThermlCond") AS max_thermal_conductivity, - max(shf."TCondUnit") AS conductivity_unit, - max(shf."Quality") AS quality, - json_agg( - json_build_object( - 'from_depth', shf."FromDepth", - 'to_depth', shf."ToDepth", - 'depth_unit', shf."DepthUnit", - 'heat_flow', shf."HeatFlow", - 'heat_flow_error', shf."HtFlowErr", - 'heat_flow_unit', shf."HtFlowUnit", - 'thermal_gradient', shf."ThermlGrad", - 'gradient_unit', shf."GradUnit", - 'thermal_conductivity', shf."ThermlCond", - 'conductivity_unit', shf."TCondUnit", - 'quality', shf."Quality" - ) - ORDER BY shf."FromDepth" - ) AS measurements, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtSumHeatFlow" AS shf - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - - -def upgrade() -> None: - bind = op.get_bind() - inspector = inspect(bind) - existing = set(inspector.get_table_names(schema="public")) - missing = [t for t in _REQUIRED_TABLES if t not in existing] - if missing: - raise RuntimeError( - "Cannot create geothermal heat-flow OGC view. Missing required " - "tables: " + ", ".join(missing) - ) - - op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) - op.execute(text(_create_view())) - op.execute( - text( - f"COMMENT ON VIEW {_VIEW} IS " - "'Geothermal wells with summary heat-flow determinations (pygeoapi).'" - ) - ) - - -def downgrade() -> None: - op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) diff --git a/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py b/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py deleted file mode 100644 index 12f570183..000000000 --- a/alembic/versions/y1z2a3b4c5d6_add_geothermal_interval_heat_flow_ogc_view.py +++ /dev/null @@ -1,112 +0,0 @@ -"""add geothermal per-interval heat-flow OGC view - -Revision ID: y1z2a3b4c5d6 -Revises: x0y1z2a3b4c5 -Create Date: 2026-06-07 00:00:02.000000 - -pygeoapi point layer of geothermal wells with per-interval heat-flow values -(NMW_GtHeatFlow), one feature per well with aggregate stats plus a -`measurements` JSON series (one element per interval, ordered by depth). -Distinct from ogc_geothermal_wells_summary_heat_flow (NMW_GtSumHeatFlow). - -Link: NMW_GtHeatFlow.IntrvlGUID -> NMW_WsIntervals.IntrvlGUID -> -NMW_WellSamples.SamplSetID -> NMW_WellRecords.RecrdSetID -> -NMW_WellLocations/Headers.WellDataID. -""" - -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import inspect, text - -# revision identifiers, used by Alembic. -revision: str = "y1z2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = "x0y1z2a3b4c5" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_VIEW = "ogc_geothermal_wells_interval_heat_flow" - -_REQUIRED_TABLES = ( - "NMW_WellLocations", - "NMW_WellHeaders", - "NMW_WellRecords", - "NMW_WellSamples", - "NMW_WsIntervals", - "NMW_GtHeatFlow", -) - - -def _create_view() -> str: - return """ - CREATE VIEW ogc_geothermal_wells_interval_heat_flow AS - SELECT - r."WellDataID" AS well_data_id, - hdr."CurWellNam" AS well_name, - hdr."API" AS api, - count(hf.*) AS interval_count, - max(hf."Q") AS max_heat_flow, - avg(hf."Q") AS avg_heat_flow, - max(hf."Q_unit") AS heat_flow_unit, - max(hf."Gradient") AS max_gradient, - max(hf."Kpr") AS max_thermal_conductivity, - max(hf."Kpr_unit") AS conductivity_unit, - max(hf."Ka") AS max_diffusivity, - max(hf."Ka_unit") AS diffusivity_unit, - json_agg( - json_build_object( - 'from_depth', i."From_Depth", - 'to_depth', i."To_Depth", - 'heat_flow', hf."Q", - 'heat_flow_unit', hf."Q_unit", - 'gradient', hf."Gradient", - 'thermal_conductivity', hf."Kpr", - 'conductivity_unit', hf."Kpr_unit", - 'diffusivity', hf."Ka", - 'diffusivity_unit', hf."Ka_unit" - ) - ORDER BY i."From_Depth" - ) AS measurements, - ST_SetSRID( - ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 - ) AS geom - FROM "NMW_GtHeatFlow" AS hf - JOIN "NMW_WsIntervals" AS i ON i."IntrvlGUID" = hf."IntrvlGUID" - JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = i."SamplSetID" - JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" - LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL - GROUP BY - r."WellDataID", - loc."Lat_dd83", - loc."Long_dd83", - hdr."CurWellNam", - hdr."API" - """ - - -def upgrade() -> None: - bind = op.get_bind() - inspector = inspect(bind) - existing = set(inspector.get_table_names(schema="public")) - missing = [t for t in _REQUIRED_TABLES if t not in existing] - if missing: - raise RuntimeError( - "Cannot create geothermal interval heat-flow OGC view. Missing " - "required tables: " + ", ".join(missing) - ) - - op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) - op.execute(text(_create_view())) - op.execute( - text( - f"COMMENT ON VIEW {_VIEW} IS " - "'Geothermal wells with per-interval heat-flow values (pygeoapi).'" - ) - ) - - -def downgrade() -> None: - op.execute(text(f"DROP VIEW IF EXISTS {_VIEW}")) diff --git a/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py b/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py deleted file mode 100644 index b42dfd5a4..000000000 --- a/alembic/versions/z2a3b4c5d6e7_add_fk_constraints_to_nmw_mirror_tables.py +++ /dev/null @@ -1,210 +0,0 @@ -"""add FK constraints to NMW staging mirror tables - -Revision ID: z2a3b4c5d6e7 -Revises: y1z2a3b4c5d6 -Create Date: 2026-06-15 - -""" - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "z2a3b4c5d6e7" -down_revision = "y1z2a3b4c5d6" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # WellLocations -> WellHeaders - op.create_foreign_key( - "fk_nmw_welllocations_welldataid", - "NMW_WellLocations", - "NMW_WellHeaders", - ["WellDataID"], - ["WellDataID"], - ) - # WellRecords -> WellHeaders - op.create_foreign_key( - "fk_nmw_wellrecords_welldataid", - "NMW_WellRecords", - "NMW_WellHeaders", - ["WellDataID"], - ["WellDataID"], - ) - # WellZDatum -> WellRecords - op.create_foreign_key( - "fk_nmw_wellzdatum_recrdsetid", - "NMW_WellZDatum", - "NMW_WellRecords", - ["RecrdsetID"], - ["RecrdSetID"], - ) - # WellSamples -> WellRecords - op.create_foreign_key( - "fk_nmw_wellsamples_recrdsetid", - "NMW_WellSamples", - "NMW_WellRecords", - ["RecrdsetID"], - ["RecrdSetID"], - ) - # GtBhtHeaders -> WellSamples - op.create_foreign_key( - "fk_nmw_gtbhtheaders_samplsetid", - "NMW_GtBhtHeaders", - "NMW_WellSamples", - ["SamplSetID"], - ["SamplSetID"], - ) - # GtBhtData -> GtBhtHeaders - op.create_foreign_key( - "fk_nmw_gtbhtdata_bhtguid", - "NMW_GtBhtData", - "NMW_GtBhtHeaders", - ["BHTGUID"], - ["BHTGUID"], - ) - # WsIntervals -> WellSamples - op.create_foreign_key( - "fk_nmw_wsintervals_samplsetid", - "NMW_WsIntervals", - "NMW_WellSamples", - ["SamplSetID"], - ["SamplSetID"], - ) - # GtConductivity -> WsIntervals - op.create_foreign_key( - "fk_nmw_gtconductivity_intrvlguid", - "NMW_GtConductivity", - "NMW_WsIntervals", - ["IntrvlGUID"], - ["IntrvlGUID"], - ) - # GtHeatFlow -> WsIntervals - op.create_foreign_key( - "fk_nmw_gtheatflow_intrvlguid", - "NMW_GtHeatFlow", - "NMW_WsIntervals", - ["IntrvlGUID"], - ["IntrvlGUID"], - ) - # GtSumHeatFlow -> WellRecords - op.create_foreign_key( - "fk_nmw_gtsumheatflow_recrdsetid", - "NMW_GtSumHeatFlow", - "NMW_WellRecords", - ["RecrdSetID"], - ["RecrdSetID"], - ) - # GtSumHeatFlow -> WellSamples - op.create_foreign_key( - "fk_nmw_gtsumheatflow_samplsetid", - "NMW_GtSumHeatFlow", - "NMW_WellSamples", - ["SamplSetID"], - ["SamplSetID"], - ) - # GtTempDepths -> WellSamples - op.create_foreign_key( - "fk_nmw_gttempdepths_samplsetid", - "NMW_GtTempDepths", - "NMW_WellSamples", - ["SamplSetID"], - ["SamplSetID"], - ) - # WsDstHeaders -> WellSamples - op.create_foreign_key( - "fk_nmw_wsdstheaders_samplsetid", - "NMW_WsDstHeaders", - "NMW_WellSamples", - ["SamplSetID"], - ["SamplSetID"], - ) - # WsDstIntervals -> WsDstHeaders - op.create_foreign_key( - "fk_nmw_wsdstintervals_dstguid", - "NMW_WsDstIntervals", - "NMW_WsDstHeaders", - ["DSTGUID"], - ["DSTGUID"], - ) - # WsDstFlowHistory -> WsDstIntervals - op.create_foreign_key( - "fk_nmw_wsdstflowhistory_dstinterval", - "NMW_WsDstFlowHistory", - "NMW_WsDstIntervals", - ["DSTInterval"], - ["DSTInterval"], - ) - # WsDstFluidProperties -> WsDstIntervals - op.create_foreign_key( - "fk_nmw_wsdstfluidproperties_dstinterval", - "NMW_WsDstFluidProperties", - "NMW_WsDstIntervals", - ["DSTInterval"], - ["DSTInterval"], - ) - # WsDstPressure -> WsDstIntervals - op.create_foreign_key( - "fk_nmw_wsdstpressure_dstinterval", - "NMW_WsDstPressure", - "NMW_WsDstIntervals", - ["DSTInterval"], - ["DSTInterval"], - ) - - -def downgrade() -> None: - op.drop_constraint( - "fk_nmw_wsdstpressure_dstinterval", "NMW_WsDstPressure", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wsdstfluidproperties_dstinterval", - "NMW_WsDstFluidProperties", - type_="foreignkey", - ) - op.drop_constraint( - "fk_nmw_wsdstflowhistory_dstinterval", - "NMW_WsDstFlowHistory", - type_="foreignkey", - ) - op.drop_constraint( - "fk_nmw_wsdstintervals_dstguid", "NMW_WsDstIntervals", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wsdstheaders_samplsetid", "NMW_WsDstHeaders", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_gttempdepths_samplsetid", "NMW_GtTempDepths", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_gtsumheatflow_samplsetid", "NMW_GtSumHeatFlow", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_gtsumheatflow_recrdsetid", "NMW_GtSumHeatFlow", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_gtheatflow_intrvlguid", "NMW_GtHeatFlow", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_gtconductivity_intrvlguid", "NMW_GtConductivity", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wsintervals_samplsetid", "NMW_WsIntervals", type_="foreignkey" - ) - op.drop_constraint("fk_nmw_gtbhtdata_bhtguid", "NMW_GtBhtData", type_="foreignkey") - op.drop_constraint( - "fk_nmw_gtbhtheaders_samplsetid", "NMW_GtBhtHeaders", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wellsamples_recrdsetid", "NMW_WellSamples", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wellzdatum_recrdsetid", "NMW_WellZDatum", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_wellrecords_welldataid", "NMW_WellRecords", type_="foreignkey" - ) - op.drop_constraint( - "fk_nmw_welllocations_welldataid", "NMW_WellLocations", type_="foreignkey" - ) diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 89b9ee1ea..28f78945a 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -106,28 +106,35 @@ class MirrorSpec: # All NMW_* mirror tables. Order is irrelevant (no enforced cross-table FKs in # the staging layer), but parents are listed before children for readability. NMW_MIRROR_SPECS: list[MirrorSpec] = [ - # Main - MirrorSpec(NMW_WellLocations, "tbl_well_locations"), + # Parents before children (FK constraints enforced on insert) + # Root MirrorSpec(NMW_WellHeaders, "tbl_well_headers"), + # -> WellHeaders + MirrorSpec(NMW_WellLocations, "tbl_well_locations"), MirrorSpec(NMW_WellRecords, "tbl_well_records"), + # -> WellRecords MirrorSpec(NMW_WellZDatum, "tbl_well_z_datum"), MirrorSpec(NMW_WellSamples, "tbl_well_samples"), - # Publications + # Publications (standalone, no FK) MirrorSpec(NMW_Sources, "tbl_sources"), - # Geothermal + # Geothermal -> WellSamples MirrorSpec(NMW_GtBhtHeaders, "tbl_gt_bht_headers"), - MirrorSpec(NMW_GtBhtData, "tbl_gt_bht_data"), + MirrorSpec(NMW_GtBhtData, "tbl_gt_bht_data"), # -> GtBhtHeaders MirrorSpec(NMW_WsIntervals, "tbl_ws_intervals"), - MirrorSpec(NMW_GtConductivity, "tbl_gt_conductivity"), - MirrorSpec(NMW_GtHeatFlow, "tbl_gt_heat_flow"), - MirrorSpec(NMW_GtSumHeatFlow, "tbl_gt_sum_heat_flow"), - MirrorSpec(NMW_GtTempDepths, "tbl_gt_temp_depths"), - # Drill Stem Tests + MirrorSpec(NMW_GtConductivity, "tbl_gt_conductivity"), # -> WsIntervals + MirrorSpec(NMW_GtHeatFlow, "tbl_gt_heat_flow"), # -> WsIntervals + MirrorSpec( + NMW_GtSumHeatFlow, "tbl_gt_sum_heat_flow" + ), # -> WellRecords + WellSamples + MirrorSpec(NMW_GtTempDepths, "tbl_gt_temp_depths"), # -> WellSamples + # Drill Stem Tests -> WellSamples MirrorSpec(NMW_WsDstHeaders, "tbl_ws_dst_headers"), - MirrorSpec(NMW_WsDstIntervals, "tbl_ws_dst_intervals"), - MirrorSpec(NMW_WsDstFlowHistory, "tbl_ws_dst_flow_history"), - MirrorSpec(NMW_WsDstFluidProperties, "tbl_ws_dst_fluid_properties"), - MirrorSpec(NMW_WsDstPressure, "tbl_ws_dst_pressure"), + MirrorSpec(NMW_WsDstIntervals, "tbl_ws_dst_intervals"), # -> WsDstHeaders + MirrorSpec(NMW_WsDstFlowHistory, "tbl_ws_dst_flow_history"), # -> WsDstIntervals + MirrorSpec( + NMW_WsDstFluidProperties, "tbl_ws_dst_fluid_properties" + ), # -> WsDstIntervals + MirrorSpec(NMW_WsDstPressure, "tbl_ws_dst_pressure"), # -> WsDstIntervals ] From d54fa6aee115152ffd9d1bb4a46ad5b37d914835 Mon Sep 17 00:00:00 2001 From: peterrowland <1530932+peterrowland@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:16:42 +0000 Subject: [PATCH 31/41] Formatting changes --- ...4b5c6_nmw_per_well_geothermal_ogc_views.py | 32 +++++-------------- .../e2f3a4b5c6d7_nmw_measurement_ogc_views.py | 32 +++++-------------- 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py index 607c9ccd3..0931b3596 100644 --- a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py +++ b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py @@ -48,9 +48,7 @@ def upgrade() -> None: # ogc_geothermal_wells_bht op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_BHT_VIEW}" AS SELECT row_number() OVER () AS id, @@ -81,15 +79,11 @@ def upgrade() -> None: hdr."CurWellNam", hdr."API", hdr."TotalDepth" - """ - ) - ) + """)) # ogc_geothermal_wells_temperature_profile (materialized) op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -130,9 +124,7 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) op.execute( text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') ) @@ -144,9 +136,7 @@ def upgrade() -> None: # ogc_geothermal_wells_summary_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_SUM_HF_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_SUM_HF_VIEW}" AS SELECT row_number() OVER () AS id, @@ -193,15 +183,11 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) # ogc_geothermal_wells_interval_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_INT_HF_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_INT_HF_VIEW}" AS SELECT row_number() OVER () AS id, @@ -248,9 +234,7 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) def downgrade() -> None: diff --git a/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py b/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py index 54958d794..f54ac694f 100644 --- a/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py +++ b/alembic/versions/e2f3a4b5c6d7_nmw_measurement_ogc_views.py @@ -50,9 +50,7 @@ def upgrade() -> None: # ogc_bht_measurements op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_MEAS_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_BHT_MEAS_VIEW}" AS SELECT d."OBJECTID" AS id, @@ -84,15 +82,11 @@ def upgrade() -> None: JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" WHERE loc."Lat_dd83" IS NOT NULL AND loc."Long_dd83" IS NOT NULL - """ - ) - ) + """)) # ogc_temp_depth_measurements op.execute(text(f'DROP VIEW IF EXISTS "{_TEMP_DEPTH_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_TEMP_DEPTH_VIEW}" AS SELECT td."OBJECTID" AS id, @@ -130,15 +124,11 @@ def upgrade() -> None: WHERE loc."Exclude" = 0 AND loc."Lat_dd83" IS NOT NULL AND loc."Long_dd83" IS NOT NULL - """ - ) - ) + """)) # ogc_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_HEAT_FLOW_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_HEAT_FLOW_VIEW}" AS SELECT shf."OBJECTID" AS id, @@ -212,15 +202,11 @@ def upgrade() -> None: WHERE loc."Exclude" = 0 AND loc."Lat_dd83" IS NOT NULL AND loc."Long_dd83" IS NOT NULL - """ - ) - ) + """)) # ogc_dst op.execute(text(f'DROP VIEW IF EXISTS "{_DST_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_DST_VIEW}" AS WITH flow_history AS ( SELECT @@ -283,9 +269,7 @@ def upgrade() -> None: LEFT JOIN flow_history AS fh ON fh."DSTInterval" = i."DSTInterval" WHERE loc."Lat_dd83" IS NOT NULL AND loc."Long_dd83" IS NOT NULL - """ - ) - ) + """)) def downgrade() -> None: From ccf566d9b41b5222f5fdebd2874996bebe2cac92 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 17:43:00 -0600 Subject: [PATCH 32/41] docs(nmw): land nm_wells-migration.md referenced by mirror code Doc existed untracked in a worktree and docs/ is gitignored, so the NM_Wells migration plan referenced 4x in db/nmw_legacy.py and transfers/nmw_mirror_transfer.py pointed at a missing file. Force-add to land it on this branch (matches existing docs/ force-add pattern). Co-Authored-By: Claude Opus 4.8 --- docs/nm_wells-migration.md | 190 +++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 docs/nm_wells-migration.md diff --git a/docs/nm_wells-migration.md b/docs/nm_wells-migration.md new file mode 100644 index 000000000..35f335b5d --- /dev/null +++ b/docs/nm_wells-migration.md @@ -0,0 +1,190 @@ +# NM_Wells → Ocotillo migration + +Migration of the legacy **NM_Wells** SQL Server database (and the related +Subsurface Library) into OcotilloAPI. Source of truth for table inventory and +field-level recommendations: `NM_Wells + Subsurface library.xlsx` (planning +workbook, not in repo). + +## Two-phase approach + +1. **Phase 1 — 1:1 staging mirror** *(current)*. Land source tables unchanged + into `NMW_*` mirror tables (`db/nmw_legacy.py`), column-for-column. No + transform. Mirrors the `db/nma_legacy.py` (NM_Aquifer) convention. This lets + us load the SQL dump first and transform later without re-reading the source. +2. **Phase 2 — transform** *(later)*. Map mirror rows into the Ocotillo data + model (`Location → Thing → FieldEvent → FieldActivity → Sample → + Observation`, plus `status_history`, `measuring_point_history`, `contact`, + `publication`, `thing_geologic_formation_association`, `thing_id_link`) using + the existing CSV→Pandas→ORM transfer pattern in `transfers/`. Per-column + targets are recorded inline in `db/nmw_legacy.py` and summarized below. + +## Source access + +NM_Wells is delivered as a SQL dump. Physical source table names are +`tbl_well_*` / `tbl_gt_*` / `tbl_ws_*` (snake_case). To use the existing +transfer pipeline, export each source table to CSV (same flow as the +NM_Aquifer `nma_csv_cache`). + +## Phase 1 scope (mirrored now) + +Five "Migrate First / Main" tables — the only ones with an authoritative +field-level mapping in the workbook (sheet 3): + +| Source table | Mirror model | Cols | +|----------------------|----------------------|------| +| `tbl_well_locations` | `NMW_WellLocations` | 40 | +| `tbl_well_headers` | `NMW_WellHeaders` | 35 | +| `tbl_well_records` | `NMW_WellRecords` | 12 | +| `tbl_well_z_datum` | `NMW_WellZDatum` | 18 | +| `tbl_well_samples` | `NMW_WellSamples` | 30 | + +Migration: `alembic/versions/u7v8w9x0y1z2_nmw_legacy_staging_mirror_tables.py`. + +Geothermal + Drill Stem Test "Migrate First" tables are also mirrored (columns ++ lengths taken straight from the SQL-dump DDL): + +| Source table | Mirror model | Cols | PK | +|-------------------------------|----------------------------|------|----| +| `tbl_gt_bht_headers` | `NMW_GtBhtHeaders` | 16 | BHTGUID | +| `tbl_gt_bht_data` | `NMW_GtBhtData` | 10 | OBJECTID | +| `tbl_ws_intervals` | `NMW_WsIntervals` | 12 | IntrvlGUID | +| `tbl_gt_conductivity` | `NMW_GtConductivity` | 7 | OBJECTID | +| `tbl_gt_heat_flow` | `NMW_GtHeatFlow` | 13 | OBJECTID | +| `tbl_gt_sum_heat_flow` | `NMW_GtSumHeatFlow` | 30 | OBJECTID | +| `tbl_gt_temp_depths` | `NMW_GtTempDepths` | 9 | OBJECTID | +| `tbl_ws_dst_headers` | `NMW_WsDstHeaders` | 11 | DSTGUID | +| `tbl_ws_dst_intervals` | `NMW_WsDstIntervals` | 17 | DSTInterval | +| `tbl_ws_dst_flow_history` | `NMW_WsDstFlowHistory` | 13 | OBJECTID | +| `tbl_ws_dst_fluid_properties` | `NMW_WsDstFluidProperties`| 9 | OBJECTID | +| `tbl_ws_dst_pressure` | `NMW_WsDstPressure` | 28 | OBJECTID | + +Migration: `alembic/versions/v8w9x0y1z2a3_nmw_geothermal_dst_mirror_tables.py`. + +Link columns (`SamplSetID`, `BHTGUID`, `IntrvlGUID`, `DSTGUID`, `DSTInterval`, +`RecrdSetID`) are kept as plain indexed GUID columns — NOT enforced FKs — since +this is staging. Relationship chains documented in `db/nmw_legacy.py`. + +### PK / type notes +- SQL Server `uniqueidentifier` → postgres `UUID`; `real/float` → `Float`; + `nvarchar` → `String` (source lengths absent from the sheet, so widened); + `datetime2` → `DateTime`; `timestamp` (rowversion) → **dropped**. +- PKs **verified against the dump DDL**: Headers→`WellDataID`, + Records→`RecrdSetID`, Samples→`SamplSetID` (declared PKs); + Locations→`OBJECTID`, ZDatum→`OBJECTID` (no declared PK in source — unique + indexes on both OBJECTID and GlobalID; OBJECTID identity is never NULL). + +## Loading the mirror (Phase 1 transfer) + +`transfers/nmw_mirror_transfer.py` (`transfer_nmw_mirror(session, limit)`) loads +each source table into its `NMW_*` table 1:1. It is data-driven over +`NMW_MIRROR_SPECS` (one `(model, source_table)` per mirror), derives column +handling from each model's `__table__` metadata, coerces types +(uuid/int/float/datetime/string; NULL/NaN/NaT → None; rowversion dropped), and +chunk-upserts via `INSERT ... ON CONFLICT () DO NOTHING`. + +**Row source** (selected at runtime): +- **SQL Server data dump** — set `NMW_SQL_DUMP` to a `.sql` file of + `INSERT [dbo].[tbl_*] (...) VALUES (...)` statements. `transfers/nmw_sql_dump.py` + splits statements with **sqlparse** and `write_table_csv` writes one CSV per + table (handles `N'...'`/escaped `''`, embedded commas/parens, `CAST(... AS ...)`, + multi-row `VALUES`, `0x` binary → NULL, UTF-16/UTF-8 BOM). The mirror then + bulk-loads each CSV with Postgres **`COPY ... FROM STDIN`** (truncate + COPY; + Postgres casts text → column types). CSV dir = `NMW_CSV_DIR` (default temp). +- **CSV exports** — fallback when `NMW_SQL_DUMP` is unset; per-table CSVs in + `nma_csv_cache` / GCS `nma_csv/`, inserted row-by-row with type coercion. + +> Note: `NMWells.sql` as provided is **schema-only** (DDL, no `INSERT`s) — it +> seeded the models/migrations. A separate **data** dump (with `INSERT`s) is +> what `NMW_SQL_DUMP` should point at. + +Run via the standalone orchestrator `transfers/transfer_geothermal.py` +(`python -m transfers.transfer_geothermal`) — **separate** from the deprecated +`transfers/transfer.py` (NM_Aquifer driver), which must not gain new migrations. +The orchestrator runs the reference→lexicon load (`TRANSFER_GEOTHERMAL_REFERENCE`) +then the mirror load (`TRANSFER_NMW_MIRROR`); both default on. After the mirror +load it calls `refresh_materialized_views` to `REFRESH` the materialized OGC views +(currently `ogc_geothermal_wells_temperature_profile`; missing views are skipped). +Assumes the schema already exists (`alembic upgrade head`). + +## OGC views (pygeoapi) + +`alembic/versions/w9x0y1z2a3b4_add_geothermal_ogc_views.py` adds two point +layers over the `NMW_*` mirror (geometry from `NMW_WellLocations` Lat/Long_dd83): + +- `ogc_geothermal_wells_bht` — one feature per geothermal well with + bottom-hole-temperature data (`NMW_GtBhtData`); aggregate BHT stats. +- `ogc_geothermal_wells_temperature_profile` — **materialized** view, one feature + per geothermal well with a downhole temperature-vs-depth series + (`NMW_GtTempDepths`, ~370k source rows), series as an ordered JSON array. + Indexed (unique `well_data_id`, GiST `geom`); `REFRESH MATERIALIZED VIEW` after + a data reload. + +- `ogc_geothermal_wells_summary_heat_flow` — one feature per geothermal well with + summary heat-flow determinations (`NMW_GtSumHeatFlow`): aggregate heat flow, + thermal gradient, thermal conductivity, quality, plus a `measurements` JSON + series (one element per determination, ordered by depth). Linked directly via + `NMW_GtSumHeatFlow.RecrdSetID → NMW_WellRecords.RecrdSetID`. +- `ogc_geothermal_wells_interval_heat_flow` — one feature per geothermal well + with **per-interval** heat-flow values (`NMW_GtHeatFlow`): aggregate heat flow + (Q), gradient, conductivity (Kpr), diffusivity (Ka), plus a `measurements` JSON + series (one element per interval, ordered by depth). Linked via + `NMW_GtHeatFlow.IntrvlGUID → NMW_WsIntervals.IntrvlGUID → NMW_WellSamples → + NMW_WellRecords`. + +Well linkage: `gt_*.SamplSetID → NMW_WellSamples.SamplSetID → +NMW_WellRecords.RecrdSetID → NMW_WellLocations.WellDataID`. + +## Phase 2 transform map (summary) + +Key relationship re-routing: legacy `WellDataID` ties header/location/records; +`RecrdSetID` ties records→children. In Ocotillo, **wells → records** becomes +**wells (Thing) → field_event**, and `RecrdClass` → `field_activity.activity_type`. + +| Source | Ocotillo target | +|-------------------------------------|-----------------| +| `tbl_well_locations` Lat/Long_dd83 | `location.point` | +| `tbl_well_locations` State/County | `location.state` / `location.county` | +| `tbl_well_locations` PLSS/UTM/dd27 | **new** `NMW_Location` (township, range, section, unit_letter, utm_zone, basin, footages, dd27, source_datum, source_units) | +| `tbl_well_headers` CurWellNam/WellClass/TotalDepth/ComplDate | `thing.name` / `thing.type` / `thing.well_depth` / `thing.well_completion_date` | +| `tbl_well_headers` CurStatus | `status_history.status` | +| `tbl_well_headers` CurOperatr/CurOwner | `contact.name` (type = operator / owner) | +| `tbl_well_headers` API | `thing_id_link.alternate_id` | +| `tbl_well_headers` Fm_TD | `thing_geologic_formation_association` | +| `tbl_well_headers` WellType/WellOrient/CurWellNum/Comments | **new** `well_purpose.purpose` / **new** `well_detail` (well_orient, well_number, comments) | +| `tbl_well_records` RecrdSetID/ActionDate/Comments | `field_event` (id/event_date/notes) | +| `tbl_well_records` RecrdClass | `field_activity.activity_type` | +| `tbl_well_z_datum` Elev_GL/DF/KB/unspc | `measuring_point_history.measuring_point_height` | +| `tbl_well_z_datum` DepthDatum/DepthUnits/ElevSource | `measuring_point_history` description / units / source | +| `tbl_well_samples` SamplSetID/SampleDate/Notes/EnteredBy | `sample` (id/sample_date/notes/created_by_name) | +| `tbl_well_samples` From_Depth/To_Depth/SmpDpUnt | `observation` (depths / unit) | + +**New tables Phase 2 needs:** `NMW_Location`, `well_detail`, `well_purpose`. + +## Reference tables → lexicon + +The legacy `ref_*` lookup tables ("Add to lexicon" in workbook sheet 1) are +loaded into the lexicon by `transfers/reference_lexicon_transfer.py` +(`transfer_reference_tables`), registered as a **foundational** transfer in +`transfers/transfer.py` (runs before wells). Each `ref_*` table becomes a +`LexiconCategory` (name = table minus `ref_`) and its rows become +`LexiconTerm`s linked to that category. Idempotent (`ON CONFLICT DO NOTHING`), +matching `core.initializers.init_lexicon`. + +Term/definition columns are **auto-detected** from each CSV header (the +workbook has no column lists for `ref_*`); override `term_col`/`definition_col` +on a `RefTableSpec` if detection is wrong. The Subsurface Library `LU_*` +lookups are also "Add to lexicon" — add them to `REFERENCE_TABLE_SPECS` once +their CSVs are available. + +## Not yet mapped (no field list / DDL in workbook) + +Remaining "Migrate First" tables not yet mirrored: + +- **Publications:** `tbl_sources` (DDL available in dump; not yet requested) +- **Subsurface Library:** `dst_scan`, `log_scanned`, `Well_Header`, + `well_operators` + +`tbl_well_bores` (Geothermal area) is "Review", not "Migrate First". + +"Don't migrate" (per workbook): OCD injection/wells imports, `*_ZOLD`, +`*_DataAsOf-*` snapshots, `geometry_columns`, `spatial_ref_sys`, `sysdiagrams`. From 83fe74673ea46ec392124dbd075043f759d70fb0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 19:25:37 -0600 Subject: [PATCH 33/41] T12: add NM_Wells mirror/loader/OGC tests; fix CAST-unwrap bug (B1) tests/test_nmw_mirror.py (19 tests) covers SPEC invariants V1 (18 mirror tables + PK), V2 (FK parent loads before child in NMW_MIRROR_SPECS), V3 (8 OGC views built), V5 (temperature-profile view materialized), V6 (geothermal pygeoapi collections back existing relations), V10 (DB-level FK constraints), and the SQL-dump value parser. Fixes B1/V11: _CAST_RE in transfers/nmw_sql_dump.py only matched AS-types without parentheses, so CAST(x AS nvarchar(10)) / CAST(n AS Decimal(18,2)) left the value as a literal "CAST(...)" string. Widened to allow one paren level in the type name. Co-Authored-By: Claude Opus 4.8 --- SPEC.md | 70 +++++++++++ tests/test_nmw_mirror.py | 243 ++++++++++++++++++++++++++++++++++++++ transfers/nmw_sql_dump.py | 4 +- 3 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 SPEC.md create mode 100644 tests/test_nmw_mirror.py diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 000000000..18a722ed1 --- /dev/null +++ b/SPEC.md @@ -0,0 +1,70 @@ +# SPEC — BDMS-826 NM_Wells migrations core + +Distilled from branch `feature/BDMS-826-NMW-migrations-core` (PR #738), POC of #686. +Status: Phase-1 impl landed in branch, NOT reviewed/tested. Spec captures built state + remaining work. + +## §G — goal + +1:1 staging mirror of legacy NM_Wells (SQL Server) into Postgres `NMW_*` tables. +Plus geothermal OGC API layers over mirror data. Phase-1 only: faithful copy, no transform to Ocotillo model. + +## §C — constraints + +- C1. Mirror = column-for-column copy. Original col names, types, PKs preserved. `SSMA_TimeStamp` dropped. +- C2. Phase split: P1 = land `NMW_*` mirror + OGC views (this branch). P2 = transform mirror → Ocotillo model (Location→Thing→Event→Sample→Observation). P2 NOT built; per-col targets + lexicon maps flagged inline. +- C3. Follow `db/nma_legacy.py` (NM_Aquifer) mirror convention. +- C4. Standalone orchestrator. Do NOT extend deprecated `transfers/transfer.py` (NM_Aquifer driver). +- C5. Tables load parent→child (FK order). Mirror load truncate+COPY (dump) or INSERT ON CONFLICT DO NOTHING (CSV). No upsert. +- C6. Coords/geom stay WGS84 4326 per project std. +- C7. Migrations idempotent, reversible (alembic up/down). + +## §I — interfaces + +- I.cli — `python -m transfers.transfer_geothermal` — runs ref→lexicon then NMW mirror load then refresh matviews. +- I.env — `NMW_SQL_DUMP` (dump path; else CSV), `NMW_CSV_DIR`, `TRANSFER_LIMIT`, `TRANSFER_GEOTHERMAL_REFERENCE` (def 1), `TRANSFER_NMW_MIRROR` (def 1). Export: `NMW_HOST/USER/PASSWORD/PORT/DATABASE`. +- I.export — `transfers/export_nmw_csvs.py` — pymssql dump SQL Server → `transfers/data/nma_csv_cache/
.csv`. +- I.db — 18 `NMW_*` mirror tables (`db/nmw_legacy.py`), PK verified vs dump DDL. +- I.migrations — `c0d1e2f3a4b5` (tables+FK), `d1e2f3a4b5c6` (per-well views), `e2f3a4b5c6d7` (measurement views). Chain down_rev: t6u7v8w9x0y1 → c0 → d1 → e2. +- I.ogc — 6 new collections in `core/pygeoapi-config.yml`: geothermal_wells_bht, geothermal_wells_temperature_profile (MATVIEW), bht_measurements, temp_depth_measurements, heat_flow, dst. +- I.views — DB: ogc_geothermal_wells_bht, ogc_geothermal_wells_temperature_profile (MAT), ogc_geothermal_wells_summary_heat_flow, ogc_geothermal_wells_interval_heat_flow, ogc_bht_measurements, ogc_temp_depth_measurements, ogc_heat_flow, ogc_dst. +- I.lexicon — `reference_lexicon_transfer.py` maps 46 `ref_*` → `nmw_ref_*` LexiconCategory + terms. + +## §V — invariants + +- V1. `NMW_*` cols match legacy NM_Wells DDL exactly (name+type+PK). No renames except dropped `SSMA_TimeStamp`. +- V2. Mirror load respects parent→child order in `NMW_MIRROR_SPECS`; child never loads before parent. +- V3. `alembic upgrade head` then `downgrade` cleanly creates+drops all 18 tables + 8 views, no orphans. +- V4. Re-running mirror load is non-destructive at row level (truncate+COPY full reload OR ON CONFLICT skip) — no dup rows, no partial-state corruption. +- V5. After mirror load, `ogc_geothermal_wells_temperature_profile` matview refreshed; stale matview never served. +- V6. Each of 6 OGC collections resolves to existing backing view; pygeoapi config view name == migration view name. +- V7. `core/pygeoapi.py` unchanged from staging (reviewer note: reverted to original). +- V8. `NMW_WellRecords.SourceID` joined as TEXT (free-text citation), not numeric FK. +- V9. P2 lexicon mapping complete: every coded NMW col either in `LEXICON_REF_BY_COLUMN` (28, has ref_*) or `LEXICON_CANDIDATES_NO_REF` (11, needs new enum). +- V10. ORM `NMW_*` models declare index only (`index=True`, no ORM `ForeignKey`); FK enforcement lives in migration `c0d1e2f3a4b5` (`op.create_foreign_key`). Keep both in sync. +- V11. SQL-dump parser unwraps `CAST(expr AS )` for parameterised types too (`Decimal(18,2)`, `nvarchar(10)`); never store the literal `CAST(...)` string in a mirror column. + +## §T — tasks + +id|status|task|cites +T1|x|18 NMW_* mirror tables in db/nmw_legacy.py|V1,I.db +T2|x|migration c0d1e2f3a4b5 tables+FK|V3,I.migrations +T3|x|migration d1e2f3a4b5c6 per-well OGC views|I.views +T4|x|migration e2f3a4b5c6d7 measurement OGC views|I.views +T5|x|nmw_sql_dump.py SSMS dump parser|I.cli +T6|x|nmw_mirror_transfer.py loader (dump+CSV)|V2,V4,I.cli +T7|x|reference_lexicon_transfer.py ref_*→lexicon|I.lexicon +T8|x|export_nmw_csvs.py pymssql export|I.export +T9|x|transfer_geothermal.py orchestrator|I.cli +T10|x|6 OGC collections in pygeoapi-config.yml|V6,I.ogc +T11|x|FK enforced via migration op.create_foreign_key; model index-only (resolved)|V2,V10 +T12|x|add NMW_* mirror/loader/migration/OGC tests (tests/test_nmw_mirror.py, 19 tests); found+fixed CAST-unwrap bug B1|V1,V2,V3,V5,V6,V10,V11 +T13|.|verify alembic down path drops all views+tables (V3) on real db|V3 +T14|.|run end-to-end load vs real dump, capture row counts per table|V2,V4 +T15|.|finish PR #738 body (truncated at "- I ") + reviewer notes|- +T16|.|P2 (later): transform NMW_* → Ocotillo model; build new enums for 11 LEXICON_CANDIDATES_NO_REF|C2,V9 +T17|x|landed docs/nm_wells-migration.md (commit ccf566d9; force-add, docs/ gitignored). Referenced 4x: nmw_legacy.py:75,81,759; nmw_mirror_transfer.py:19|- + +## §B — bugs + +id|date|cause|fix +B1|2026-06-23|_CAST_RE in transfers/nmw_sql_dump.py matched AS-type without parens only; parenthesised types (nvarchar(10), Decimal(18,2)) left value as literal "CAST(...)" string|V11; widened regex to allow one paren level diff --git a/tests/test_nmw_mirror.py b/tests/test_nmw_mirror.py new file mode 100644 index 000000000..2756172b5 --- /dev/null +++ b/tests/test_nmw_mirror.py @@ -0,0 +1,243 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Structural + unit tests for the NM_Wells Phase-1 staging mirror. + +Covers SPEC §V invariants for the mirror schema, migrations and OGC views: + + V1 - all 18 NMW_* mirror tables exist with a primary key + V2 - mirror load order respects parent->child (FK parent precedes child) + V3 - migrations build all 8 OGC views + V5 - the temperature-profile OGC view is MATERIALIZED + V6 - each geothermal pygeoapi collection maps to an existing DB relation + V10 - FK enforcement lives in the migration (DB-level FK constraints exist) + +Full data round-trip against a real SQL dump is out of scope here (SPEC §T.T14); +the dump parser is unit-tested directly instead. +""" + +import os + +import pytest +import yaml +from sqlalchemy import inspect as sa_inspect, text + +from db.engine import engine, session_ctx +from transfers.nmw_mirror_transfer import NMW_MIRROR_SPECS +from transfers.nmw_sql_dump import _parse_value, iter_table_rows + +ROOT = os.path.dirname(os.path.dirname(__file__)) + +# DB relations created by the OGC-view migrations (d1e2f3a4b5c6, e2f3a4b5c6d7). +OGC_VIEWS = [ + "ogc_geothermal_wells_bht", + "ogc_geothermal_wells_temperature_profile", # MATERIALIZED + "ogc_geothermal_wells_summary_heat_flow", + "ogc_geothermal_wells_interval_heat_flow", + "ogc_bht_measurements", + "ogc_temp_depth_measurements", + "ogc_heat_flow", + "ogc_dst", +] +MATERIALIZED_VIEW = "ogc_geothermal_wells_temperature_profile" + +# pygeoapi collections added by this PR and the DB relation each is backed by. +GEOTHERMAL_COLLECTIONS = { + "geothermal_wells_bht": "ogc_geothermal_wells_bht", + "geothermal_wells_temperature_profile": "ogc_geothermal_wells_temperature_profile", + "bht_measurements": "ogc_bht_measurements", + "temp_depth_measurements": "ogc_temp_depth_measurements", + "heat_flow": "ogc_heat_flow", + "dst": "ogc_dst", +} + + +def _mirror_tablenames() -> list[str]: + return [spec.model.__tablename__ for spec in NMW_MIRROR_SPECS] + + +# --------------------------------------------------------------------------- V1 +def test_all_mirror_tables_present_with_pk(): + """All 18 NMW_* mirror tables exist in the schema, each with a PK (V1).""" + names = _mirror_tablenames() + assert len(names) == 18, f"expected 18 mirror specs, got {len(names)}" + + insp = sa_inspect(engine) + existing = set(insp.get_table_names()) + for table in names: + assert table in existing, f"mirror table {table} missing from schema" + pk = insp.get_pk_constraint(table)["constrained_columns"] + assert pk, f"mirror table {table} has no primary key" + + +def test_well_headers_pk_is_well_data_id(): + """Spot-check that original SQL Server column names are preserved (V1).""" + insp = sa_inspect(engine) + pk = insp.get_pk_constraint("NMW_WellHeaders")["constrained_columns"] + assert pk == ["WellDataID"] + + +# ----------------------------------------------------------------------- V2/V10 +def test_mirror_tables_have_fk_constraints(): + """The migration creates DB-level FK constraints (V10) - at least one + child table must carry a foreign key.""" + insp = sa_inspect(engine) + total_fks = sum(len(insp.get_foreign_keys(t)) for t in _mirror_tablenames()) + assert total_fks > 0, "no FK constraints found on NMW_* mirror tables" + + +def test_fk_parent_loads_before_child(): + """Every FK parent table is loaded before its child in NMW_MIRROR_SPECS so + the parent row exists when the child is inserted (V2).""" + order = {name: i for i, name in enumerate(_mirror_tablenames())} + insp = sa_inspect(engine) + checked = 0 + for child in order: + for fk in insp.get_foreign_keys(child): + parent = fk["referred_table"] + if parent not in order or parent == child: + continue # self-ref or FK to a non-mirror table + checked += 1 + assert order[parent] <= order[child], ( + f"{parent} (parent) must load before {child} (child) " + f"in NMW_MIRROR_SPECS" + ) + assert checked > 0, "expected at least one intra-mirror FK to validate" + + +# --------------------------------------------------------------------------- V3 +def test_ogc_views_exist(): + """All 8 OGC views built by the migrations exist as relations (V3).""" + with session_ctx() as session: + rows = session.execute( + text( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'public' " + "UNION SELECT matviewname FROM pg_matviews WHERE schemaname = 'public'" + ) + ).all() + relations = {r[0] for r in rows} + for view in OGC_VIEWS: + assert view in relations, f"OGC view {view} missing" + + +# --------------------------------------------------------------------------- V5 +def test_temperature_profile_is_materialized(): + """The temperature-profile view is MATERIALIZED so it can be refreshed + after a mirror load (V5).""" + with session_ctx() as session: + names = { + r[0] + for r in session.execute( + text("SELECT matviewname FROM pg_matviews WHERE schemaname = 'public'") + ).all() + } + assert MATERIALIZED_VIEW in names, f"{MATERIALIZED_VIEW} is not a materialized view" + + +# --------------------------------------------------------------------------- V6 +class _Default(dict): + """format_map() helper: unknown placeholders render empty.""" + + def __missing__(self, key): # noqa: D401 + return "" + + +def _load_pygeoapi_config() -> dict: + """pygeoapi-config.yml is a ``{placeholder}`` template (see core/pygeoapi.py + _write_config); substitute dummy values before parsing as YAML.""" + raw = open(os.path.join(ROOT, "core", "pygeoapi-config.yml")).read() + rendered = raw.format_map( + _Default( + server_url="http://test", + postgres_host="h", + postgres_port="5432", + postgres_db="d", + postgres_user="u", + postgres_password_env="p", + thing_collections_block="", + ) + ) + return yaml.safe_load(rendered) + + +def test_geothermal_collections_back_existing_relations(): + """Each new geothermal pygeoapi collection points at a DB relation that + actually exists (V6).""" + cfg = _load_pygeoapi_config() + resources = cfg["resources"] + + with session_ctx() as session: + rows = session.execute( + text( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'public' " + "UNION SELECT matviewname FROM pg_matviews WHERE schemaname = 'public'" + ) + ).all() + relations = {r[0] for r in rows} + + for coll, expected_table in GEOTHERMAL_COLLECTIONS.items(): + assert coll in resources, f"collection {coll} missing from pygeoapi config" + tables = { + p.get("table") + for p in resources[coll].get("providers", []) + if p.get("table") + } + assert ( + expected_table in tables + ), f"collection {coll} should be backed by {expected_table}, got {tables}" + assert ( + expected_table in relations + ), f"backing relation {expected_table} for {coll} does not exist in DB" + + +# ----------------------------------------------------------------- dump parser +@pytest.mark.parametrize( + "raw,expected", + [ + ("NULL", None), + ("null", None), + ("123", 123), + ("-5", -5), + ("-1.5", -1.5), + ("'abc'", "abc"), + ("N'abc'", "abc"), + ("'O''Brien'", "O'Brien"), # doubled '' unescaped + ("CAST(42 AS int)", 42), + ("CAST(N'x' AS nvarchar(10))", "x"), + ("0xDEADBEEF", None), # binary / rowversion not mirrored + ], +) +def test_parse_value_coercion(raw, expected): + assert _parse_value(raw) == expected + + +def test_iter_table_rows_parses_inserts(tmp_path): + """iter_table_rows decodes column/value pairs from SSMS INSERT statements.""" + dump = tmp_path / "dump.sql" + dump.write_text( + "INSERT [dbo].[tbl_demo] ([OBJECTID], [Name], [Note]) " + "VALUES (1, N'alpha', NULL), (2, 'beta', CAST(N'c' AS nvarchar(1)));\n", + encoding="utf-8", + ) + rows = list(iter_table_rows(str(dump), "tbl_demo")) + assert rows == [ + {"OBJECTID": 1, "Name": "alpha", "Note": None}, + {"OBJECTID": 2, "Name": "beta", "Note": "c"}, + ] + + +# ============= EOF ============================================= diff --git a/transfers/nmw_sql_dump.py b/transfers/nmw_sql_dump.py index 29f72e735..f7010b849 100644 --- a/transfers/nmw_sql_dump.py +++ b/transfers/nmw_sql_dump.py @@ -122,7 +122,9 @@ def _iter_value_groups(s: str) -> Iterator[str]: i += 1 -_CAST_RE = re.compile(r"(?is)^CAST\s*\((.*)\s+AS\s+[^)]+\)$") +# The AS-target type may itself be parameterised, e.g. CAST(1.50 AS Decimal(18, 2)) +# or CAST(N'x' AS nvarchar(10)); allow one level of parens in the type name. +_CAST_RE = re.compile(r"(?is)^CAST\s*\((.*)\s+AS\s+[^()]+(?:\([^)]*\))?\s*\)$") def _parse_value(tok: str): From 9fba496cef143fa2f9b129a73ef218629cab997b Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 19:40:15 -0600 Subject: [PATCH 34/41] fix(alembic): merge nmw mirror chain into staging head Merging staging left two alembic heads: e2f3a4b5c6d7 (NMW mirror + OGC views) and x2y3z4a5b6c7 (staging pg_cron matview refresh). Add a merge revision so `alembic upgrade head` resolves to a single head. Fixes the bdd-tests CI failure. Co-Authored-By: Claude Opus 4.8 --- ...erge_nmw_mirror_chain_into_staging_head.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py diff --git a/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py b/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py new file mode 100644 index 000000000..57bbd0909 --- /dev/null +++ b/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py @@ -0,0 +1,26 @@ +"""merge nmw mirror chain into staging head + +Revision ID: 03ef547ed7be +Revises: e2f3a4b5c6d7, x2y3z4a5b6c7 +Create Date: 2026-06-23 19:39:25.256933 + +""" + +from typing import Sequence, Union + + +# revision identifiers, used by Alembic. +revision: str = "03ef547ed7be" +down_revision: Union[str, Sequence[str], None] = ("e2f3a4b5c6d7", "x2y3z4a5b6c7") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + pass + + +def downgrade() -> None: + """Downgrade schema.""" + pass From e44dba66cc05f4d08be0e51100e4ebfbb8a31c4b Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:40:37 +0000 Subject: [PATCH 35/41] Formatting changes --- .../03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py b/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py index 57bbd0909..42da6e150 100644 --- a/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py +++ b/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py @@ -8,7 +8,6 @@ from typing import Sequence, Union - # revision identifiers, used by Alembic. revision: str = "03ef547ed7be" down_revision: Union[str, Sequence[str], None] = ("e2f3a4b5c6d7", "x2y3z4a5b6c7") From a26b7f392ae2dcf015f910d9e7723d128d03c4df Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 19:41:23 -0600 Subject: [PATCH 36/41] refactor(alembic): linearize nmw mirror chain onto staging head Replace the merge revision (03ef547ed7be) with a linear history: repoint c0d1e2f3a4b5.down_revision from the old shared base t6u7v8w9x0y1 to staging's head x2y3z4a5b6c7. Single head e2f3a4b5c6d7; no merge point. Co-Authored-By: Claude Opus 4.8 --- SPEC.md | 13 +++++++++- ...erge_nmw_mirror_chain_into_staging_head.py | 25 ------------------- .../c0d1e2f3a4b5_nmw_mirror_tables.py | 2 +- 3 files changed, 13 insertions(+), 27 deletions(-) delete mode 100644 alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py diff --git a/SPEC.md b/SPEC.md index 18a722ed1..35919cbeb 100644 --- a/SPEC.md +++ b/SPEC.md @@ -24,10 +24,11 @@ Plus geothermal OGC API layers over mirror data. Phase-1 only: faithful copy, no - I.env — `NMW_SQL_DUMP` (dump path; else CSV), `NMW_CSV_DIR`, `TRANSFER_LIMIT`, `TRANSFER_GEOTHERMAL_REFERENCE` (def 1), `TRANSFER_NMW_MIRROR` (def 1). Export: `NMW_HOST/USER/PASSWORD/PORT/DATABASE`. - I.export — `transfers/export_nmw_csvs.py` — pymssql dump SQL Server → `transfers/data/nma_csv_cache/
.csv`. - I.db — 18 `NMW_*` mirror tables (`db/nmw_legacy.py`), PK verified vs dump DDL. -- I.migrations — `c0d1e2f3a4b5` (tables+FK), `d1e2f3a4b5c6` (per-well views), `e2f3a4b5c6d7` (measurement views). Chain down_rev: t6u7v8w9x0y1 → c0 → d1 → e2. +- I.migrations — `c0d1e2f3a4b5` (tables+FK), `d1e2f3a4b5c6` (per-well views), `e2f3a4b5c6d7` (measurement views). Linear chain (rebased onto staging head): x2y3z4a5b6c7 → c0 → d1 → e2. - I.ogc — 6 new collections in `core/pygeoapi-config.yml`: geothermal_wells_bht, geothermal_wells_temperature_profile (MATVIEW), bht_measurements, temp_depth_measurements, heat_flow, dst. - I.views — DB: ogc_geothermal_wells_bht, ogc_geothermal_wells_temperature_profile (MAT), ogc_geothermal_wells_summary_heat_flow, ogc_geothermal_wells_interval_heat_flow, ogc_bht_measurements, ogc_temp_depth_measurements, ogc_heat_flow, ogc_dst. - I.lexicon — `reference_lexicon_transfer.py` maps 46 `ref_*` → `nmw_ref_*` LexiconCategory + terms. +- I.jira — BDMS-826 "Geothermal Migration Planning" (Story, In Progress) under epic BDMS-843 "Geothermal Migration". 6 linked tasks tracked in §T (T18-T23). ## §V — invariants @@ -64,6 +65,16 @@ T15|.|finish PR #738 body (truncated at "- I ") + reviewer notes|- T16|.|P2 (later): transform NMW_* → Ocotillo model; build new enums for 11 LEXICON_CANDIDATES_NO_REF|C2,V9 T17|x|landed docs/nm_wells-migration.md (commit ccf566d9; force-add, docs/ gitignored). Referenced 4x: nmw_legacy.py:75,81,759; nmw_mirror_transfer.py:19|- +### BDMS-826 linked tasks (six; status mirrors Jira) + +id|status|task|cites +T18|x|BDMS-827 read/review Geothermal Data Discovery Report (Jira Done)|I.jira +T19|x|BDMS-846 technical review complete (Jira Done)|I.jira +T20|x|BDMS-845 Geothermal Report finalized (Jira Done)|I.jira +T21|x|BDMS-847 update data-migration tracking mechanism (Jira Done)|I.jira +T22|x|BDMS-907 stakeholder engagement follow-up (Jira Done; blocks BDMS-826)|I.jira +T23|~|BDMS-848 Geothermal Migration Technical Implementation Plan (Jira In Progress) — umbrella for code tasks T1-T16|I.jira,T1,T16 + ## §B — bugs id|date|cause|fix diff --git a/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py b/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py deleted file mode 100644 index 42da6e150..000000000 --- a/alembic/versions/03ef547ed7be_merge_nmw_mirror_chain_into_staging_head.py +++ /dev/null @@ -1,25 +0,0 @@ -"""merge nmw mirror chain into staging head - -Revision ID: 03ef547ed7be -Revises: e2f3a4b5c6d7, x2y3z4a5b6c7 -Create Date: 2026-06-23 19:39:25.256933 - -""" - -from typing import Sequence, Union - -# revision identifiers, used by Alembic. -revision: str = "03ef547ed7be" -down_revision: Union[str, Sequence[str], None] = ("e2f3a4b5c6d7", "x2y3z4a5b6c7") -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - pass - - -def downgrade() -> None: - """Downgrade schema.""" - pass diff --git a/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py b/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py index 42a8cc8ba..f59a760ca 100644 --- a/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py +++ b/alembic/versions/c0d1e2f3a4b5_nmw_mirror_tables.py @@ -44,7 +44,7 @@ from sqlalchemy.dialects import postgresql revision: str = "c0d1e2f3a4b5" -down_revision: Union[str, Sequence[str], None] = "t6u7v8w9x0y1" +down_revision: Union[str, Sequence[str], None] = "x2y3z4a5b6c7" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 25db7345ff0ee9fd74816569f61164bbfb418932 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 19:46:21 -0600 Subject: [PATCH 37/41] docs(spec): verify all NM_Wells Migrate-First tables mirrored (V12) Cross-checked NMW_MIRROR_SPECS against the planning workbook: all 18 NM_Wells "Migrate First" tables are handled. Flagged 4 Subsurface Library "Migrate First" tables (dst_scan, log_scanned, Well_Header, well_operators) as out-of-scope (separate source DB) under new task T24. Co-Authored-By: Claude Opus 4.8 --- SPEC.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SPEC.md b/SPEC.md index 35919cbeb..048c4bc1f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -43,6 +43,7 @@ Plus geothermal OGC API layers over mirror data. Phase-1 only: faithful copy, no - V9. P2 lexicon mapping complete: every coded NMW col either in `LEXICON_REF_BY_COLUMN` (28, has ref_*) or `LEXICON_CANDIDATES_NO_REF` (11, needs new enum). - V10. ORM `NMW_*` models declare index only (`index=True`, no ORM `ForeignKey`); FK enforcement lives in migration `c0d1e2f3a4b5` (`op.create_foreign_key`). Keep both in sync. - V11. SQL-dump parser unwraps `CAST(expr AS )` for parameterised types too (`Decimal(18,2)`, `nvarchar(10)`); never store the literal `CAST(...)` string in a mirror column. +- V12. Every "Migrate First" table in the NM_Wells inventory (planning workbook) has a `NMW_*` mirror in `NMW_MIRROR_SPECS`. Verified 18/18 (2026-06-23). Subsurface Library is a separate source DB — NOT covered by this invariant (see T24). ## §T — tasks @@ -63,6 +64,7 @@ T13|.|verify alembic down path drops all views+tables (V3) on real db|V3 T14|.|run end-to-end load vs real dump, capture row counts per table|V2,V4 T15|.|finish PR #738 body (truncated at "- I ") + reviewer notes|- T16|.|P2 (later): transform NMW_* → Ocotillo model; build new enums for 11 LEXICON_CANDIDATES_NO_REF|C2,V9 +T24|.|Subsurface Library "Migrate First" tables NOT mirrored (separate source DB, out of NM_Wells scope): dst_scan, log_scanned, Well_Header, well_operators. Workbook lacks field map/DDL — needs own ticket|V12 T17|x|landed docs/nm_wells-migration.md (commit ccf566d9; force-add, docs/ gitignored). Referenced 4x: nmw_legacy.py:75,81,759; nmw_mirror_transfer.py:19|- ### BDMS-826 linked tasks (six; status mirrors Jira) From f4c805f8a1c18aec694af3e8c9b07f46ffbf4227 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Jun 2026 19:51:53 -0600 Subject: [PATCH 38/41] fix(nmw): FK-safe truncate + dedup locations in per-well OGC views Addresses PR #740 Codex review: - P1 (B2/V13): dump-load reload used a bare TRUNCATE, rejected by the FK refs to NMW_WellHeaders. Use TRUNCATE ... CASCADE (parents load before children per V2, so cascaded children are reloaded after). - P2 (B3/V14): per-well geothermal views joined NMW_WellLocations directly; multiple OBJECTID rows per WellDataID inflated counts / emitted >1 feature per well. Dedup via DISTINCT ON loc CTE in all 4 views (bht, summary and interval heat-flow; profile already did this). Adds two regression tests. 21 passing. Co-Authored-By: Claude Opus 4.8 --- SPEC.md | 4 + ...4b5c6_nmw_per_well_geothermal_ogc_views.py | 68 +++++++++++---- tests/test_nmw_mirror.py | 86 +++++++++++++++++++ transfers/nmw_mirror_transfer.py | 6 +- 4 files changed, 146 insertions(+), 18 deletions(-) diff --git a/SPEC.md b/SPEC.md index 048c4bc1f..3731fa02e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -44,6 +44,8 @@ Plus geothermal OGC API layers over mirror data. Phase-1 only: faithful copy, no - V10. ORM `NMW_*` models declare index only (`index=True`, no ORM `ForeignKey`); FK enforcement lives in migration `c0d1e2f3a4b5` (`op.create_foreign_key`). Keep both in sync. - V11. SQL-dump parser unwraps `CAST(expr AS )` for parameterised types too (`Decimal(18,2)`, `nvarchar(10)`); never store the literal `CAST(...)` string in a mirror column. - V12. Every "Migrate First" table in the NM_Wells inventory (planning workbook) has a `NMW_*` mirror in `NMW_MIRROR_SPECS`. Verified 18/18 (2026-06-23). Subsurface Library is a separate source DB — NOT covered by this invariant (see T24). +- V13. Mirror dump-reload truncates with `CASCADE` (mirror tables carry FK constraints; bare TRUNCATE of a referenced parent is rejected). Safe because parents load before children (V2). +- V14. Per-well OGC views dedup `NMW_WellLocations` by `WellDataID` (`DISTINCT ON ... ORDER BY "WellDataID","OBJECTID"`); one feature per well, counts not multiplied by duplicate location rows. Applies to all 4 per-well geothermal views. ## §T — tasks @@ -81,3 +83,5 @@ T23|~|BDMS-848 Geothermal Migration Technical Implementation Plan (Jira In Progr id|date|cause|fix B1|2026-06-23|_CAST_RE in transfers/nmw_sql_dump.py matched AS-type without parens only; parenthesised types (nvarchar(10), Decimal(18,2)) left value as literal "CAST(...)" string|V11; widened regex to allow one paren level +B2|2026-06-23|dump-load reload used bare TRUNCATE; FK from NMW_WellLocations/NMW_WellRecords to NMW_WellHeaders makes Postgres reject it, aborting load before COPY (PR#740 P1)|V13; TRUNCATE ... CASCADE +B3|2026-06-23|per-well geothermal OGC views (bht, summary_heat_flow, interval_heat_flow) joined NMW_WellLocations directly; multiple OBJECTID rows per WellDataID multiplied counts / emitted >1 feature per well (PR#740 P2)|V14; DISTINCT ON loc CTE in all 4 views diff --git a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py index 0931b3596..e2cf47df4 100644 --- a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py +++ b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py @@ -48,8 +48,18 @@ def upgrade() -> None: # ogc_geothermal_wells_bht op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) - op.execute(text(f""" + op.execute( + text( + f""" CREATE VIEW "{_BHT_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) SELECT row_number() OVER () AS id, r."WellDataID"::text AS well_data_id, @@ -68,10 +78,8 @@ def upgrade() -> None: JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + JOIN loc ON loc."WellDataID" = r."WellDataID" LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL GROUP BY r."WellDataID", loc."Lat_dd83", @@ -79,11 +87,15 @@ def upgrade() -> None: hdr."CurWellNam", hdr."API", hdr."TotalDepth" - """)) + """ + ) + ) # ogc_geothermal_wells_temperature_profile (materialized) op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) - op.execute(text(f""" + op.execute( + text( + f""" CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -124,7 +136,9 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """)) + """ + ) + ) op.execute( text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') ) @@ -136,8 +150,18 @@ def upgrade() -> None: # ogc_geothermal_wells_summary_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_SUM_HF_VIEW}"')) - op.execute(text(f""" + op.execute( + text( + f""" CREATE VIEW "{_SUM_HF_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) SELECT row_number() OVER () AS id, r."WellDataID"::text AS well_data_id, @@ -173,22 +197,32 @@ def upgrade() -> None: ) AS geom FROM "NMW_GtSumHeatFlow" AS shf JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = shf."RecrdSetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + JOIN loc ON loc."WellDataID" = r."WellDataID" LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL GROUP BY r."WellDataID", loc."Lat_dd83", loc."Long_dd83", hdr."CurWellNam", hdr."API" - """)) + """ + ) + ) # ogc_geothermal_wells_interval_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_INT_HF_VIEW}"')) - op.execute(text(f""" + op.execute( + text( + f""" CREATE VIEW "{_INT_HF_VIEW}" AS + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) SELECT row_number() OVER () AS id, r."WellDataID"::text AS well_data_id, @@ -224,17 +258,17 @@ def upgrade() -> None: JOIN "NMW_WsIntervals" AS i ON i."IntrvlGUID" = hf."IntrvlGUID" JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = i."SamplSetID" JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" - JOIN "NMW_WellLocations" AS loc ON loc."WellDataID" = r."WellDataID" + JOIN loc ON loc."WellDataID" = r."WellDataID" LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" - WHERE loc."Lat_dd83" IS NOT NULL - AND loc."Long_dd83" IS NOT NULL GROUP BY r."WellDataID", loc."Lat_dd83", loc."Long_dd83", hdr."CurWellNam", hdr."API" - """)) + """ + ) + ) def downgrade() -> None: diff --git a/tests/test_nmw_mirror.py b/tests/test_nmw_mirror.py index 2756172b5..a6a785867 100644 --- a/tests/test_nmw_mirror.py +++ b/tests/test_nmw_mirror.py @@ -240,4 +240,90 @@ def test_iter_table_rows_parses_inserts(tmp_path): ] +# ------------------------------------------------------ regression: FK truncate +def test_truncate_referenced_parent_needs_cascade(): + """A bare TRUNCATE of an FK-referenced mirror parent is rejected; the loader + must use CASCADE (V13 / B2). Guards the dump-load reload path.""" + import sqlalchemy.exc + + # Bare truncate of the FK-referenced parent should error... + with session_ctx() as session: + with pytest.raises(sqlalchemy.exc.DBAPIError): + session.execute(text('TRUNCATE TABLE "NMW_WellHeaders"')) + # ...while CASCADE (what the loader does) succeeds. + with session_ctx() as session: + session.execute(text('TRUNCATE TABLE "NMW_WellHeaders" CASCADE')) + session.commit() + + +# --------------------------------------------- regression: one feature per well +def test_bht_view_one_feature_per_well_with_duplicate_locations(): + """Two location rows for one WellDataID must not multiply BHT features or + inflate bht_count; the view dedups locations via DISTINCT ON (V14 / B3).""" + wid = "11111111-1111-1111-1111-111111111111" + rsid = "22222222-2222-2222-2222-222222222222" + ssid = "33333333-3333-3333-3333-333333333333" + bht = "44444444-4444-4444-4444-444444444444" + with session_ctx() as session: + session.execute( + text('INSERT INTO "NMW_WellHeaders" ("WellDataID") VALUES (:w)'), + {"w": wid}, + ) + # TWO location rows, same WellDataID, valid coords (the bug trigger). + session.execute( + text( + 'INSERT INTO "NMW_WellLocations" ' + '("OBJECTID","WellDataID","Lat_dd83","Long_dd83") VALUES ' + "(901,:w,33.0,-107.0),(902,:w,33.0,-107.0)" + ), + {"w": wid}, + ) + session.execute( + text( + 'INSERT INTO "NMW_WellRecords" ("RecrdSetID","WellDataID") ' + "VALUES (:r,:w)" + ), + {"r": rsid, "w": wid}, + ) + session.execute( + text( + 'INSERT INTO "NMW_WellSamples" ("SamplSetID","RecrdsetID") ' + "VALUES (:s,:r)" + ), + {"s": ssid, "r": rsid}, + ) + session.execute( + text( + 'INSERT INTO "NMW_GtBhtHeaders" ("BHTGUID","SamplSetID") ' + "VALUES (:b,:s)" + ), + {"b": bht, "s": ssid}, + ) + session.execute( + text( + 'INSERT INTO "NMW_GtBhtData" ("OBJECTID","BHTGUID","BHT","Depth") ' + "VALUES (911,:b,150.0,1000.0)" + ), + {"b": bht}, + ) + session.commit() + try: + with session_ctx() as session: + rows = session.execute( + text( + 'SELECT bht_count FROM "ogc_geothermal_wells_bht" ' + "WHERE well_data_id = :w" + ), + {"w": wid}, + ).all() + assert len(rows) == 1, f"expected one feature per well, got {len(rows)}" + assert ( + rows[0][0] == 1 + ), f"bht_count inflated by duplicate locations: {rows[0][0]}" + finally: + with session_ctx() as session: + session.execute(text('TRUNCATE TABLE "NMW_WellHeaders" CASCADE')) + session.commit() + + # ============= EOF ============================================= diff --git a/transfers/nmw_mirror_transfer.py b/transfers/nmw_mirror_transfer.py index 28f78945a..d59ef4eaf 100644 --- a/transfers/nmw_mirror_transfer.py +++ b/transfers/nmw_mirror_transfer.py @@ -237,7 +237,11 @@ def _copy_load_table( return {"table": name, "skipped": True, "reason": "no rows", "source": "sql"} # Staging reload: truncate then COPY (no upsert; tables are a 1:1 snapshot). - session.execute(text(f'TRUNCATE TABLE "{table.name}"')) + # CASCADE because mirror tables carry FK constraints (e.g. NMW_WellLocations + # / NMW_WellRecords -> NMW_WellHeaders); a bare TRUNCATE of a referenced + # parent is rejected. Specs load parents before children (see V2), so a + # cascaded truncate only clears child tables that are reloaded afterwards. + session.execute(text(f'TRUNCATE TABLE "{table.name}" CASCADE')) _copy_csv_into_table(session, table.name, header, out_csv) session.commit() logger.info("COPY %s -> %s: %d rows (%s)", name, table.name, n, out_csv) From 2a4ffb2332818f4aeaa89a59e6d1ad1a6e8f5bee Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:52:13 +0000 Subject: [PATCH 39/41] Formatting changes --- ...4b5c6_nmw_per_well_geothermal_ogc_views.py | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py index e2cf47df4..696b8118a 100644 --- a/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py +++ b/alembic/versions/d1e2f3a4b5c6_nmw_per_well_geothermal_ogc_views.py @@ -48,9 +48,7 @@ def upgrade() -> None: # ogc_geothermal_wells_bht op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_BHT_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -87,15 +85,11 @@ def upgrade() -> None: hdr."CurWellNam", hdr."API", hdr."TotalDepth" - """ - ) - ) + """)) # ogc_geothermal_wells_temperature_profile (materialized) op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -136,9 +130,7 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) op.execute( text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') ) @@ -150,9 +142,7 @@ def upgrade() -> None: # ogc_geothermal_wells_summary_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_SUM_HF_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_SUM_HF_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -205,15 +195,11 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) # ogc_geothermal_wells_interval_heat_flow op.execute(text(f'DROP VIEW IF EXISTS "{_INT_HF_VIEW}"')) - op.execute( - text( - f""" + op.execute(text(f""" CREATE VIEW "{_INT_HF_VIEW}" AS WITH loc AS ( SELECT DISTINCT ON ("WellDataID") @@ -266,9 +252,7 @@ def upgrade() -> None: loc."Long_dd83", hdr."CurWellNam", hdr."API" - """ - ) - ) + """)) def downgrade() -> None: From 8aa89c141405f14de5d3251377530a6d0e5a6b94 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 24 Jun 2026 07:55:21 -0600 Subject: [PATCH 40/41] docs(nmw): add NM_Wells mirror transfer + verification runbook Operational steps to run the Phase-1 NM_Wells 1:1 mirror (export, load, refresh) and verify it: row-count parity, FK orphan checks, OGC view/API checks, reversible migrations. Maps sign-off to BDMS-969/951/954 and documents the B1/B2/B3 fixes in troubleshooting. docs/ is gitignored; force-added. Co-Authored-By: Claude Opus 4.8 --- docs/nm_wells-transfer-runbook.md | 248 ++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 docs/nm_wells-transfer-runbook.md diff --git a/docs/nm_wells-transfer-runbook.md b/docs/nm_wells-transfer-runbook.md new file mode 100644 index 000000000..b1168f2ac --- /dev/null +++ b/docs/nm_wells-transfer-runbook.md @@ -0,0 +1,248 @@ +# NM_Wells 1:1 Mirror Transfer — Runbook + +Operational steps to run the NM_Wells (geothermal) Phase-1 mirror transfer and verify it +worked. Phase 1 is a faithful, column-for-column copy of the legacy NM_Wells SQL Server +tables into the Postgres `NMW_*` staging mirror — no transform to the Ocotillo model. + +- Code: `transfers/transfer_geothermal.py` (orchestrator), `transfers/nmw_mirror_transfer.py` + (loader), `transfers/export_nmw_csvs.py` (CSV export), `transfers/nmw_sql_dump.py` (dump parser). +- Models: `db/nmw_legacy.py` (18 `NMW_*` tables). +- Design notes: [docs/nm_wells-migration.md](nm_wells-migration.md). +- Jira: [BDMS-945](https://nmbgmr.atlassian.net/browse/BDMS-945) (story), + [BDMS-969](https://nmbgmr.atlassian.net/browse/BDMS-969) (this e2e run), + [BDMS-970](https://nmbgmr.atlassian.net/browse/BDMS-970) (SQL Server access — blocker). + +--- + +## 0. Prerequisites + +- [ ] **SQL Server access** ([BDMS-970](https://nmbgmr.atlassian.net/browse/BDMS-970)): + password reset done, can reach the NM_Wells host (Argon / Agustin / Sediment / + SQL dev / SQLServer2019 as applicable). +- [ ] Python env ready: `uv venv && source .venv/bin/activate && uv sync --locked`. +- [ ] Target Postgres + PostGIS reachable; `.env` has `POSTGRES_*` (or Cloud SQL) creds. +- [ ] `.env` has the SQL Server source creds (only needed for the live export, step 1): + +```bash +NMW_HOST= +NMW_PORT=1433 +NMW_USER= +NMW_PASSWORD= +NMW_DATABASE=NM_Wells +``` + +Pick **one** row source for the load: + +| Source | When | Set | +|--------|------|-----| +| Per-table CSVs | live export via pymssql (default path) | nothing (`NMW_SQL_DUMP` unset) | +| SQL dump `.sql` | you have an SSMS data dump | `NMW_SQL_DUMP=/path/to/dump.sql` | + +--- + +## 1. Apply schema (migrations) + +The transfer assumes the schema already exists — it does not create/drop tables. + +```bash +alembic upgrade head +``` + +Creates the 18 `NMW_*` tables + FKs and the 8 OGC backing views. Migration chain: +`c0d1e2f3a4b5` (tables+FK) → `d1e2f3a4b5c6` (per-well views) → `e2f3a4b5c6d7` (measurement views). + +Verify the tables and views exist: + +```bash +psql "$DATABASE_URL" -c '\dt "NMW_*"' # expect 18 tables +psql "$DATABASE_URL" -c '\dv ogc_*' # ogc_* views +psql "$DATABASE_URL" -c '\dm ogc_*' # matview: ogc_geothermal_wells_temperature_profile +``` + +--- + +## 2. Export source tables to CSV (live source) + +Skip if you're loading from a `.sql` dump (`NMW_SQL_DUMP` set). + +```bash +uv run python -m transfers.export_nmw_csvs +``` + +- Writes `transfers/data/nma_csv_cache/
.csv`, one per mirrored table. +- Prints per-table row counts — **record these**; they are the source-of-truth counts for + the post-load comparison in step 4. +- Any `FAILED: ...` line means that table didn't export — investigate before loading. + +--- + +## 3. Run the transfer + +Smoke-test with a row cap first, then run the full load. + +```bash +# Smoke test: 1000 rows/table +TRANSFER_LIMIT=1000 uv run python -m transfers.transfer_geothermal + +# Full load (all rows) +uv run python -m transfers.transfer_geothermal +``` + +Relevant env (all optional, sane defaults): + +| Var | Default | Effect | +|-----|---------|--------| +| `TRANSFER_LIMIT` | 0 (all) | rows per table | +| `NMW_SQL_DUMP` | unset | load from `.sql` dump instead of CSVs | +| `NMW_CSV_DIR` | temp dir | where dump-derived CSVs are written (dump path) | +| `TRANSFER_GEOTHERMAL_REFERENCE` | 1 | load `ref_*` → lexicon | +| `TRANSFER_NMW_MIRROR` | 1 | load `NMW_*` mirror + refresh matviews | + +The orchestrator: loads reference→lexicon, loads the mirror parent→child in FK order, then +refreshes the materialized OGC view. It prints a summary dict — confirm +`mirror.errors == 0` and `reference.errors == 0`. + +Re-running is safe: dump path is truncate+COPY (CASCADE), CSV path is +`INSERT ... ON CONFLICT DO NOTHING`. No duplicate rows. + +--- + +## 4. Verify — row counts + +Compare each mirror table's row count against the source counts captured in step 2 +(or against SQL Server directly). + +```bash +psql "$DATABASE_URL" <<'SQL' +SELECT 'NMW_WellHeaders' AS t, count(*) FROM "NMW_WellHeaders" +UNION ALL SELECT 'NMW_WellLocations', count(*) FROM "NMW_WellLocations" +UNION ALL SELECT 'NMW_WellRecords', count(*) FROM "NMW_WellRecords" +UNION ALL SELECT 'NMW_WellSamples', count(*) FROM "NMW_WellSamples" +UNION ALL SELECT 'NMW_WellZDatum', count(*) FROM "NMW_WellZDatum" +UNION ALL SELECT 'NMW_Sources', count(*) FROM "NMW_Sources" +UNION ALL SELECT 'NMW_GtBhtHeaders', count(*) FROM "NMW_GtBhtHeaders" +UNION ALL SELECT 'NMW_GtBhtData', count(*) FROM "NMW_GtBhtData" +UNION ALL SELECT 'NMW_GtTempDepths', count(*) FROM "NMW_GtTempDepths" +UNION ALL SELECT 'NMW_GtConductivity', count(*) FROM "NMW_GtConductivity" +UNION ALL SELECT 'NMW_GtHeatFlow', count(*) FROM "NMW_GtHeatFlow" +UNION ALL SELECT 'NMW_GtSumHeatFlow', count(*) FROM "NMW_GtSumHeatFlow" +UNION ALL SELECT 'NMW_WsDstHeaders', count(*) FROM "NMW_WsDstHeaders" +UNION ALL SELECT 'NMW_WsDstIntervals', count(*) FROM "NMW_WsDstIntervals" +UNION ALL SELECT 'NMW_WsDstFlowHistory', count(*) FROM "NMW_WsDstFlowHistory" +UNION ALL SELECT 'NMW_WsDstFluidProperties', count(*) FROM "NMW_WsDstFluidProperties" +UNION ALL SELECT 'NMW_WsDstPressure', count(*) FROM "NMW_WsDstPressure" +UNION ALL SELECT 'NMW_WsIntervals', count(*) FROM "NMW_WsIntervals" +ORDER BY t; +SQL +``` + +**Pass:** every count matches source (or matches `TRANSFER_LIMIT` if capped). Note any table +where the count is 0 or short — likely a failed export or an FK-skipped child row. + +--- + +## 5. Verify — FK integrity + +No child row should reference a missing parent. Spot-check the main hierarchy +(`NMW_WellHeaders` is the root parent): + +```bash +psql "$DATABASE_URL" <<'SQL' +-- locations / records with no matching well header (expect 0) +SELECT 'orphan_locations' AS check, count(*) +FROM "NMW_WellLocations" l +LEFT JOIN "NMW_WellHeaders" h ON l."WellDataID" = h."WellDataID" +WHERE h."WellDataID" IS NULL +UNION ALL +SELECT 'orphan_records', count(*) +FROM "NMW_WellRecords" r +LEFT JOIN "NMW_WellHeaders" h ON r."WellDataID" = h."WellDataID" +WHERE h."WellDataID" IS NULL; +SQL +``` + +**Pass:** both counts are 0. (FK constraints are enforced at load, so a non-zero here means +data was loaded out of order or a constraint is missing — investigate.) + +--- + +## 6. Verify — OGC views + matview + +Refresh happens automatically in step 3. To refresh manually: + +```bash +psql "$DATABASE_URL" -c 'REFRESH MATERIALIZED VIEW ogc_geothermal_wells_temperature_profile;' +``` + +Confirm each backing view returns rows and the per-well views emit **one feature per well** +(no count multiplication from duplicate location rows): + +```bash +psql "$DATABASE_URL" <<'SQL' +SELECT 'ogc_geothermal_wells_bht' AS v, count(*) FROM ogc_geothermal_wells_bht +UNION ALL SELECT 'ogc_geothermal_wells_temperature_profile', count(*) FROM ogc_geothermal_wells_temperature_profile +UNION ALL SELECT 'ogc_geothermal_wells_summary_heat_flow', count(*) FROM ogc_geothermal_wells_summary_heat_flow +UNION ALL SELECT 'ogc_geothermal_wells_interval_heat_flow', count(*) FROM ogc_geothermal_wells_interval_heat_flow +UNION ALL SELECT 'ogc_bht_measurements', count(*) FROM ogc_bht_measurements +UNION ALL SELECT 'ogc_temp_depth_measurements',count(*) FROM ogc_temp_depth_measurements +UNION ALL SELECT 'ogc_heat_flow', count(*) FROM ogc_heat_flow +UNION ALL SELECT 'ogc_dst', count(*) FROM ogc_dst; +SQL +``` + +Then hit the OGC API (with the app running) — all 6 collections should resolve and return +GeoJSON features: + +```bash +for c in geothermal_wells_bht geothermal_wells_temperature_profile \ + bht_measurements temp_depth_measurements heat_flow dst; do + echo "== $c ==" + curl -s "http://localhost:8000/ogcapi/collections/$c/items?limit=1" | head -c 400 + echo +done +``` + +**Pass:** each returns HTTP 200 with a `FeatureCollection`; per-well collections show +distinct wells (no duplicate `WellDataID`). + +--- + +## 7. Verify — migrations reversible (non-prod only) + +On a scratch/test DB, confirm a clean down/up cycle drops and recreates all 18 tables + 8 +views with no orphans: + +```bash +alembic downgrade base +psql "$DATABASE_URL" -c '\dt "NMW_*"' # expect 0 +psql "$DATABASE_URL" -c '\dv ogc_*' # expect 0 +alembic upgrade head # recreate +``` + +Automated coverage for this lives in `tests/test_nmw_mirror.py` (19 tests): +`uv run pytest tests/test_nmw_mirror.py`. + +--- + +## Sign-off checklist (closes BDMS-969 → unblocks BDMS-951 / BDMS-954) + +- [ ] Schema applied; 18 tables + 8 views present (step 1). +- [ ] Source CSVs exported; per-table source counts recorded (step 2). +- [ ] Transfer ran with `errors == 0` (step 3). +- [ ] Row counts match source (step 4). +- [ ] No orphan FK rows (step 5). +- [ ] All 6 OGC collections resolve; one feature per well (step 6). +- [ ] Migrations down/up clean on scratch DB (step 7). + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `export_nmw_csvs` connection refused | SQL Server access not granted | [BDMS-970](https://nmbgmr.atlassian.net/browse/BDMS-970); check `NMW_HOST/PORT`, VPN | +| `TRUNCATE ... cannot truncate a table referenced in a foreign key` | parent truncated before child | loader uses `TRUNCATE ... CASCADE` (B2); confirm you're on current branch | +| Mirror column holds literal `CAST(...)` string | dump parser missed a parameterised type | fixed in `nmw_sql_dump.py` (B1); confirm branch is current | +| Per-well OGC view count > # wells | duplicate `NMW_WellLocations` rows | views dedup via `DISTINCT ON (WellDataID)` (B3); confirm branch is current | +| matview empty / stale | refresh skipped | `REFRESH MATERIALIZED VIEW ogc_geothermal_wells_temperature_profile;` | +| child table row count short | FK-skipped rows (`ON CONFLICT`/missing parent) | check parent loaded first; re-run full load | From f550ad954e9f8fbd8b1e8d0c7bab1906a5c9f428 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 24 Jun 2026 12:00:59 -0600 Subject: [PATCH 41/41] feat: allow configurable output directory for NMW CSV exports --- docs/nm_wells-transfer-runbook.md | 6 ++++++ transfers/export_nmw_csvs.py | 5 ++++- transfers/util.py | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/nm_wells-transfer-runbook.md b/docs/nm_wells-transfer-runbook.md index b1168f2ac..60731a8b5 100644 --- a/docs/nm_wells-transfer-runbook.md +++ b/docs/nm_wells-transfer-runbook.md @@ -164,6 +164,12 @@ SQL **Pass:** both counts are 0. (FK constraints are enforced at load, so a non-zero here means data was loaded out of order or a constraint is missing — investigate.) +**Known exception:** `orphan_locations` reports **51** rows. These are `NMW_WellLocations` +rows whose `WellDataID` is blank in the source (`tbl_well_locations.csv`): empty values load +as NULL, NULL FK columns are exempt from FK enforcement, and the `LEFT JOIN ... IS NULL` +check counts them as orphans. This is a source data-quality issue, not a load-order or +constraint problem — accepted as-is. `orphan_records` must still be 0. + --- ## 6. Verify — OGC views + matview diff --git a/transfers/export_nmw_csvs.py b/transfers/export_nmw_csvs.py index e2cf438b3..51daf403e 100644 --- a/transfers/export_nmw_csvs.py +++ b/transfers/export_nmw_csvs.py @@ -27,7 +27,10 @@ TABLES = [spec.source_table for spec in NMW_MIRROR_SPECS] -OUT_DIR = Path(__file__).parent / "data" / "nma_csv_cache" +_data_root = os.environ.get("TRANSFERS_DATA_DIR") +OUT_DIR = ( + Path(_data_root) if _data_root else Path(__file__).parent / "data" / "nma_csv_cache" +) def _get_connection(): diff --git a/transfers/util.py b/transfers/util.py index 5fd1a4710..ff5c4f4e7 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -406,6 +406,11 @@ def extract_organization(alternate_id: str) -> str: def get_transfers_data_path(name: str) -> Path: + # Explicit override wins: CSVs live flat in this dir (no nma_csv_cache subdir). + env_root = os.environ.get("TRANSFERS_DATA_DIR") + if env_root: + return Path(env_root) / Path(name).name + def data_path(r): return Path(r) / "transfers" / "data"