diff --git a/api/sensor.py b/api/sensor.py index 92360a4d1..49e1c0ba5 100644 --- a/api/sensor.py +++ b/api/sensor.py @@ -16,7 +16,7 @@ from fastapi import APIRouter, Query, Response from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy import select, and_ +from sqlalchemy import select from starlette import status from api.pagination import CustomPage @@ -28,8 +28,9 @@ ) from db import Observation, Sensor, Deployment, Thing from schemas.sensor import SensorResponse, CreateSensor, UpdateSensor -from services.crud_helper import model_patcher, model_deleter, model_adder -from services.exceptions_helper import PydanticStyleException +from services.crud_helper import model_deleter, model_adder, model_patcher + +# from services.exceptions_helper import PydanticStyleException from services.query_helper import order_sort_filter, simple_get_by_id router = APIRouter(prefix="/sensor", tags=["sensor"]) @@ -50,6 +51,9 @@ async def add_sensor( # ====== PATCH ================================================================= +# TODO: datetime_installed and datetime_removed have been moved from the Sensor model to the Deployment model. Do we need to keep the validation for datetime_installed and datetime_removed? + + @router.patch("/{sensor_id}", status_code=status.HTTP_200_OK) async def update_sensor( sensor_id: int, @@ -60,50 +64,50 @@ async def update_sensor( """ Update a sensor in the system. """ - if ( - sensor_data.datetime_installed is not None - and sensor_data.datetime_removed is None - ): - sensor = simple_get_by_id(session, Sensor, sensor_id) - existing_datetime_removed = sensor.datetime_removed - if ( - existing_datetime_removed is not None - and sensor_data.datetime_installed >= existing_datetime_removed - ): - detail = { - "loc": ["body", "datetime_installed"], - "msg": f"new datetime installed must be before existing datetime removed of {existing_datetime_removed.isoformat().replace('+00:00', 'Z')}", - "type": "value_error", - "input": { - "datetime_installed": sensor_data.datetime_installed.isoformat().replace( - "+00:00", "Z" - ) - }, - } - raise PydanticStyleException( - status_code=status.HTTP_409_CONFLICT, detail=[detail] - ) - elif ( - sensor_data.datetime_installed is None - and sensor_data.datetime_removed is not None - ): - sensor = simple_get_by_id(session, Sensor, sensor_id) - existing_datetime_installed = sensor.datetime_installed - if sensor_data.datetime_removed <= existing_datetime_installed: - detail = { - "loc": ["body", "datetime_removed"], - "msg": f"new datetime removed must be after existing datetime installed of {existing_datetime_installed.isoformat().replace('+00:00', 'Z')}", - "type": "value_error", - "input": { - "datetime_removed": sensor_data.datetime_removed.isoformat().replace( - "+00:00", "Z" - ) - }, - } - raise PydanticStyleException( - status_code=status.HTTP_409_CONFLICT, detail=[detail] - ) - + # if ( + # sensor_data.datetime_installed is not None + # and sensor_data.datetime_removed is None + # ): + # sensor = simple_get_by_id(session, Sensor, sensor_id) + # existing_datetime_removed = sensor.datetime_removed + # if ( + # existing_datetime_removed is not None + # and sensor_data.datetime_installed >= existing_datetime_removed + # ): + # detail = { + # "loc": ["body", "datetime_installed"], + # "msg": f"new datetime installed must be before existing datetime removed of {existing_datetime_removed.isoformat().replace('+00:00', 'Z')}", + # "type": "value_error", + # "input": { + # "datetime_installed": sensor_data.datetime_installed.isoformat().replace( + # "+00:00", "Z" + # ) + # }, + # } + # raise PydanticStyleException( + # status_code=status.HTTP_409_CONFLICT, detail=[detail] + # ) + # elif ( + # sensor_data.datetime_installed is None + # and sensor_data.datetime_removed is not None + # ): + # sensor = simple_get_by_id(session, Sensor, sensor_id) + # existing_datetime_installed = sensor.datetime_installed + # if sensor_data.datetime_removed <= existing_datetime_installed: + # detail = { + # "loc": ["body", "datetime_removed"], + # "msg": f"new datetime removed must be after existing datetime installed of {existing_datetime_installed.isoformat().replace('+00:00', 'Z')}", + # "type": "value_error", + # "input": { + # "datetime_removed": sensor_data.datetime_removed.isoformat().replace( + # "+00:00", "Z" + # ) + # }, + # } + # raise PydanticStyleException( + # status_code=status.HTTP_409_CONFLICT, detail=[detail] + # ) + # return model_patcher(session, Sensor, sensor_id, sensor_data, user=user) @@ -127,8 +131,8 @@ async def delete_sensor( async def get_sensors( session: session_dependency, user: viewer_dependency, - thing_id: int = None, # Optional filter for thing_id - observed_property: str = None, # Optional filter for observed_property + thing_id: int = None, # Optional filter for thing_id. Filter by the Thing where equipment is deployed + parameter_id: int = None, # Filter by the parameter the sensor/equipment measures sort: str | None = None, order: str | None = None, filter_: str = Query(alias="filter", default=None), @@ -138,24 +142,15 @@ async def get_sensors( This endpoint is a placeholder and should be implemented with actual logic. """ sql = select(Sensor) - if thing_id is not None or observed_property is not None: - conditions = [] - joins = [] - if observed_property is not None: - joins.append(Observation) - conditions.append(Observation.observed_property == observed_property) - - if thing_id is not None: - joins.append(Deployment) - joins.append(Thing) - conditions.append(Thing.id == thing_id) - - if joins: - for j in joins: - sql = sql.join(j) - - if conditions: - sql = sql.where(and_(*conditions)) + # --- Logic to filter by Thing --- + # The path is now: Sensor <-> Deployment <-> Thing + if thing_id is not None: + sql = sql.join(Deployment).join(Thing).where(Thing.id == thing_id) + + # --- Logic to filter by Parameter --- + # The path is now: Sensor <-> Observation <-> Parameter + if parameter_id is not None: + sql = sql.join(Observation).where(Observation.parameter_id == parameter_id) sql = order_sort_filter(sql, Sensor, sort=sort, order=order, filter_=filter_) return paginate(conn=session, query=sql) diff --git a/core/lexicon.json b/core/lexicon.json index daa4b0148..e1ffd9cf4 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -9,7 +9,6 @@ {"name": "coordinate_method", "description": null}, {"name": "country", "description": null}, {"name": "county", "description": null}, - {"name": "well_purpose", "description": null}, {"name": "data_quality", "description": null}, {"name": "data_source", "description": null}, {"name": "depth_completion_source", "description": null}, @@ -37,12 +36,16 @@ {"name": "sample_matrix", "description": null}, {"name": "sample_method", "description": null}, {"name": "sample_type", "description": null}, + {"name": "sensor_type", "description": null}, + {"name": "sensor_status", "description": null}, {"name": "spring_type", "description": null}, {"name": "state", "description": null}, {"name": "status", "description": null}, {"name": "thing_type", "description": null}, {"name": "unit", "description": null}, - {"name": "vertical_datum", "description": null} + {"name": "vertical_datum", "description": null}, + {"name": "well_purpose", "description": null}, + {"name": "well_status", "description": null} ], "terms": [ {"categories": ["qc_type"], "term": "Normal", "definition": "The primary environmental sample collected from the well, spring, or soil boring."}, @@ -97,6 +100,8 @@ {"categories": ["well_purpose"], "term": "Institutional", "definition": "Institutional"}, {"categories": ["well_purpose"], "term": "Unused", "definition": "Unused"}, {"categories": ["well_purpose"], "term": "Exploration", "definition": "Exploration well"}, + {"categories": ["well_purpose"], "term": "Monitoring", "definition": "Monitoring"}, + {"categories": ["well_purpose"], "term": "Production", "definition": "Production"}, {"categories": ["well_purpose"], "term": "Injection", "definition": "Injection"}, {"categories": ["data_quality"], "term": "Water level accurate to within two hundreths of a foot", "definition": "Good"}, {"categories": ["data_quality"], "term": "Water level accurate to within one foot", "definition": "Fair"}, @@ -312,10 +317,10 @@ {"categories": ["groundwater_level_reason"], "term": "Water level affected by stage in nearby surface-water site", "definition": "Water level affected by stage in nearby surface-water site"}, {"categories": ["groundwater_level_reason"], "term": "Other conditions exist that would affect the level (remarks)", "definition": "Other conditions exist that would affect the level (remarks)"}, {"categories": ["groundwater_level_reason"], "term": "Water level not affected", "definition": "Water level not affected"}, - {"categories": ["status"], "term": "Abandoned", "definition": "Abandoned"}, - {"categories": ["status"], "term": "Active, pumping well", "definition": "Active, pumping well"}, - {"categories": ["status"], "term": "Destroyed, exists but not usable", "definition": "Destroyed, exists but not usable"}, - {"categories": ["status"], "term": "Inactive, exists but not used", "definition": "Inactive, exists but not used"}, + {"categories": ["well_status"], "term": "Abandoned", "definition": "Abandoned"}, + {"categories": ["well_status"], "term": "Active, pumping well", "definition": "Active, pumping well"}, + {"categories": ["well_status"], "term": "Destroyed, exists but not usable", "definition": "Destroyed, exists but not usable"}, + {"categories": ["well_status"], "term": "Inactive, exists but not used", "definition": "Inactive, exists but not used"}, {"categories": ["sample_method"], "term": "Airline measurement", "definition": "Airline measurement"}, {"categories": ["sample_method"], "term": "Analog or graphic recorder", "definition": "Analog or graphic recorder"}, {"categories": ["sample_method"], "term": "Calibrated airline measurement", "definition": "Calibrated airline measurement"}, @@ -508,6 +513,18 @@ {"categories": ["parameter_type"], "term": "Radionuclide", "definition": "Radionuclide"}, {"categories": ["parameter_type"], "term": "Major Element", "definition": "Major Element"}, {"categories": ["parameter_type"], "term": "Minor Element", "definition": "Minor Element"}, - {"categories": ["parameter_type"], "term": "Physical property", "definition": "Physical property"} + {"categories": ["parameter_type"], "term": "Physical property", "definition": "Physical property"}, + {"categories": ["sensor_type"], "term": "Pressure Transducer", "definition": "Pressure Transducer"}, + {"categories": ["sensor_type"], "term": "Data Logger", "definition": "Data Logger"}, + {"categories": ["sensor_type"], "term": "Barometer", "definition": "Barometer"}, + {"categories": ["sensor_type"], "term": "Acoustic Sounder", "definition": "Acoustic Sounder"}, + {"categories": ["sensor_type"], "term": "Precip Collector", "definition": "Precip Collector"}, + {"categories": ["sensor_type"], "term": "Camera", "definition": "Camera"}, + {"categories": ["sensor_type"], "term": "Soil Moisture Sensor", "definition": "Soil Moisture Sensor"}, + {"categories": ["sensor_type"], "term": "Tipping Bucket", "definition": "Tipping Bucket"}, + {"categories": ["sensor_status"], "term": "In Service", "definition": "In Service"}, + {"categories": ["sensor_status"], "term": "In Repair", "definition": "In Repair"}, + {"categories": ["sensor_status"], "term": "Retired", "definition": "Retired"}, + {"categories": ["sensor_status"], "term": "Lost", "definition": "Lost"} ] } \ No newline at end of file diff --git a/db/deployment.py b/db/deployment.py index 72586c980..0b2dc61df 100644 --- a/db/deployment.py +++ b/db/deployment.py @@ -37,7 +37,11 @@ class Deployment(Base, AutoBaseMixin, ReleaseMixin): removal_date: Mapped[Date] = mapped_column(Date, nullable=True) recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) recording_interval_units: Mapped[str] = lexicon_term(nullable=True) - hanging_cable_length: Mapped[float] = mapped_column(Numeric, nullable=True) + hanging_cable_length: Mapped[float] = mapped_column( + Numeric, + nullable=True, + comment="Length of cable from sensor to hanging point, in ft", + ) hanging_point_height: Mapped[float] = mapped_column(Numeric, nullable=True) hanging_point_description: Mapped[str] = mapped_column(Text, nullable=True) notes: Mapped[str] = mapped_column(Text, nullable=True) diff --git a/db/sensor.py b/db/sensor.py index 5face8a96..7a0517805 100644 --- a/db/sensor.py +++ b/db/sensor.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime +""" +SQLAlchemy model for the Sensor table. +""" -from sqlalchemy import String, Integer, DateTime +from sqlalchemy import String, Text from sqlalchemy.ext.associationproxy import AssociationProxy, association_proxy from sqlalchemy.orm import relationship, mapped_column, Mapped -from db.base import Base, AutoBaseMixin, ReleaseMixin +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term from typing import List, TYPE_CHECKING @@ -31,22 +33,54 @@ class Sensor(Base, AutoBaseMixin, ReleaseMixin): """ - Base class for all sensor types. + The `Sensor` table serves as the central asset inventory for all physical hardware used for data collection. + Its purpose is to track each unique piece of equipment as a distinct asset. + + This table is distinct from the `AnalysisMethod` table, as it deals exclusively with tangible, physical objects. + This class can be extended to create specific sensor types. """ - # Define common attributes for sensors here + # --- Columns --- + nma_pk_equipment: Mapped[str] = mapped_column( + String(36), + nullable=True, + comment="Preserves the primary key (GlobalID) of the Equipment table from NMAquifer.", + ) + # TODO: Is a name field necessary? If it is, we should consider standardizing naming conventions. name: Mapped[str] = mapped_column(String(255), nullable=False) - model: Mapped[str] = mapped_column(String(50), nullable=True) - serial_no: Mapped[str] = mapped_column(String(50), nullable=True) - datetime_installed: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False + sensor_type: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field to categorize the equipment (e.g., 'Pressure Transducer', 'Barometer', 'Data Logger', etc).", + ) + model: Mapped[str] = mapped_column( + String(50), nullable=True, comment="The specific model of the equipment." + ) + serial_no: Mapped[str] = mapped_column( + String(50), + nullable=True, + unique=True, + comment="The serial number of the equipment.", + ) + # TODO: What data type should `pcn_number` be? Should it be a string or integer? + pcn_number: Mapped[str] = mapped_column( + String(50), + nullable=True, + unique=True, + comment="The Property Control Number (PCN) assigned to equipment for inventory tracking. This is only available for equipment owned by the NMBGMR.", + ) + owner_agency: Mapped[str] = lexicon_term( + nullable=True, comment="The agency or organization that owns the equipment." + ) + sensor_status: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary field to indicate the current status of the equipment (e.g., 'In Service', 'In Repair', 'Retired', 'Lost', etc).", ) - datetime_removed: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=True + notes: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="A field for general notes or comments about the equipment.", ) - recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) - notes: Mapped[str] = mapped_column(String(50), nullable=True) # --- Relationships --- # One-To-Many: A piece of Equipment can generate many Observations. diff --git a/schemas/sensor.py b/schemas/sensor.py index 6dbcf9f19..1d5e8d169 100644 --- a/schemas/sensor.py +++ b/schemas/sensor.py @@ -13,78 +13,81 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from typing_extensions import Annotated, Self -from datetime import timezone -from pydantic import ( - BaseModel, - AwareDatetime, - PastDatetime, - model_validator, - field_validator, -) +# from typing_extensions import Annotated, Self +# from datetime import timezone +# from pydantic import ( +# BaseModel, +# AwareDatetime, +# PastDatetime, +# model_validator, +# field_validator, +# ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel # ------- VALIDATION ------ +# TODO: datetime_installed and datetime_removed were removed from the Sensor model, so this validation class may no longer be relevant. -class ValidateSensor(BaseModel): - - datetime_installed: AwareDatetime | None = None - datetime_removed: AwareDatetime | None = None - - @field_validator("datetime_installed", "datetime_removed") - def convert_datetime_fields_to_utc(cls, field: AwareDatetime) -> AwareDatetime: - if field is not None and field.tzinfo != timezone.utc: - field = field.astimezone(timezone.utc) - return field - - @model_validator(mode="after") - def check_datetime_values(self) -> Self: - if ( - getattr(self, "datetime_removed", None) is not None - and getattr(self, "datetime_installed", None) is not None - ): - if self.datetime_removed <= self.datetime_installed: - raise ValueError("datetime removed must be after datetime installed") - return self +# class ValidateSensor(BaseModel): +# +# datetime_installed: AwareDatetime | None = None +# datetime_removed: AwareDatetime | None = None +# +# @field_validator("datetime_installed", "datetime_removed") +# def convert_datetime_fields_to_utc(cls, field: AwareDatetime) -> AwareDatetime: +# if field is not None and field.tzinfo != timezone.utc: +# field = field.astimezone(timezone.utc) +# return field +# +# @model_validator(mode="after") +# def check_datetime_values(self) -> Self: +# if ( +# getattr(self, "datetime_removed", None) is not None +# and getattr(self, "datetime_installed", None) is not None +# ): +# if self.datetime_removed <= self.datetime_installed: +# raise ValueError("datetime removed must be after datetime installed") +# return self # -------- CREATE ---------- -class CreateSensor(BaseCreateModel, ValidateSensor): +class CreateSensor(BaseCreateModel): """ Schema for creating a new sensor. """ name: str - # equipment_type: str | None = None + sensor_type: str model: str | None = None serial_no: str | None = None - datetime_installed: Annotated[AwareDatetime, PastDatetime()] - datetime_removed: AwareDatetime | None = None # ISO format date string - recording_interval: int | None = None + pcn_number: str | None = None + owner_agency: str | None = None + sensor_status: str | None = None notes: str | None = None # -------- UPDATE ---------- -class UpdateSensor(BaseUpdateModel, ValidateSensor): +class UpdateSensor(BaseUpdateModel): name: str | None = None + sensor_type: str | None = None model: str | None = None serial_no: str | None = None - datetime_installed: AwareDatetime | None = None - datetime_removed: AwareDatetime | None = None - recording_interval: int | None = None + pcn_number: str | None = None + owner_agency: str | None = None + sensor_status: str | None = None notes: str | None = None # -------- RESPONSE ---------- class SensorResponse(BaseResponseModel): name: str + sensor_type: str model: str | None # = Column(String(50)) serial_no: str | None # = Column(String(50)) - datetime_installed: AwareDatetime - datetime_removed: AwareDatetime | None # = Column(DateTime) - recording_interval: int | None # = Column(Integer) + pcn_number: str | None + owner_agency: str | None # = Column(String(50)) + sensor_status: str | None # = Column(String(50)) notes: str | None # = Column(String(50)) diff --git a/tests/conftest.py b/tests/conftest.py index 61fe086e3..53832d6a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -206,11 +206,12 @@ def sensor(): with session_ctx() as session: sensor = Sensor( name=f"Test Sensor {uuid.uuid4()}", + sensor_type="Pressure Transducer", model="Model X", serial_no="123456", - datetime_installed="2023-01-01T00:00:00Z", - datetime_removed="2023-01-02T00:00:00Z", - recording_interval=60, + pcn_number="PCN123456", + owner_agency="NMBGMR", + sensor_status="In Service", notes="Test equipment", release_status="draft", ) @@ -226,11 +227,12 @@ def second_sensor(): with session_ctx() as session: sensor = Sensor( name="Test Sensor 2", + sensor_type="Pressure Transducer", model="Model X", serial_no="123456", - datetime_installed="2023-01-01T00:00:00Z", - datetime_removed="2023-01-02T00:00:00Z", - recording_interval=60, + pcn_number="PCN123456", + owner_agency="NMBGMR", + sensor_status="In Service", notes="Test equipment", release_status="draft", ) diff --git a/tests/test_sensor.py b/tests/test_sensor.py index 4ff87c80f..73106eb60 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -16,11 +16,13 @@ from core.dependencies import admin_function, editor_function, viewer_function from db import Sensor from main import app -from schemas.sensor import ValidateSensor + +# from schemas.sensor import ValidateSensor from tests import client, cleanup_post_test, cleanup_patch_test, override_authentication import pytest -from pydantic import ValidationError + +# from pydantic import ValidationError @pytest.fixture(scope="module", autouse=True) @@ -40,15 +42,16 @@ def override_dependencies_fixture(): # ====== VALIDATION tests ====================================================== +# TODO: installation and removal dates were removed from the Sensor model, so these tests may no longer be relevant. -def test_validate_datetime_installed_datetime_removed(): - with pytest.raises( - ValidationError, match="datetime removed must be after datetime installed" - ): - ValidateSensor( - datetime_installed="2023-01-02T00:00:00Z", - datetime_removed="2023-01-01T00:00:00Z", - ) +# def test_validate_datetime_installed_datetime_removed(): +# with pytest.raises( +# ValidationError, match="datetime removed must be after datetime installed" +# ): +# ValidateSensor( +# datetime_installed="2023-01-02T00:00:00Z", +# datetime_removed="2023-01-01T00:00:00Z", +# ) # ====== POST tests ============================================================ @@ -57,11 +60,12 @@ def test_validate_datetime_installed_datetime_removed(): def test_add_sensor(): payload = { "name": "Test Sensor 2", + "sensor_type": "Pressure Transducer", "model": "Model X", "serial_no": "12345678", - "datetime_installed": "2024-01-01T00:00:00Z", - "datetime_removed": None, - "recording_interval": 60, + "pcn_number": "PCN-001", + "owner_agency": "NMBGMR", + "sensor_status": "In Service", "notes": "Test equipment", "release_status": "draft", } @@ -72,11 +76,12 @@ def test_add_sensor(): assert "created_at" in data assert data["release_status"] == payload["release_status"] assert data["name"] == payload["name"] + assert data["sensor_type"] == payload["sensor_type"] assert data["model"] == payload["model"] assert data["serial_no"] == payload["serial_no"] - assert data["datetime_installed"] == payload["datetime_installed"] - assert data["datetime_removed"] == payload["datetime_removed"] - assert data["recording_interval"] == payload["recording_interval"] + assert data["pcn_number"] == payload["pcn_number"] + assert data["owner_agency"] == payload["owner_agency"] + assert data["sensor_status"] == payload["sensor_status"] assert data["notes"] == payload["notes"] # cleanup after post test @@ -89,7 +94,10 @@ def test_add_sensor(): def test_patch_sensor(sensor): payload = { "name": "patched name", + "sensor_type": "Data Logger", "model": "patched model", + "owner_agency": "USGS", + "sensor_status": "In Repair", "release_status": "draft", } response = client.patch(f"/sensor/{sensor.id}", json=payload) @@ -97,7 +105,10 @@ def test_patch_sensor(sensor): data = response.json() assert data["id"] == sensor.id assert data["name"] == payload["name"] + assert data["sensor_type"] == payload["sensor_type"] assert data["model"] == payload["model"] + assert data["owner_agency"] == payload["owner_agency"] + assert data["sensor_status"] == payload["sensor_status"] assert data["release_status"] == payload["release_status"] # cleanup after patch test @@ -113,30 +124,33 @@ def test_patch_sensor_404_not_found(sensor): assert data["detail"] == f"Sensor with ID {bad_sensor_id} not found." -def test_patch_sensor_409_conflicting_datetime_installed(sensor): - payload = {"datetime_installed": "2025-01-01T00:00:00Z"} - response = client.patch(f"/sensor/{sensor.id}", json=payload) - assert response.status_code == 409 - data = response.json() - assert data["detail"][0]["loc"] == ["body", "datetime_installed"] - assert ( - data["detail"][0]["msg"] - == f"new datetime installed must be before existing datetime removed of {sensor.datetime_removed}" - ) - assert data["detail"][0]["type"] == "value_error" +# TODO: datetime_installed and datetime_removed were removed from the Sensor model, so these tests may no longer be relevant. +# def test_patch_sensor_409_conflicting_datetime_installed(sensor): +# payload = {"datetime_installed": "2025-01-01T00:00:00Z"} +# response = client.patch(f"/sensor/{sensor.id}", json=payload) +# assert response.status_code == 409 +# data = response.json() +# assert data["detail"][0]["loc"] == ["body", "datetime_installed"] +# assert ( +# data["detail"][0]["msg"] +# == f"new datetime installed must be before existing datetime removed of {sensor.datetime_removed}" +# ) +# assert data["detail"][0]["type"] == "value_error" -def test_patch_sensor_409_conflicting_datetime_removed(sensor): - payload = {"datetime_removed": "2020-01-01T00:00:00Z"} - response = client.patch(f"/sensor/{sensor.id}", json=payload) - assert response.status_code == 409 - data = response.json() - assert data["detail"][0]["loc"] == ["body", "datetime_removed"] - assert ( - data["detail"][0]["msg"] - == f"new datetime removed must be after existing datetime installed of {sensor.datetime_installed}" - ) - assert data["detail"][0]["type"] == "value_error" +# TODO: datetime_installed and datetime_removed were removed from the Sensor model, so these tests may no longer be relevant. + +# def test_patch_sensor_409_conflicting_datetime_removed(sensor): +# payload = {"datetime_removed": "2020-01-01T00:00:00Z"} +# response = client.patch(f"/sensor/{sensor.id}", json=payload) +# assert response.status_code == 409 +# data = response.json() +# assert data["detail"][0]["loc"] == ["body", "datetime_removed"] +# assert ( +# data["detail"][0]["msg"] +# == f"new datetime removed must be after existing datetime installed of {sensor.datetime_installed}" +# ) +# assert data["detail"][0]["type"] == "value_error" # ====== GET tests ============================================================= @@ -153,11 +167,12 @@ def test_get_sensors(sensor): ) assert data["items"][0]["release_status"] == sensor.release_status assert data["items"][0]["name"] == sensor.name + assert data["items"][0]["sensor_type"] == sensor.sensor_type assert data["items"][0]["model"] == sensor.model assert data["items"][0]["serial_no"] == sensor.serial_no - assert data["items"][0]["datetime_installed"] == sensor.datetime_installed - assert data["items"][0]["datetime_removed"] == sensor.datetime_removed - assert data["items"][0]["recording_interval"] == sensor.recording_interval + assert data["items"][0]["pcn_number"] == sensor.pcn_number + assert data["items"][0]["owner_agency"] == sensor.owner_agency + assert data["items"][0]["sensor_status"] == sensor.sensor_status assert data["items"][0]["notes"] == sensor.notes @@ -176,11 +191,12 @@ def test_get_sensors_by_thing_id( ) assert data["items"][0]["release_status"] == sensor.release_status assert data["items"][0]["name"] == sensor.name + assert data["items"][0]["sensor_type"] == sensor.sensor_type assert data["items"][0]["model"] == sensor.model assert data["items"][0]["serial_no"] == sensor.serial_no - assert data["items"][0]["datetime_installed"] == sensor.datetime_installed - assert data["items"][0]["datetime_removed"] == sensor.datetime_removed - assert data["items"][0]["recording_interval"] == sensor.recording_interval + assert data["items"][0]["pcn_number"] == sensor.pcn_number + assert data["items"][0]["owner_agency"] == sensor.owner_agency + assert data["items"][0]["sensor_status"] == sensor.sensor_status assert data["items"][0]["notes"] == sensor.notes @@ -192,11 +208,12 @@ def test_get_sensor_by_id(sensor): assert data["created_at"] == sensor.created_at.isoformat().replace("+00:00", "Z") assert data["release_status"] == sensor.release_status assert data["name"] == sensor.name + assert data["sensor_type"] == sensor.sensor_type assert data["model"] == sensor.model assert data["serial_no"] == sensor.serial_no - assert data["datetime_installed"] == sensor.datetime_installed - assert data["datetime_removed"] == sensor.datetime_removed - assert data["recording_interval"] == sensor.recording_interval + assert data["pcn_number"] == sensor.pcn_number + assert data["owner_agency"] == sensor.owner_agency + assert data["sensor_status"] == sensor.sensor_status assert data["notes"] == sensor.notes diff --git a/transfers/sensor_transfer.py b/transfers/sensor_transfer.py index d6a3e7f8d..86635643a 100644 --- a/transfers/sensor_transfer.py +++ b/transfers/sensor_transfer.py @@ -14,8 +14,106 @@ # limitations under the License. # =============================================================================== from datetime import datetime +import pandas as pd -from db import Sensor +from db import Sensor, Deployment, Thing +from transfers.util import read_csv, logger + +EQUIPMENT_TO_SENSOR_TYPE_MAP = { + "Pressure transducer": "Pressure Transducer", + "Acoustic sounder": "Acoustic Sounder", + "Barometer": "Barometer", +} + + +def transfer_sensors(session): + equipment = read_csv("Equipment") + equipment.columns = equipment.columns.str.replace(" ", "_") + grouped_equipment = equipment.groupby(["PointID"]) + + for index, group in grouped_equipment: + pointid = index[0] + thing = session.query(Thing).filter(Thing.name == pointid).first() + if thing is None: + logger.warning( + f"Skipping sensor transfer for Thing with PointID {pointid} since it is not in the DB" + ) + continue + ordered_group = group.sort_values(by=["DateInstalled"]) + + if pointid == "SO-0168": + print(ordered_group) + + try: + for row in ordered_group.itertuples(): + if row.EquipmentType not in EQUIPMENT_TO_SENSOR_TYPE_MAP: + logger.critical( + f"Skipping equipment with type {row.EquipmentType} for point {pointid}" + ) + continue + + sensor = ( + session.query(Sensor) + .filter(Sensor.serial_no == row.SerialNo) + .one_or_none() + ) + if sensor: + logger.info( + f"Sensor with serial number {row.SerialNo} already exists. Only creating deployment for that record" + ) + else: + sensor = Sensor( + nma_pk_equipment=row.GlobalID, + name=row.ID, + sensor_type=EQUIPMENT_TO_SENSOR_TYPE_MAP[row.EquipmentType], + model=row.Model, + serial_no=row.SerialNo, + owner_agency="NMBGMR", + notes=row.Equipment_Notes, + ) + session.add(sensor) + logger.info( + f"Added sensor {sensor.name} with serial number {sensor.serial_no}" + ) + + installation_date = datetime.strptime( + row.DateInstalled, "%Y-%m-%d %H:%M:%S.%f" + ).date() + removal_date = ( + datetime.strptime(row.DateRemoved, "%Y-%m-%d %H:%M:%S.%f").date() + if not pd.isna(row.DateRemoved) + else None + ) + deployment = Deployment( + thing=thing, + sensor=sensor, + installation_date=installation_date, + removal_date=removal_date, + recording_interval=int(row.RecordingInterval), + recording_interval_units="hour", + hanging_cable_length=row.HangingCableLength, + hanging_point_height=row.HangingPointHgt, + hanging_point_description=row.HangingPointDescription, + ) + session.add(deployment) + logger.info( + f"Added deployment for sensor with serial number {sensor.serial_no}, deployed to {thing.name}: | Installation Date: {installation_date} | Removal Date: {removal_date}" + ) + + """ + Developer's notes + + Since it's unclear beforehand if a sensor has been removed just update + the sensor_status based off of each deployments installation/removal + dates + """ + if installation_date: + sensor.sensor_status = "In Service" + if removal_date: + sensor.sensor_status = "Retired" + session.commit() + except Exception as e: + logger.critical(f"Could not add sensor and deployment: {e}") # ============= EOF ============================================= @@ -27,3 +125,7 @@ def init_sensor(session): sensor.datetime_installed = datetime.now() session.add(sensor) session.commit() + + +if __name__ == "__main__": + transfer_sensors("abc") diff --git a/transfers/transfer.py b/transfers/transfer.py index 10f4a0fe6..77dd29ef4 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -28,7 +28,7 @@ from transfers.group_transfer import transfer_groups from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata from transfers.contact_transfer import transfer_contacts -from transfers.sensor_transfer import init_sensor +from transfers.sensor_transfer import transfer_sensors from transfers.waterlevels_transfer import transfer_water_levels from transfers.well_transfer import ( transfer_wells, @@ -54,13 +54,6 @@ def erase_and_initalize(session: Session) -> None: erase(session) lexicon() parameter() - sensor(session) - - -@timeit -def sensor(session: Session): - logger.info("Initializing sensors") - init_sensor(session) @timeit @@ -125,6 +118,9 @@ def transfer_all(sess, limit=100): message("TRANSFERRING METEOROLOGICAL") timeit_direct(transfer_met, sess, limit) + message("TRANSFERRING SENSORS") + timeit_direct(transfer_sensors, sess) + message("TRANSFERRING CONTACTS") timeit_direct(transfer_contacts, sess)