feat: transfer sensors and deployments

Author: jacob-a-brown

.pre-commit-config.yaml

@@ -15,15 +15,15 @@ repos:
           '--statistics'
           ]
         exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly
-#  - repo: local
-#    hooks:
-#      - id: pytest
-#        name: pytest
-#        entry: pytest  # Or your specific test command, e.g., poetry run pytest
-#        language: system
-#        types: [python] # Specify relevant file types for your tests
-#        pass_filenames: false
-#        always_run: true
+  - repo: local
+    hooks:
+      - id: pytest
+        name: pytest
+        entry: pytest  # Or your specific test command, e.g., poetry run pytest
+        language: system
+        types: [python] # Specify relevant file types for your tests
+        pass_filenames: false
+        always_run: true
 
   # - repo: https://github.com/pre-commit/mirrors-mypy
   #   rev: v1.10.0  # Use the latest stable version or pin to your preference

db/deployment.py

@@ -37,7 +37,11 @@ class Deployment(Base, AutoBaseMixin, ReleaseMixin):
     removal_date: Mapped[Date] = mapped_column(Date, nullable=True)
     recording_interval: Mapped[int] = mapped_column(Integer, nullable=True)
     recording_interval_units: Mapped[str] = lexicon_term(nullable=True)
-    hanging_cable_length: Mapped[float] = mapped_column(Numeric, nullable=True)
+    hanging_cable_length: Mapped[float] = mapped_column(
+        Numeric,
+        nullable=True,
+        comment="Length of cable from sensor to hanging point, in ft",
+    )
     hanging_point_height: Mapped[float] = mapped_column(Numeric, nullable=True)
     hanging_point_description: Mapped[str] = mapped_column(Text, nullable=True)
     notes: Mapped[str] = mapped_column(Text, nullable=True)

transfers/sensor_transfer.py

@@ -14,8 +14,106 @@
 # limitations under the License.
 # ===============================================================================
 from datetime import datetime
+import pandas as pd
 
-from db import Sensor
+from db import Sensor, Deployment, Thing
+from transfers.util import read_csv, logger
+
+EQUIPMENT_TO_SENSOR_TYPE_MAP = {
+    "Pressure transducer": "Pressure Transducer",
+    "Acoustic sounder": "Acoustic Sounder",
+    "Barometer": "Barometer",
+}
+
+
+def transfer_sensors(session):
+    equipment = read_csv("Equipment")
+    equipment.columns = equipment.columns.str.replace(" ", "_")
+    grouped_equipment = equipment.groupby(["PointID"])
+
+    for index, group in grouped_equipment:
+        pointid = index[0]
+        thing = session.query(Thing).filter(Thing.name == pointid).first()
+        if thing is None:
+            logger.warning(
+                f"Skipping sensor transfer for Thing with PointID {pointid} since it is not in the DB"
+            )
+            continue
+        ordered_group = group.sort_values(by=["DateInstalled"])
+
+        if pointid == "SO-0168":
+            print(ordered_group)
+
+        try:
+            for row in ordered_group.itertuples():
+                if row.EquipmentType not in EQUIPMENT_TO_SENSOR_TYPE_MAP:
+                    logger.critical(
+                        f"Skipping equipment with type {row.EquipmentType} for point {pointid}"
+                    )
+                    continue
+
+                sensor = (
+                    session.query(Sensor)
+                    .filter(Sensor.serial_no == row.SerialNo)
+                    .one_or_none()
+                )
+                if sensor:
+                    logger.info(
+                        f"Sensor with serial number {row.SerialNo} already exists. Only creating deployment for that record"
+                    )
+                else:
+                    sensor = Sensor(
+                        nma_pk_equipment=row.GlobalID,
+                        name=row.ID,
+                        sensor_type=EQUIPMENT_TO_SENSOR_TYPE_MAP[row.EquipmentType],
+                        model=row.Model,
+                        serial_no=row.SerialNo,
+                        owner_agency="NMBGMR",
+                        notes=row.Equipment_Notes,
+                    )
+                    session.add(sensor)
+                    logger.info(
+                        f"Added sensor {sensor.name} with serial number {sensor.serial_no}"
+                    )
+
+                installation_date = datetime.strptime(
+                    row.DateInstalled, "%Y-%m-%d %H:%M:%S.%f"
+                ).date()
+                removal_date = (
+                    datetime.strptime(row.DateRemoved, "%Y-%m-%d %H:%M:%S.%f").date()
+                    if not pd.isna(row.DateRemoved)
+                    else None
+                )
+                deployment = Deployment(
+                    thing=thing,
+                    sensor=sensor,
+                    installation_date=installation_date,
+                    removal_date=removal_date,
+                    recording_interval=int(row.RecordingInterval),
+                    recording_interval_units="hour",
+                    hanging_cable_length=row.HangingCableLength,
+                    hanging_point_height=row.HangingPointHgt,
+                    hanging_point_description=row.HangingPointDescription,
+                )
+                session.add(deployment)
+                logger.info(
+                    f"Added deployment for sensor with serial number {sensor.serial_no}, deployed to {thing.name}: | Installation Date: {installation_date} | Removal Date: {removal_date}"
+                )
+
+                """
+                Developer's notes
+
+                Since it's unclear beforehand if a sensor has been removed just update
+                the sensor_status based off of each deployments installation/removal
+                dates
+                """
+                if installation_date:
+                    sensor.sensor_status = "In Service"
+                if removal_date:
+                    sensor.sensor_status = "Retired"
+            session.commit()
+        except Exception as e:
+            logger.critical(f"Could not add sensor and deployment: {e}")
 
 
 # ============= EOF =============================================
@@ -27,3 +125,7 @@ def init_sensor(session):
     sensor.datetime_installed = datetime.now()
     session.add(sensor)
     session.commit()
+
+
+if __name__ == "__main__":
+    transfer_sensors("abc")

transfers/transfer.py

@@ -28,7 +28,7 @@
 from transfers.group_transfer import transfer_groups
 from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata
 from transfers.contact_transfer import transfer_contacts
-from transfers.sensor_transfer import init_sensor
+from transfers.sensor_transfer import transfer_sensors
 from transfers.waterlevels_transfer import transfer_water_levels
 from transfers.well_transfer import (
     transfer_wells,
@@ -54,13 +54,6 @@ def erase_and_initalize(session: Session) -> None:
     erase(session)
     lexicon()
     parameter()
-    sensor(session)
-
-
-@timeit
-def sensor(session: Session):
-    logger.info("Initializing sensors")
-    init_sensor(session)
 
 
 @timeit
@@ -125,6 +118,9 @@ def transfer_all(sess, limit=100):
     message("TRANSFERRING METEOROLOGICAL")
     timeit_direct(transfer_met, sess, limit)
 
+    message("TRANSFERRING SENSORS")
+    timeit_direct(transfer_sensors, sess)
+
     message("TRANSFERRING CONTACTS")
     timeit_direct(transfer_contacts, sess)