Update main to latest version for Feb 5, 2025, training and workshop

.github/workflows/publish-to-pypi.yml

@@ -1,12 +1,15 @@
 name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
 
 on:
-  push:
-    tags:
-      - '*'
+  pull_request:
+    branches:
+      - main
+    types:
+      - closed
 
 jobs:
-  build-n-publish:
+  build-and-publish-if-merged:
+    if: github.event.pull_request.merged == true
     name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
     runs-on: ubuntu-latest
     permissions:
@@ -36,3 +39,5 @@ jobs:
       - name: Publish distribution 📦 to PyPI
         if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          attestations: false

.gitignore

@@ -171,10 +171,4 @@ cython_debug/
 #.idea/
 
 # outputs
-output_timeseries
-output.combined.csv
-output.csv
-output.sites.csv
-output.timeseries.csv
-output.logs.txt
-output.warnings.txt
+output*

backend/config.py

@@ -57,37 +57,37 @@
 from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource
 
 SOURCE_KEYS = (
-    "nmbgmr",
-    "wqp",
-    "iscsevenrivers",
+    "bernco",
+    "bor",
+    "nmbgmr_amp",
+    "nmed_dwb",
+    "nmose_isc_seven_rivers",
+    "nmose_roswell",
     "nwis",
-    "oseroswell",
     "pvacd",
-    "bor",
-    "dwb",
-    "bernco",
+    "wqp",
 )
 
 
 def get_source(source):
-    if source == "nmbgmr":
+    if source == "bernco":
+        return BernCoSiteSource()
+    elif source == "bor":
+        return BORSiteSource()
+    elif source == "nmbgmr_amp":
         return NMBGMRSiteSource()
-    elif source == "wqp":
-        return WQPSiteSource()
-    elif source == "iscsevenrivers":
+    elif source == "nmed_dwb":
+        return DWBSiteSource()
+    elif source == "nmose_isc_seven_rivers":
         return ISCSevenRiversSiteSource()
+    elif source == "nmose_roswell":
+        return OSERoswellSiteSource(HONDO_RESOURCE_ID)
     elif source == "nwis":
         return NWISSiteSource()
-    elif source == "oseroswell":
-        return OSERoswellSiteSource(HONDO_RESOURCE_ID)
     elif source == "pvacd":
         return PVACDSiteSource()
-    elif source == "bor":
-        return BORSiteSource()
-    elif source == "dwb":
-        return DWBSiteSource()
-    elif source == "bernco":
-        return BernCoSiteSource()
+    elif source == "wqp":
+        return WQPSiteSource()
 
     return None
 
@@ -106,27 +106,29 @@ class Config(Loggable):
     wkt: str = ""
 
     # sources
-    use_source_nmbgmr: bool = True
-    use_source_wqp: bool = True
-    use_source_iscsevenrivers: bool = True
+    use_source_bernco: bool = True
+    use_source_bor: bool = True
+    use_source_nmbgmr_amp: bool = True
+    use_source_nmed_dwb: bool = True
+    use_source_nmose_isc_seven_rivers: bool = True
+    use_source_nmose_roswell: bool = True
     use_source_nwis: bool = True
-    use_source_oseroswell: bool = True
     use_source_pvacd: bool = True
-    use_source_bor: bool = True
-    use_source_dwb: bool = True
-    use_source_bernco: bool = True
+    use_source_wqp: bool = True
 
-    analyte: str = ""
+    # parameter
+    parameter: str = ""
 
     # output
     use_cloud_storage: bool = False
-    output_dir: str = ""
+    output_dir: str = "."
     output_name: str = "output"
     output_horizontal_datum: str = WGS84
     output_elevation_units: str = FEET
     output_well_depth_units: str = FEET
     output_summary: bool = False
-    output_single_timeseries: bool = False
+    output_timeseries_unified: bool = False
+    output_timeseries_separated: bool = False
 
     latest_water_level_only: bool = False
 
@@ -157,56 +159,54 @@ def __init__(self, model=None, payload=None):
             self.wkt = payload.get("wkt", "")
             self.county = payload.get("county", "")
             self.output_summary = payload.get("output_summary", False)
+            self.output_timeseries_unified = payload.get(
+                "output_timeseries_unified", False
+            )
+            self.output_timeseries_separated = payload.get(
+                "output_timeseries_separated", False
+            )
             self.output_name = payload.get("output_name", "output")
             self.start_date = payload.get("start_date", "")
             self.end_date = payload.get("end_date", "")
-            self.analyte = payload.get("analyte", "")
-            self.output_single_timeseries = payload.get(
-                "output_single_timeseries", False
-            )
+            self.parameter = payload.get("parameter", "")
 
             for s in SOURCE_KEYS:
                 setattr(self, f"use_source_{s}", s in payload.get("sources", []))
 
     def analyte_sources(self):
         sources = []
 
-        # if self.use_source_wqp:
-        # sources.append((WQPSiteSource, WQPAnalyteSource))
         if self.use_source_bor:
             sources.append((BORSiteSource(), BORAnalyteSource()))
         if self.use_source_wqp:
             sources.append((WQPSiteSource(), WQPAnalyteSource()))
-        if self.use_source_iscsevenrivers:
+        if self.use_source_nmose_isc_seven_rivers:
             sources.append((ISCSevenRiversSiteSource(), ISCSevenRiversAnalyteSource()))
-        if self.use_source_nmbgmr:
+        if self.use_source_nmbgmr_amp:
             sources.append((NMBGMRSiteSource(), NMBGMRAnalyteSource()))
-        if self.use_source_dwb:
+        if self.use_source_nmed_dwb:
             sources.append((DWBSiteSource(), DWBAnalyteSource()))
 
         for s, ss in sources:
             s.set_config(self)
             ss.set_config(self)
 
-            # s.config = self
-            # ss.config = self
-
         return sources
 
     def water_level_sources(self):
         sources = []
-        if self.use_source_nmbgmr:
+        if self.use_source_nmbgmr_amp:
             sources.append((NMBGMRSiteSource(), NMBGMRWaterLevelSource()))
 
-        if self.use_source_iscsevenrivers:
+        if self.use_source_nmose_isc_seven_rivers:
             sources.append(
                 (ISCSevenRiversSiteSource(), ISCSevenRiversWaterLevelSource())
             )
 
         if self.use_source_nwis:
             sources.append((NWISSiteSource(), NWISWaterLevelSource()))
 
-        if self.use_source_oseroswell:
+        if self.use_source_nmose_roswell:
             sources.append(
                 (
                     OSERoswellSiteSource(HONDO_RESOURCE_ID),
@@ -227,48 +227,15 @@ def water_level_sources(self):
             )
         if self.use_source_pvacd:
             sources.append((PVACDSiteSource(), PVACDWaterLevelSource()))
-            # sources.append((EBIDSiteSource, EBIDWaterLevelSource))
         if self.use_source_bernco:
             sources.append((BernCoSiteSource(), BernCoWaterLevelSource()))
-        # if self.use_source_bor:
-        #     sources.append((BORSiteSource(), BORWaterLevelSource()))
 
         for s, ss in sources:
             s.set_config(self)
             ss.set_config(self)
 
         return sources
 
-    # def site_sources(self):
-    #     sources = [
-    #         NMBGMRSiteSource(),
-    #         WQPSiteSource(),
-    #         ISCSevenRiversSiteSource(),
-    #         NWISSiteSource(),
-    #         DWBSiteSource(),
-    #         BORSiteSource(),
-    #         PVACDSiteSource(),
-    #         EBIDSiteSource(),
-    #         OSERoswellSiteSource(HONDO_RESOURCE_ID),
-    #         OSERoswellSiteSource(FORT_SUMNER_RESOURCE_ID),
-    #         OSERoswellSiteSource(ROSWELL_RESOURCE_ID),
-    #     ]
-    #
-    #     # if self.use_source_nmbgmr:
-    #     #     sources.append(NMBGMRSiteSource)
-    #     # if self.use_source_isc_seven_rivers:
-    #     #     sources.append(ISCSevenRiversSiteSource)
-    #     # if self.use_source_ose_roswell:
-    #     #     sources.append(OSERoswellSiteSource)
-    #     # if self.use_source_nwis:
-    #     #     sources.append(USGSSiteSource)
-    #     # if self.use_source_st2:
-    #     #     sources.append(PVACDSiteSource)
-    #     #     sources.append(EBIDSiteSource)
-    #     # if self.use_source_bor:
-    #     #     sources.append(BORSiteSource)
-    #     return sources
-
     def bbox_bounding_points(self, bbox=None):
         if bbox is None:
             bbox = self.bbox
@@ -340,7 +307,7 @@ def _report_attributes(title, attrs):
             "county",
             "bbox",
             "wkt",
-            "analyte",
+            "parameter",
             "site_limit",
         ] + sources
         # inputs
@@ -353,10 +320,10 @@ def _report_attributes(title, attrs):
         _report_attributes(
             "Outputs",
             (
-                "output_dir",
-                "output_name",
+                "output_path",
                 "output_summary",
-                "output_single_timeseries",
+                "output_timeseries_unified",
+                "output_timeseries_separated",
                 "output_horizontal_datum",
                 "output_elevation_units",
             ),
@@ -415,6 +382,45 @@ def _validate_county(self):
 
         return True
 
+    def _update_output_name(self):
+        """
+        Generate a unique output name based on existing directories in the output directory.
+
+        If there are no directories with the string "output" in their name, the output name will be "output".
+
+        If there is a directory called "output", then output_name will be "output_1".
+
+        If there are directories called "output_{n}" where n is an integer, then output_name will be "output_{m+1}"
+        where m is the highest integer in the existing directories.
+        """
+        output_name = self.output_name
+
+        # find if there are already directories with the string "output" their names
+        output_names = [
+            name
+            for name in os.listdir(self.output_dir)
+            if os.path.isdir(name) and output_name in name
+        ]
+
+        if len(output_names) > 0:
+            max_count = 0
+            # find the highest number appended to directories with "output" in their name
+            counts = [
+                name.split("_")[-1]
+                for name in output_names
+                if name.split("_")[-1].isdigit()
+            ]
+            counts = [int(count) for count in counts]
+            if len(counts) > 0:
+                max_count = max(counts)
+            output_name = f"{output_name}_{max_count + 1}"
+
+        self.output_name = output_name
+
+    def _make_output_path(self):
+        if not os.path.exists(self.output_path):
+            os.mkdir(self.output_path)
+
     @property
     def start_dt(self):
         return self._extract_date(self.start_date)

backend/connectors/bor/source.py

@@ -44,6 +44,9 @@
 class BORSiteSource(BaseSiteSource):
     transformer_klass = BORSiteTransformer
 
+    def __repr__(self):
+        return "BORSiteSource"
+
     def health(self):
         try:
             self.get_records()
@@ -66,6 +69,9 @@ class BORAnalyteSource(BaseAnalyteSource):
     transformer_klass = BORAnalyteTransformer
     _catalog_item_idx = None
 
+    def __repr__(self):
+        return "BORAnalyteSource"
+
     def _extract_parameter_record(self, record):
         record[PARAMETER_VALUE] = record["attributes"]["result"]
         record[PARAMETER_UNITS] = record["attributes"]["resultAttributes"]["units"]
@@ -102,9 +108,14 @@ def _reorder_catalog_items(self, items):
         return items
 
     def get_records(self, site_record):
-        code = get_analyte_search_param(self.config.analyte, BOR_ANALYTE_MAPPING)
+        code = get_analyte_search_param(self.config.parameter, BOR_ANALYTE_MAPPING)
+
+        catalog_record_data = self._execute_json_request(
+            f"https://data.usbr.gov{site_record.catalogRecords[0]['id']}"
+        )
+        catalog_items = catalog_record_data["relationships"]["catalogItems"]["data"]
 
-        for i, item in enumerate(self._reorder_catalog_items(site_record.catalogItems)):
+        for i, item in enumerate(self._reorder_catalog_items(catalog_items)):
 
             data = self._execute_json_request(f'https://data.usbr.gov{item["id"]}')
             if not data:

backend/connectors/bor/transformer.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 # ===============================================================================
 import pprint
+import json
 
 from backend.record import SiteRecord, WaterLevelRecord, AnalyteSummaryRecord
 from backend.transformer import (
@@ -57,7 +58,6 @@ def _transform(self, record):
             "well_depth": WELL_DEPTHS.get(props["_id"]),
             "well_depth_units": "ft",
             "catalogRecords": record["relationships"]["catalogRecords"]["data"],
-            "catalogItems": record["relationships"]["catalogItems"]["data"],
         }
         return rec