Persist logs and warnings | Updated sources and analytes | Non-detects and null values | Site limits | Unaccounted datums | Silica | Single timeseries file flag

.github/workflows/cicd.yml

@@ -5,7 +5,7 @@ name: CI/CD
 
 on:
   push:
-    branches: [ "main", "feature/jir"]
+    branches: [ "main", "feature/jir", "dev/jab"]
   pull_request:
     branches: [ "main"]
 

.github/workflows/format_code.yml

@@ -1,9 +1,9 @@
 name: Format code
 on:
   pull_request:
-    branches: [feature/jir,  ]
+    branches: [feature/jir]
   push:
-    branches: [feature/jir,]
+    branches: [feature/jir, dev/jab]
 jobs:
   format:
     runs-on: ubuntu-latest

.gitignore

@@ -169,3 +169,12 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# outputs
+output_timeseries
+output.combined.csv
+output.csv
+output.sites.csv
+output.timeseries.csv
+output.logs.txt
+output.warnings.txt

README.md

@@ -34,6 +34,14 @@ pip install nmuwd
 ```
 
 ## Usage
+
+### Timeseries & Summary
+The flag `--separate_timeseries_files` exports timeseries for every location in their own file in the directory output_series (e.g. AB-0002.csv, AB-0003.csv). Locations with only one observation are gathered and exported to the file `output.combined.csv.
+
+The flag `--single_timeseries_file` exports all timeseries for all locations in one file titled output.timeseries.csv. It also exports a file titled output.sites.csv that contains site information, such as latitude, longitude, and elevation.
+
+If neither of the above flags are specified, a summary table called output.csv is exported.
+
 ### Water Levels
 
 Get water levels for a county. Return a summary csv
@@ -92,6 +100,7 @@ Available analytes:
 - Nitrate
 - pH
 - Potassium
+- Silica
 - Sodium
 - Sulfate
 - TDS

backend/config.py

@@ -135,6 +135,9 @@ class Config(object):
     use_csv: bool = True
     use_geojson: bool = False
 
+    logs: list = []
+    warnings: list = []
+
     def __init__(self, model=None, payload=None):
         self.bbox = {}
         if model:
@@ -313,21 +316,33 @@ def now_ms(self, days=0):
         # return current time in milliseconds
         return int((datetime.now() - td).timestamp() * 1000)
 
-    def report(self):
+    def report(self, log_report: bool = False):
         def _report_attributes(title, attrs):
-            click.secho(
-                f"---- {title} --------------------------------------------------",
-                fg="yellow",
-            )
+            s = f"---- {title} --------------------------------------------------"
+            click.secho(s, fg="yellow")
+            if log_report:
+                self.logs.append(s)
+
             for k in attrs:
                 v = getattr(self, k)
-                click.secho(f"{k}: {v}", fg="yellow")
-            click.secho("", fg="yellow")
+                s = f"{k}: {v}"
+                click.secho(s, fg="yellow")
+
+                if log_report:
+                    self.logs.append(s)
+
+            s = ""
+            click.secho(s, fg="yellow")
+
+            if log_report:
+                self.logs.append(s)
+
+        s = "---- Begin configuration -------------------------------------\n"
+        click.secho(s, fg="yellow")
+
+        if log_report:
+            self.logs.append(s)
 
-        click.secho(
-            "---- Begin configuration -------------------------------------\n",
-            fg="yellow",
-        )
         sources = [f"use_source_{s}" for s in SOURCE_KEYS]
         attrs = [
             "start_date",
@@ -356,9 +371,11 @@ def _report_attributes(title, attrs):
             ),
         )
 
-        click.secho(
-            "---- End configuration -------------------------------------", fg="yellow"
-        )
+        s = "---- End configuration -------------------------------------\n"
+        click.secho(s, fg="yellow")
+
+        if log_report:
+            self.logs.append(s)
 
     def validate(self):
         if not self._validate_bbox():

backend/connectors/README.md

@@ -8,6 +8,8 @@ The following are necessary for adding a source:
 - the `use_source_<source>` flag needs to be added to the `analyte_sources` method in the `Config` class in **/backend/config.py** if analytes are available for that source
 - the `use_source_<source>` flag needs to be added to the `water_level_sources` method in the `Config` class in **/backend/config.py** if water levels are available for that source
 
+**IMPORTANT: add tests for the source**
+
 For the sake of discription, the example source is called Faux.
 
 # /backend/connectors/faux
@@ -30,7 +32,7 @@ The following methods need to be defined for Faux. See `BaseSiteSource` for doc
 The following methods need to be defined for Faux. See `BaseAnalyteSource` for doc strings for each of the methods:
 
 - `get_records`
-- `_extract_parent_records`
+- `_extract_site_records`
 - `_extract_parameter_units`
 - `_extract_most_recent`
 - `_extract_parameter_result`
@@ -45,7 +47,7 @@ The following method is optional:
 The following methods need to be defined for Faux. See `BaseWaterLevelSource` for doc strings for each of the methods:
 
 - `get_records`
-- `_extract_parent_records`
+- `_extract_site_records`
 - `_extract_parameter_units`
 - `_extract_most_recent`
 - `_extract_parameter_result`

backend/connectors/bor/source.py

@@ -87,9 +87,9 @@ def _extract_most_recent(self, rs):
             "units": record["attributes"]["resultAttributes"]["units"],
         }
 
-    def _extract_parent_records(self, records, parent_record):
+    def _extract_site_records(self, records, site_record):
         return [
-            ri for ri in records if ri["attributes"]["locationId"] == parent_record.id
+            ri for ri in records if ri["attributes"]["locationId"] == site_record.id
         ]
 
     def _reorder_catalog_items(self, items):
@@ -98,12 +98,10 @@ def _reorder_catalog_items(self, items):
             items = items[self._catalog_item_idx :] + items[: self._catalog_item_idx]
         return items
 
-    def get_records(self, parent_record):
+    def get_records(self, site_record):
         code = get_analyte_search_param(self.config.analyte, BOR_ANALYTE_MAPPING)
 
-        for i, item in enumerate(
-            self._reorder_catalog_items(parent_record.catalogItems)
-        ):
+        for i, item in enumerate(self._reorder_catalog_items(site_record.catalogItems)):
 
             data = self._execute_json_request(f'https://data.usbr.gov{item["id"]}')
             if not data:

backend/connectors/ckan/source.py

@@ -112,12 +112,12 @@ def _parse_response(self, resp):
 class OSERoswellWaterLevelSource(OSERoswellSource, BaseWaterLevelSource):
     transformer_klass = OSERoswellWaterLevelTransformer
 
-    def get_records(self, parent_record):
-        return self._parse_response(parent_record, self.get_response())
+    def get_records(self, site_record):
+        return self._parse_response(site_record, self.get_response())
 
-    def _parse_response(self, parent_record, resp):
+    def _parse_response(self, site_record, resp):
         records = resp.json()["result"]["records"]
-        return [record for record in records if record["Site_ID"] == parent_record.id]
+        return [record for record in records if record["Site_ID"] == site_record.id]
 
     def _extract_parameter_results(self, records):
         return [float(r["DTWGS"]) for r in records]

backend/connectors/ckan/transformer.py

@@ -48,17 +48,17 @@ def _transform(self, record):
 class OSERoswellWaterLevelTransformer(WaterLevelTransformer):
     source_tag = "CKAN/OSERoswell"
 
-    # def _transform_hook(self, record, config, parent_record):
+    # def _transform_hook(self, record, config, site_record):
     #     rec = {
-    #         "id": parent_record.id,
+    #         "id": site_record.id,
     #         "source": "CKAN/OSERoswell",
-    #         "location": parent_record.name,
-    #         "usgs_site_id": parent_record.id,
-    #         "latitude": parent_record.latitude,
-    #         "longitude": parent_record.longitude,
-    #         "elevation": parent_record.elevation,
+    #         "location": site_record.name,
+    #         "usgs_site_id": site_record.id,
+    #         "latitude": site_record.latitude,
+    #         "longitude": site_record.longitude,
+    #         "elevation": site_record.elevation,
     #         "elevation_units": "ft",
-    #         "well_depth": parent_record.well_depth,
+    #         "well_depth": site_record.well_depth,
     #         "well_depth_units": "ft",
     #     }
     #     if config.output_summary_waterlevel_stats:

backend/connectors/isc_seven_rivers/source.py

@@ -109,12 +109,12 @@ def _extract_parameter_results(self, records):
     def _extract_parameter_units(self, records):
         return [r["units"] for r in records]
 
-    def get_records(self, parent_record):
+    def get_records(self, site_record):
         config = self.config
         analyte_id = self._get_analyte_id(config.analyte)
         if analyte_id:
             params = {
-                "monitoringPointId": parent_record.id,
+                "monitoringPointId": site_record.id,
                 "analyteId": analyte_id,
                 "start": 0,
                 "end": config.now_ms(days=1),
@@ -146,9 +146,9 @@ def get_datetime(record):
 class ISCSevenRiversWaterLevelSource(BaseWaterLevelSource):
     transformer_klass = ISCSevenRiversWaterLevelTransformer
 
-    def get_records(self, parent_record):
+    def get_records(self, site_record):
         params = {
-            "id": parent_record.id,
+            "id": site_record.id,
             "start": 0,
             "end": self.config.now_ms(days=1),
         }

backend/connectors/isc_seven_rivers/transformer.py

@@ -53,14 +53,14 @@ class ISCSevenRiversAnalyteTransformer(AnalyteTransformer):
 class ISCSevenRiversWaterLevelTransformer(WaterLevelTransformer):
     source_tag = "ISCSevenRivers"
 
-    # def _transform_hook(self, record, config, parent_record):
+    # def _transform_hook(self, record, config, site_record):
     #     rec = {
     #         "source": "ISCSevenRivers",
-    #         "id": parent_record.id,
-    #         "location": parent_record.name,
-    #         "latitude": parent_record.latitude,
-    #         "longitude": parent_record.longitude,
-    #         "elevation": parent_record.elevation,
+    #         "id": site_record.id,
+    #         "location": site_record.name,
+    #         "latitude": site_record.latitude,
+    #         "longitude": site_record.longitude,
+    #         "elevation": site_record.elevation,
     #         "elevation_units": "ft",
     #     }
     #     if config.output_summary_waterlevel_stats:

backend/connectors/mappings.py

@@ -22,6 +22,7 @@
     ARSENIC,
     NITRATE,
     CALCIUM,
+    SILICA,
     SODIUM,
     POTASSIUM,
     MAGNESIUM,
@@ -31,22 +32,24 @@
 )
 
 # DWB ===============================================================================
+# the mapping below is the corresponding "@iot.id" for the ObservedProperties
 DWB_ANALYTE_MAPPING: dict = {
     ARSENIC: 3,
-    BICARBONATE: None,
-    CALCIUM: None,
+    BICARBONATE: 22,  # BICARBONATE AS HCO3
+    CALCIUM: 11,
     CARBONATE: None,
     CHLORIDE: 15,
     FLUORIDE: 19,
-    MAGNESIUM: None,
+    MAGNESIUM: 23,
     NITRATE: 35,
-    POTASSIUM: None,
-    SODIUM: None,
+    POTASSIUM: 33,
+    SILICA: 37,
+    SODIUM: 38,
     SULFATE: 41,
     TDS: 90,
     # "Uranium-238": 386,
     URANIUM: 385,  # "Combined Uranium"
-    PH: None,
+    PH: 81,
 }
 # ISC Seven Rivers ===============================================================================
 """
@@ -95,6 +98,7 @@
     MAGNESIUM: "Magnesium",
     NITRATE: "Nitrate",
     POTASSIUM: "Potassium",
+    SILICA: "SiO2",
     SODIUM: "Sodium",
     SULFATE: "Sulfate",
     TDS: "TDS calc",
@@ -137,6 +141,7 @@
     MAGNESIUM: "Magnesium",
     NITRATE: "Nitrate (as N)",
     POTASSIUM: "Potassium",
+    SILICA: "Silica",
     SODIUM: "Sodium",
     SULFATE: "Sulfate",
     TDS: "Total Dissolved Solids",
@@ -155,6 +160,7 @@
     MAGNESIUM: ["Magnesium"],
     NITRATE: ["Nitrate", "Nitrate-N", "Nitrate as N"],
     POTASSIUM: ["Potassium"],
+    SILICA: ["Silica"],
     SODIUM: ["Sodium"],
     SULFATE: [
         "Sulfate",
@@ -171,8 +177,8 @@
 """
 Temp
 DO
-ALK HCO3
-ALK CO3
+ALK HCO3    <-- this is a measure of alkalinity, and not necessarily the same as Bicarbonate
+ALK CO3     <-- this is a measure of alkalinity, and not necessarily the same as Carbonate
 ALK OH
 ALK
 P ALK
@@ -224,14 +230,15 @@
 """
 BOR_ANALYTE_MAPPING: dict = {
     ARSENIC: "As",
-    BICARBONATE: "ALK HCO3",
+    BICARBONATE: None,
     CALCIUM: "Ca",
-    CARBONATE: "ALK CO3",
+    CARBONATE: None,
     CHLORIDE: "Cl",
     FLUORIDE: "F",
     MAGNESIUM: "Mg",
     NITRATE: "NO3",
     POTASSIUM: "K",
+    SILICA: "SiO2",
     SODIUM: "Na",
     SULFATE: "SO4",
     TDS: "TDS",