CLI MVP

.github/workflows/tests.yml

@@ -67,7 +67,7 @@ jobs:
 
       - name: Run BDD tests
         run: |
-          uv run behave tests/features --tags="@backend and @production" --no-capture
+          uv run behave tests/features --tags="@backend and @production and not @skip" --no-capture
 
       - name: Upload results to Codecov
         uses: codecov/codecov-action@v4

README.md

@@ -1,4 +1,4 @@
-# NMSampleLocations
+# NMSampleLocations aka OcotilloAPI
 
 [![Code Format](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/format_code.yml/badge.svg)](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/format_code.yml)
 [![Dependabot Updates](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/dependabot/dependabot-updates)
@@ -9,7 +9,8 @@
 **Geospatial Sample Data Management System**  
 _New Mexico Bureau of Geology and Mineral Resources_
 
-NMSampleLocations is a FastAPI-based backend service designed to manage geospatial sample location data across New Mexico. It supports research, field operations, and public data delivery for the Bureau of Geology and Mineral Resources.
+OcotilloAPI is a FastAPI-based backend service designed to manage geospatial sample location data across New Mexico. It 
+supports research, field operations, and public data delivery for the Bureau of Geology and Mineral Resources.
 
 ---
 
@@ -197,4 +198,39 @@ Notes:
 - All `Update` schema fields are optional and default to `None`
 - All `Response` schema fields are defined as `<type>` if non-nullable and `<type> | None` if nullable
 - All raised exceptions should use the `PydanticStyleException` as defined in `services/exceptions_helper.py`
-- Errors handled by the database should be enumerated and handled in a database_error_handler in each router's file
+- Errors handled by the database should be enumerated and handled in a database_error_handler in each router's file---
+
+## 📦 Ocotillo CLI
+
+The `oco` command exposes project automation and bulk data utilities.
+
+```bash
+# Display available commands
+oco --help
+
+# Bulk import water level data from a CSV
+oco water-levels bulk-upload --file water_levels.csv --output json
+```
+
+The bulk upload command parses and validates each row, creates the corresponding field events/samples/observations, and prints a JSON summary (matching the API response shape) so the workflow can be automated or scripted.
+## 🧪 Testing
+
+```bash
+# Run unit tests
+pytest
+
+# Run Behave BDD specs
+behave tests/features
+```
+
+> Tests require a local Postgres/PostGIS instance. Set `POSTGRES_*` values in `.env`, run migrations, and ensure the database is reachable before running the suites.
+
+## 🔄 Data Transfers
+
+Legacy or staging datasets can be imported using the transfer utilities:
+
+```bash
+python -m transfers.transfer
+```
+
+Configure the `.env` file with the appropriate credentials before running transfers.

api/observation.py

@@ -15,8 +15,13 @@
 # ===============================================================================
 from datetime import datetime
 
-from fastapi import APIRouter, Query, Request
-from starlette.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_204_NO_CONTENT
+from fastapi import APIRouter, Query, Request, UploadFile, File, HTTPException
+from starlette.status import (
+    HTTP_200_OK,
+    HTTP_201_CREATED,
+    HTTP_204_NO_CONTENT,
+    HTTP_400_BAD_REQUEST,
+)
 
 from api.pagination import CustomPage
 from core.dependencies import (
@@ -36,6 +41,7 @@
     UpdateWaterChemistryObservation,
 )
 from schemas.transducer import TransducerObservationWithBlockResponse
+from schemas.water_level_csv import WaterLevelBulkUploadResponse
 from services.crud_helper import model_deleter, model_adder
 from services.observation_helper import (
     get_observations,
@@ -44,6 +50,7 @@
     get_transducer_observations,
 )
 from services.query_helper import simple_get_by_id
+from services.water_level_csv import bulk_upload_water_levels
 
 router = APIRouter(prefix="/observation", tags=["observation"])
 
@@ -81,6 +88,30 @@ async def add_water_chemistry_observation(
     return model_adder(session, Observation, obs_data, user=user)
 
 
+@router.post(
+    "/groundwater-level/bulk-upload",
+    response_model=WaterLevelBulkUploadResponse,
+    status_code=HTTP_200_OK,
+)
+async def bulk_upload_groundwater_levels(
+    user: amp_admin_dependency,
+    file: UploadFile = File(...),
+):
+    contents = await file.read()
+    if not contents:
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail="Uploaded file is empty",
+        )
+
+    result = bulk_upload_water_levels(contents)
+
+    if result.exit_code != 0:
+        raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=result.payload)
+
+    return result.payload
+
+
 # PATCH ========================================================================
 
 

manage.py → cli/__init__.py

@@ -13,37 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import click
-from core.initializers import init_lexicon
-
-
-# from migration.migration2 import migrate_wells, migrate_water_levels
-#
-#
-# def wells():
-#     with session_ctx() as sess:
-#         migrate_wells(sess, 1000)
-#
-#
-# def waterlevels():
-#     with session_ctx() as sess:
-#         migrate_water_levels(sess, 800)
-@click.group()
-def cli():
-    """Command line interface for managing the application."""
-    pass
-
-
-@cli.command()
-def initialize_lexicon():
-    init_lexicon()
-
-
-if __name__ == "__main__":
-    cli()
 
 # ============= EOF =============================================

cli/cli.py

@@ -0,0 +1,88 @@
+# ===============================================================================
+# Copyright 2025 ross
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===============================================================================
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import click
+
+
+@click.group()
+def cli():
+    """Command line interface for managing the application."""
+    pass
+
+
+@cli.command()
+def initialize_lexicon():
+    from core.initializers import init_lexicon
+
+    init_lexicon()
+
+
+@cli.command()
+@click.argument(
+    "root_directory",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
+)
+def associate_assets_command(root_directory: str):
+    from cli.service_adapter import associate_assets
+
+    associate_assets(root_directory)
+
+
+@cli.command()
+@click.argument(
+    "file_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+)
+def well_inventory_csv(file_path: str):
+    """
+    parse and upload a csv to database
+    """
+    # TODO: use the same helper function used by api to parse and upload a WI csv
+    from cli.service_adapter import well_inventory_csv
+
+    well_inventory_csv(file_path)
+
+
+@cli.command()
+@click.argument(
+    "file_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+)
+@click.option(
+    "--output",
+    "output_format",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Optional output format",
+)
+def water_levels_csv(file_path: str, output_format: str | None):
+    """
+    parse and upload a csv
+    """
+    # TODO: use the same helper function used by api to parse and upload a WL csv
+    from cli.service_adapter import water_levels_csv
+
+    pretty_json = (output_format or "").lower() == "json"
+    water_levels_csv(file_path, pretty_json=pretty_json)
+
+
+if __name__ == "__main__":
+    cli()
+
+# ============= EOF =============================================

cli/service_adapter.py

@@ -0,0 +1,100 @@
+# ===============================================================================
+# Copyright 2025 ross
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===============================================================================
+import csv
+import io
+import mimetypes
+import sys
+from pathlib import Path
+
+from fastapi import UploadFile
+from sqlalchemy import select
+
+from db import Thing, Asset
+from db.engine import session_ctx
+from services.asset_helper import upload_and_associate
+from services.gcs_helper import get_storage_bucket, make_blob_name_and_uri
+from services.water_level_csv import bulk_upload_water_levels
+
+
+def well_inventory_csv(source_file: Path | str):
+    if isinstance(source_file, str):
+        source_file = Path(source_file)
+
+
+def water_levels_csv(source_file: Path | str, *, pretty_json: bool = False):
+    if isinstance(source_file, str):
+        source_file = Path(source_file)
+
+    result = bulk_upload_water_levels(source_file, pretty_json=pretty_json)
+    print(result.stdout)
+    if result.stderr:
+        print(result.stderr, file=sys.stderr)
+    return result.exit_code
+
+
+def associate_assets(source_directory: Path | str) -> list[str]:
+    """
+    given a directory
+    and the directory contains a manifest file
+    and the manifest file is a 3-column csv (asset_file_name, thing_name aka pointid, asset_type)
+    and the directory contains a set of photos
+
+    then when i run the associate photos command
+    the app should save the photos to gcs
+    and associate each uploaded photo with the corresponding thing
+
+    """
+    if isinstance(source_directory, str):
+        source_directory = Path(source_directory)
+    m = source_directory / "manifest.txt"
+
+    bucket = get_storage_bucket()
+    uris = []
+    with session_ctx() as sess:
+        with open(m, "r") as rf:
+            reader = csv.DictReader(rf)
+            for row in reader:
+                # save file to gcs
+                path = row["asset_file_name"].strip()
+
+                with open(source_directory / path, "rb") as fp:
+                    file = UploadFile(
+                        io.BytesIO(fp.read()), filename=path, size=len(fp.read())
+                    )
+
+                sql = select(Thing).where(Thing.name == row["thing_name"].strip())
+                thing = sess.scalars(sql).one_or_none()
+                if thing:
+                    # get mime_type from file
+                    mime_type, encoding = mimetypes.guess_type(path)
+                    blob_name, uri = make_blob_name_and_uri(file)
+                    sql = select(Asset).where(Asset.uri == uri)
+                    existing_asset = sess.scalars(sql).one_or_none()
+                    if existing_asset:
+                        continue
+                    uri, blob_name = upload_and_associate(
+                        sess, file, bucket, thing, path, **{"mime_type": mime_type}
+                    )
+                    uris.append(uri)
+
+                else:
+                    print(f"no thing with name={row['thing_name']} found in db")
+        sess.commit()
+
+    return uris
+
+
+# ============= EOF =============================================