NO TICKET: get transfer branch in pre-production | data validations

.github/workflows/tests.yml

@@ -5,7 +5,7 @@ name: Tests
 
 on:
   pull_request:
-    branches: [ "main",'pre-production']
+    branches: [ "main",'pre-production', 'transfer']
 
 permissions:
   contents: read

Procfile

@@ -0,0 +1 @@
+web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker

main.py

@@ -1,6 +1,8 @@
 import os
 import sentry_sdk
 from dotenv import load_dotenv
+from starlette.middleware.base import BaseHTTPMiddleware
+from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
 
 load_dotenv()
 

pyproject.toml

@@ -65,7 +65,7 @@ dependencies = [
     "propcache==0.3.2",
     "proto-plus==1.26.1",
     "protobuf==6.32.0",
-    "psycopg2==2.9.10",
+    "psycopg2-binary>=2.9.10",
     "pyasn1==0.6.1",
     "pyasn1-modules==0.4.2",
     "pycparser==2.22",

services/gcs_helper.py

@@ -42,6 +42,8 @@ def get_storage_client() -> storage.Client:
 
         # Create storage client
         client = storage.Client(credentials=creds)
+    elif settings.mode == "transfer":
+        client = storage.Client()
     else:
         client = storage.Client.from_service_account_json(
             os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")

transfer_test.sh

@@ -0,0 +1,4 @@
+curl -X POST "https://nma-ocotillo-transfer-95715287188.us-central1.run.app/health" \
+-H "Authorization: bearer $(gcloud auth print-identity-token)" \
+-H "Content-Type: application/json" \
+-d '{"limit": "10"}'

transfers/asset_transfer.py

@@ -41,16 +41,21 @@ def transfer_assets(session: Session) -> None:
     bucket = get_storage_bucket(client)
     logger.info(f"Using bucket {bucket.name}")
 
+    # for name in ['AR0001']: # for testing
     for thing in get_valid_things(session):
+        name = thing.name
         # find images in temp bucket
         logger.info(f"Processing PointID: {thing.name}")
         blobs = bucket.list_blobs(prefix=f"nma-photos/{thing.name}")
         # move blobs from temp to assets bucket
         for srcblob in blobs:
             f = srcblob.download_as_bytes()
-            ff = UploadFile(file=io.BytesIO(f), filename=srcblob.name, size=len(f))
+            head, filename = srcblob.name.split("/")
+
+            ff = UploadFile(file=io.BytesIO(f), filename=filename, size=len(f))
+
             uri, blob_name = gcs_upload(ff, bucket)
-            add_asset(session, ff, srcblob.name, thing.id, uri, blob_name)
+            add_asset(session, ff, filename, thing.id, uri, blob_name)
 
 
 def transfer_assets_testing(session: Session) -> None: