diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e7ef3d18..f2a7c5aea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,6 +59,7 @@ jobs: MODE: development POSTGRES_HOST: localhost POSTGRES_PORT: 5432 + POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres DB_DRIVER: postgres # SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so diff --git a/db/engine.py b/db/engine.py index e084fac6a..bc177eb8e 100644 --- a/db/engine.py +++ b/db/engine.py @@ -15,9 +15,11 @@ # =============================================================================== import asyncio -from dotenv import load_dotenv +import getpass import os from contextlib import contextmanager + +from dotenv import load_dotenv from sqlalchemy import ( create_engine, ) @@ -108,7 +110,8 @@ def getconn(): password = os.environ.get("POSTGRES_PASSWORD", "") host = os.environ.get("POSTGRES_HOST", "localhost") port = os.environ.get("POSTGRES_PORT", "5432") - user = os.environ.get("POSTGRES_USER", "postgres") + # Default to current OS user if POSTGRES_USER not set or empty + user = os.environ.get("POSTGRES_USER", "").strip() or getpass.getuser() name = os.environ.get("POSTGRES_DB", "postgres") auth = f"{user}:{password}@" if user and password else "" diff --git a/db/geochronology.py b/db/geochronology.py index 72342f106..18cf1dab7 100644 --- a/db/geochronology.py +++ b/db/geochronology.py @@ -14,8 +14,8 @@ # limitations under the License. # =============================================================================== from db.base import AutoBaseMixin, Base, lexicon_term -from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Float, Boolean -from sqlalchemy.orm import relationship, backref, mapped_column +from sqlalchemy import Integer, Float +from sqlalchemy.orm import mapped_column class GeochronologyAge(Base, AutoBaseMixin): diff --git a/manage.py b/manage.py index 95b693058..7b9f24a1c 100644 --- a/manage.py +++ b/manage.py @@ -19,7 +19,6 @@ import click from core.initializers import init_lexicon -from db.engine import session_ctx # from migration.migration2 import migrate_wells, migrate_water_levels diff --git a/schemas/__init__.py b/schemas/__init__.py index 9fdd22198..c18cec53e 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from datetime import datetime, timezone from pydantic import BaseModel, ConfigDict, AwareDatetime +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import core_schema class ResourceNotFoundResponse(BaseModel): @@ -28,9 +31,45 @@ class BaseUpdateModel(BaseCreateModel): release_status: str | None = None +# Custom type for UTC datetime serialization +class UTCAwareDatetime(AwareDatetime): + """Custom datetime type that always serializes to UTC with 'Z' suffix.""" + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type, handler + ) -> core_schema.CoreSchema: + def serialize_dt(value: datetime) -> str: + """Serialize datetime to UTC format with Z suffix.""" + if value is None: + return None + # Convert to UTC if not already + if value.tzinfo != timezone.utc: + value = value.astimezone(timezone.utc) + # Format with Z suffix + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + # Use generate_schema instead of calling handler directly + python_schema = handler.generate_schema(datetime) + return core_schema.no_info_after_validator_function( + lambda x: x, + python_schema, + serialization=core_schema.plain_serializer_function_ser_schema( + serialize_dt, + return_schema=core_schema.str_schema(), + ), + ) + + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: core_schema.CoreSchema, handler + ) -> JsonSchemaValue: + return handler(core_schema) + + class BaseResponseModel(BaseModel): id: int # every ORM model should have an id field - created_at: AwareDatetime + created_at: UTCAwareDatetime release_status: str model_config = ConfigDict( diff --git a/schemas/lexicon.py b/schemas/lexicon.py index a4d37c5a8..00f624378 100644 --- a/schemas/lexicon.py +++ b/schemas/lexicon.py @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from pydantic import BaseModel, ConfigDict, AwareDatetime +from pydantic import BaseModel, ConfigDict from typing import List +from schemas import UTCAwareDatetime # -------- CREATE ---------- @@ -74,7 +75,7 @@ class UpdateLexiconTriple(BaseModel): class BaseLexiconResponse(BaseModel): id: int - created_at: AwareDatetime + created_at: UTCAwareDatetime model_config = ConfigDict( from_attributes=True, diff --git a/schemas/observation.py b/schemas/observation.py index 581c16cba..c8a06994f 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -24,7 +24,12 @@ from typing import Annotated from typing_extensions import Self -from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel +from schemas import ( + BaseCreateModel, + BaseUpdateModel, + BaseResponseModel, + UTCAwareDatetime, +) from schemas.parameter import ParameterResponse @@ -103,7 +108,7 @@ class UpdateWaterChemistryObservation(UpdateBaseObservation): class BaseObservationResponse(BaseResponseModel): sample_id: int sensor_id: int | None - observation_datetime: AwareDatetime + observation_datetime: UTCAwareDatetime parameter: ParameterResponse release_status: str value: float | None diff --git a/schemas/sample.py b/schemas/sample.py index 43c364beb..b928af7e4 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -24,7 +24,12 @@ from typing import Annotated from typing_extensions import Self -from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel +from schemas import ( + BaseCreateModel, + BaseUpdateModel, + BaseResponseModel, + UTCAwareDatetime, +) from schemas.thing import ThingResponse from schemas.field import FieldEventResponse, FieldActivityResponse from schemas.contact import ContactResponse @@ -124,7 +129,7 @@ class SampleResponse(BaseResponseModel): field_event: FieldEventResponse field_activity: FieldActivityResponse contact: ContactResponse - sample_date: Annotated[AwareDatetime, PastDatetime()] + sample_date: UTCAwareDatetime sample_name: str sample_matrix: str sample_method: str diff --git a/services/validation/chemistry.py b/services/validation/chemistry.py index 8f107c6d2..ba8756322 100644 --- a/services/validation/chemistry.py +++ b/services/validation/chemistry.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from db.engine import database_sessionmaker -from db.lexicon import Lexicon - # from schemas.create.chemistry import CreateWaterChemistryAnalysis -from services.validation import get_category # async def validate_analyte(analysis_data: CreateWaterChemistryAnalysis): diff --git a/tests/__init__.py b/tests/__init__.py index 02566806e..f341d6c89 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,7 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import os + +# Load .env file BEFORE importing anything else +# Use override=True to override conflicting shell environment variables +from dotenv import load_dotenv + +load_dotenv(override=True) + +# Set timezone to UTC for consistent datetime handling in tests +os.environ["TZ"] = "UTC" + +# Also set time.tzset() to apply the timezone change +import time + +time.tzset() + from fastapi.testclient import TestClient +from sqlalchemy import text from core.initializers import init_lexicon, init_parameter from db import Base, Parameter @@ -21,7 +38,15 @@ from main import app -Base.metadata.drop_all(engine) +# Force clean database state by dropping and recreating schema +# This ensures test isolation similar to Docker environment +with engine.connect() as conn: + # Drop all tables with CASCADE to handle foreign key dependencies + conn.execute(text("DROP SCHEMA IF EXISTS public CASCADE")) + conn.execute(text("CREATE SCHEMA public")) + conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + conn.commit() + Base.metadata.create_all(engine) init_lexicon() diff --git a/tests/test_asset.py b/tests/test_asset.py index 094bbf0fa..1e0859843 100644 --- a/tests/test_asset.py +++ b/tests/test_asset.py @@ -20,6 +20,7 @@ from tests import client, cleanup_post_test, override_authentication, cleanup_patch_test import pytest +from datetime import timezone from unittest.mock import patch # CLASSES, FIXTURES, AND FUNCTIONS ============================================= @@ -147,9 +148,9 @@ def test_get_assets(asset, asset_with_associated_thing): data = response.json() assert data["total"] == 2 assert data["items"][0]["id"] == asset.id - assert data["items"][0]["created_at"] == asset.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == asset.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == asset.release_status assert data["items"][0]["name"] == asset.name assert data["items"][0]["label"] == asset.label @@ -163,7 +164,9 @@ def test_get_assets(asset, asset_with_associated_thing): assert data["items"][1]["id"] == asset_with_associated_thing.id assert data["items"][1][ "created_at" - ] == asset_with_associated_thing.created_at.isoformat().replace("+00:00", "Z") + ] == asset_with_associated_thing.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert ( data["items"][1]["release_status"] == asset_with_associated_thing.release_status ) @@ -199,7 +202,9 @@ def test_get_asset_by_id(asset): assert response.status_code == 200 data = response.json() assert data["id"] == asset.id - assert data["created_at"] == asset.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == asset.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["release_status"] == asset.release_status assert data["name"] == asset.name assert data["label"] == asset.label diff --git a/tests/test_contact.py b/tests/test_contact.py index 5959e48a1..bfa2c8781 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -9,6 +9,7 @@ from schemas.contact import ValidateEmail, ValidatePhone, ValidateContact import pytest +from datetime import timezone from pydantic import ValidationError import re @@ -365,9 +366,9 @@ def test_get_contacts(contact, email, address, phone): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == contact.id - assert data["items"][0]["created_at"] == contact.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == contact.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["name"] == contact.name assert data["items"][0]["role"] == contact.role assert data["items"][0]["contact_type"] == contact.contact_type @@ -376,9 +377,9 @@ def test_get_contacts(contact, email, address, phone): assert len(data["items"][0]["emails"]) == 1 assert data["items"][0]["emails"][0]["id"] == email.id - assert data["items"][0]["emails"][0][ - "created_at" - ] == email.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["emails"][0]["created_at"] == email.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["emails"][0]["contact_id"] == email.contact_id assert data["items"][0]["emails"][0]["email"] == email.email assert data["items"][0]["emails"][0]["email_type"] == email.email_type @@ -386,9 +387,9 @@ def test_get_contacts(contact, email, address, phone): assert len(data["items"][0]["phones"]) == 1 assert data["items"][0]["phones"][0]["id"] == phone.id - assert data["items"][0]["phones"][0][ - "created_at" - ] == phone.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["phones"][0]["created_at"] == phone.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["phones"][0]["contact_id"] == phone.contact_id assert data["items"][0]["phones"][0]["phone_number"] == phone.phone_number assert data["items"][0]["phones"][0]["phone_type"] == phone.phone_type @@ -398,7 +399,7 @@ def test_get_contacts(contact, email, address, phone): assert data["items"][0]["addresses"][0]["id"] == address.id assert data["items"][0]["addresses"][0][ "created_at" - ] == address.created_at.isoformat().replace("+00:00", "Z") + ] == address.created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["addresses"][0]["contact_id"] == address.contact_id assert data["items"][0]["addresses"][0]["address_line_1"] == address.address_line_1 assert data["items"][0]["addresses"][0]["address_line_2"] == address.address_line_2 @@ -424,7 +425,9 @@ def test_get_contact_by_id(contact, email, address, phone): assert response.status_code == 200 data = response.json() assert data["id"] == contact.id - assert data["created_at"] == contact.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == contact.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["name"] == contact.name assert data["role"] == contact.role assert data["contact_type"] == contact.contact_type @@ -433,9 +436,9 @@ def test_get_contact_by_id(contact, email, address, phone): assert len(data["emails"]) == 1 assert data["emails"][0]["id"] == email.id - assert data["emails"][0]["created_at"] == email.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["emails"][0]["created_at"] == email.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["emails"][0]["contact_id"] == email.contact_id assert data["emails"][0]["email"] == email.email assert data["emails"][0]["email_type"] == email.email_type @@ -443,9 +446,9 @@ def test_get_contact_by_id(contact, email, address, phone): assert len(data["phones"]) == 1 assert data["phones"][0]["id"] == phone.id - assert data["phones"][0]["created_at"] == phone.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["phones"][0]["created_at"] == phone.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["phones"][0]["contact_id"] == phone.contact_id assert data["phones"][0]["phone_number"] == phone.phone_number assert data["phones"][0]["phone_type"] == phone.phone_type @@ -453,9 +456,9 @@ def test_get_contact_by_id(contact, email, address, phone): assert len(data["addresses"]) == 1 assert data["addresses"][0]["id"] == address.id - assert data["addresses"][0]["created_at"] == address.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["addresses"][0]["created_at"] == address.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["addresses"][0]["contact_id"] == address.contact_id assert data["addresses"][0]["address_line_1"] == address.address_line_1 assert data["addresses"][0]["address_line_2"] == address.address_line_2 @@ -481,9 +484,9 @@ def test_get_contact_emails(contact, email): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == email.id - assert data["items"][0]["created_at"] == email.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == email.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == email.contact_id assert data["items"][0]["email"] == email.email assert data["items"][0]["email_type"] == email.email_type @@ -504,9 +507,9 @@ def test_get_contact_phones(contact, phone): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == phone.id - assert data["items"][0]["created_at"] == phone.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == phone.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == phone.contact_id assert data["items"][0]["phone_number"] == phone.phone_number assert data["items"][0]["phone_type"] == phone.phone_type @@ -527,9 +530,9 @@ def test_get_contact_addresses(contact, address): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == address.id - assert data["items"][0]["created_at"] == address.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == address.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == address.contact_id assert data["items"][0]["address_line_1"] == address.address_line_1 assert data["items"][0]["address_line_2"] == address.address_line_2 @@ -555,9 +558,9 @@ def test_get_emails(email): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == email.id - assert data["items"][0]["created_at"] == email.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == email.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == email.contact_id assert data["items"][0]["email"] == email.email assert data["items"][0]["email_type"] == email.email_type @@ -569,7 +572,9 @@ def test_get_email_by_id(email): assert response.status_code == 200 data = response.json() assert data["id"] == email.id - assert data["created_at"] == email.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == email.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["contact_id"] == email.contact_id assert data["email"] == email.email assert data["email_type"] == email.email_type @@ -590,9 +595,9 @@ def test_get_phones(phone): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == phone.id - assert data["items"][0]["created_at"] == phone.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == phone.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == phone.contact_id assert data["items"][0]["phone_number"] == phone.phone_number assert data["items"][0]["phone_type"] == phone.phone_type @@ -604,7 +609,9 @@ def test_get_phone_by_id(phone): assert response.status_code == 200 data = response.json() assert data["id"] == phone.id - assert data["created_at"] == phone.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == phone.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["contact_id"] == phone.contact_id assert data["phone_number"] == phone.phone_number assert data["phone_type"] == phone.phone_type @@ -625,9 +632,9 @@ def test_get_addresses(address): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == address.id - assert data["items"][0]["created_at"] == address.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == address.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["contact_id"] == address.contact_id assert data["items"][0]["address_line_1"] == address.address_line_1 assert data["items"][0]["address_line_2"] == address.address_line_2 @@ -644,7 +651,9 @@ def test_get_address_by_id(address): assert response.status_code == 200 data = response.json() assert data["id"] == address.id - assert data["created_at"] == address.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == address.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["contact_id"] == address.contact_id assert data["address_line_1"] == address.address_line_1 assert data["address_line_2"] == address.address_line_2 diff --git a/tests/test_group.py b/tests/test_group.py index 2b5be0d0b..1591a7255 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1,6 +1,7 @@ from geoalchemy2.shape import to_shape from pydantic import ValidationError import pytest +from datetime import timezone from db import Group from core.dependencies import admin_function, viewer_function, editor_function @@ -87,9 +88,9 @@ def test_get_groups(group): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == group.id - assert data["items"][0]["created_at"] == group.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == group.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == group.release_status assert data["items"][0]["name"] == group.name assert data["items"][0]["project_area"] == to_shape(group.project_area).wkt @@ -102,7 +103,9 @@ def test_get_group_by_id(group): assert response.status_code == 200 data = response.json() assert data["id"] == group.id - assert data["created_at"] == group.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == group.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["name"] == group.name assert data["project_area"] == to_shape(group.project_area).wkt assert data["description"] == group.description diff --git a/tests/test_lexicon.py b/tests/test_lexicon.py index 435c9066e..cd2fca0a9 100644 --- a/tests/test_lexicon.py +++ b/tests/test_lexicon.py @@ -24,6 +24,7 @@ from main import app import pytest +from datetime import timezone @pytest.fixture(scope="module", autouse=True) @@ -342,16 +343,16 @@ def test_get_lexicon_term_by_id(lexicon_term): assert response.status_code == 200 data = response.json() assert data["id"] == lexicon_term.id - assert data["created_at"] == lexicon_term.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == lexicon_term.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["term"] == lexicon_term.term assert data["definition"] == lexicon_term.definition assert len(data["categories"]) == 1 assert data["categories"][0]["id"] == lexicon_term.categories[0].id assert data["categories"][0]["created_at"] == lexicon_term.categories[ 0 - ].created_at.isoformat().replace("+00:00", "Z") + ].created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["categories"][0]["name"] == lexicon_term.categories[0].name assert ( data["categories"][0]["description"] == lexicon_term.categories[0].description @@ -398,9 +399,9 @@ def test_get_lexicon_category_by_id(lexicon_category): assert response.status_code == 200 data = response.json() assert data["id"] == lexicon_category.id - assert data["created_at"] == lexicon_category.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == lexicon_category.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["name"] == lexicon_category.name assert data["description"] == lexicon_category.description @@ -419,9 +420,9 @@ def test_get_lexicon_triples(lexicon_triple): data = response.json() assert data["total"] > 0 assert data["items"][0]["id"] == lexicon_triple.id - assert data["items"][0][ - "created_at" - ] == lexicon_triple.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["created_at"] == lexicon_triple.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["subject"] == lexicon_triple.subject assert data["items"][0]["predicate"] == lexicon_triple.predicate assert data["items"][0]["object_"] == lexicon_triple.object_ @@ -432,9 +433,9 @@ def test_get_lexicon_triple_by_id(lexicon_triple): assert response.status_code == 200 data = response.json() assert data["id"] == lexicon_triple.id - assert data["created_at"] == lexicon_triple.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == lexicon_triple.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["subject"] == lexicon_triple.subject assert data["predicate"] == lexicon_triple.predicate assert data["object_"] == lexicon_triple.object_ diff --git a/tests/test_location.py b/tests/test_location.py index 628c1c352..8feff3a99 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -15,6 +15,7 @@ # =============================================================================== from geoalchemy2.shape import to_shape import pytest +from datetime import timezone from core.dependencies import admin_function, editor_function, viewer_function from db import Location @@ -140,9 +141,9 @@ def test_get_locations(location): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == location.id - assert data["items"][0]["created_at"] == location.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == location.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") # assert data["items"][0]["name"] == location.name assert data["items"][0]["notes"] == location.notes assert data["items"][0]["point"] == to_shape(location.point).wkt @@ -162,7 +163,9 @@ def test_get_location_by_id(location): assert response.status_code == 200 data = response.json() assert data["id"] == location.id - assert data["created_at"] == location.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == location.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) # assert data["name"] == location.name assert data["point"] == to_shape(location.point).wkt assert data["elevation"] == location.elevation diff --git a/tests/test_observation.py b/tests/test_observation.py index 853fe9a05..ce9aa1ad0 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import pytest +from datetime import timezone + from db import Observation from core.dependencies import ( amp_admin_function, @@ -30,7 +33,6 @@ groundwater_level_parameter_id, pH_parameter_id, ) -import pytest @pytest.fixture(scope="module", autouse=True) @@ -238,7 +240,11 @@ def test_get_observation_by_id( data = response.json() assert data["id"] == obs.id - assert data["created_at"] == obs.created_at.isoformat().replace("+00:00", "Z") + # Convert created_at to UTC and format with Z suffix + expected_created_at = obs.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + assert data["created_at"] == expected_created_at assert data["release_status"] == obs.release_status if obs.parameter.id == groundwater_level_parameter_id: assert data["depth_to_water_bgs"] == obs.value - obs.measuring_point_height @@ -262,9 +268,11 @@ def test_get_groundwater_level_observations(groundwater_level_observation): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == groundwater_level_observation.id - assert data["items"][0][ - "created_at" - ] == groundwater_level_observation.created_at.isoformat().replace("+00:00", "Z") + # Convert created_at to UTC and format with Z suffix + expected_created_at = groundwater_level_observation.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + assert data["items"][0]["created_at"] == expected_created_at assert data["items"][0]["sample_id"] == groundwater_level_observation.sample_id assert data["items"][0]["sensor_id"] == groundwater_level_observation.sensor_id assert ( @@ -300,9 +308,11 @@ def test_get_groundwater_level_observation_by_id(groundwater_level_observation): assert response.status_code == 200 data = response.json() assert data["id"] == groundwater_level_observation.id - assert data[ - "created_at" - ] == groundwater_level_observation.created_at.isoformat().replace("+00:00", "Z") + # Convert created_at to UTC and format with Z suffix + expected_created_at = groundwater_level_observation.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + assert data["created_at"] == expected_created_at assert data["sample_id"] == groundwater_level_observation.sample_id assert data["sensor_id"] == groundwater_level_observation.sensor_id assert ( @@ -442,9 +452,11 @@ def test_get_water_chemistry_observations(water_chemistry_observation): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == water_chemistry_observation.id - assert data["items"][0][ - "created_at" - ] == water_chemistry_observation.created_at.isoformat().replace("+00:00", "Z") + # Convert created_at to UTC and format with Z suffix + expected_created_at = water_chemistry_observation.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + assert data["items"][0]["created_at"] == expected_created_at assert ( data["items"][0]["release_status"] == water_chemistry_observation.release_status ) @@ -466,9 +478,11 @@ def test_get_water_chemistry_observation_by_id(water_chemistry_observation): assert response.status_code == 200 data = response.json() assert data["id"] == water_chemistry_observation.id - assert data[ - "created_at" - ] == water_chemistry_observation.created_at.isoformat().replace("+00:00", "Z") + # Convert created_at to UTC and format with Z suffix + expected_created_at = water_chemistry_observation.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + assert data["created_at"] == expected_created_at assert data["release_status"] == water_chemistry_observation.release_status assert data["sample_id"] == water_chemistry_observation.sample_id assert data["sensor_id"] == water_chemistry_observation.sensor_id diff --git a/tests/test_people.py b/tests/test_people.py index 059d46166..34ef95c63 100644 --- a/tests/test_people.py +++ b/tests/test_people.py @@ -13,7 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from tests import client - - # ============= EOF ============================================= diff --git a/tests/test_sample.py b/tests/test_sample.py index c7fb83ee2..6c817f1fe 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import pytest +from datetime import timezone from pydantic import ValidationError from main import app @@ -328,9 +329,11 @@ def test_get_sample_by_id( assert response.status_code == 200 data = response.json() assert data["id"] == water_chemistry_sample.id - assert data["created_at"] == water_chemistry_sample.created_at.isoformat().replace( - "+00:00", "Z" - ) + # Convert created_at to UTC and format with Z suffix + expected_created_at = water_chemistry_sample.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + assert data["created_at"] == expected_created_at assert data["thing"]["id"] == water_well_thing.id assert data["field_event"]["id"] == field_event.id assert data["field_activity"]["id"] == water_chemistry_field_activity.id diff --git a/tests/test_sensor.py b/tests/test_sensor.py index 12f699b50..294cda201 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -27,6 +27,7 @@ ) import pytest +from datetime import timezone # from pydantic import ValidationError @@ -168,9 +169,9 @@ def test_get_sensors(sensor): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == sensor.id - assert data["items"][0]["created_at"] == sensor.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == sensor.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == sensor.release_status assert data["items"][0]["name"] == sensor.name assert data["items"][0]["sensor_type"] == sensor.sensor_type @@ -192,9 +193,9 @@ def test_get_sensors_by_thing_id( data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == sensor.id - assert data["items"][0]["created_at"] == sensor.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == sensor.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == sensor.release_status assert data["items"][0]["name"] == sensor.name assert data["items"][0]["sensor_type"] == sensor.sensor_type @@ -212,9 +213,9 @@ def test_get_sensors_by_parameter_id(sensor, groundwater_level_observation): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == sensor.id - assert data["items"][0]["created_at"] == sensor.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == sensor.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == sensor.release_status assert data["items"][0]["name"] == sensor.name assert data["items"][0]["sensor_type"] == sensor.sensor_type @@ -231,7 +232,9 @@ def test_get_sensor_by_id(sensor): assert response.status_code == 200 data = response.json() assert data["id"] == sensor.id - assert data["created_at"] == sensor.created_at.isoformat().replace("+00:00", "Z") + assert data["created_at"] == sensor.created_at.astimezone(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) assert data["release_status"] == sensor.release_status assert data["name"] == sensor.name assert data["sensor_type"] == sensor.sensor_type diff --git a/tests/test_series.py b/tests/test_series.py index 0dbcd31db..0cf48e9a8 100644 --- a/tests/test_series.py +++ b/tests/test_series.py @@ -13,13 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import datetime - import pytest -from sqlalchemy import func, select, cast, Interval -from sqlalchemy_utils.types.range import intervals - -from db.engine import get_db_session # from db.timeseries import GroundwaterLevelObservation from tests import client diff --git a/tests/test_thing.py b/tests/test_thing.py index f94defc37..82d0c9d45 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import pytest +from datetime import timezone from db import Thing, WellScreen, ThingIdLink from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test @@ -111,9 +112,7 @@ def test_add_water_well(location, group): assert data["well_casing_materials"] == payload["well_casing_materials"] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location cleanup_post_test(Thing, data["id"]) @@ -187,9 +186,7 @@ def test_add_spring(location, group): assert data["spring_type"] == payload["spring_type"] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location cleanup_post_test(Thing, data["id"]) @@ -364,9 +361,9 @@ def test_get_water_wells(water_well_thing, location): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == water_well_thing.id - assert data["items"][0][ - "created_at" - ] == water_well_thing.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["created_at"] == water_well_thing.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["name"] == water_well_thing.name assert ( data["items"][0]["first_visit_date"] @@ -397,9 +394,7 @@ def test_get_water_wells(water_well_thing, location): ] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["items"][0]["current_location"] == expected_location @@ -408,9 +403,9 @@ def test_get_water_well_by_id(water_well_thing, location): assert response.status_code == 200 data = response.json() assert data["id"] == water_well_thing.id - assert data["created_at"] == water_well_thing.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == water_well_thing.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["name"] == water_well_thing.name assert data["first_visit_date"] == water_well_thing.first_visit_date.isoformat() assert data["thing_type"] == water_well_thing.thing_type @@ -430,9 +425,7 @@ def test_get_water_well_by_id(water_well_thing, location): ] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location @@ -464,9 +457,9 @@ def test_get_springs(spring_thing, location): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == spring_thing.id - assert data["items"][0][ - "created_at" - ] == spring_thing.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["created_at"] == spring_thing.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["name"] == spring_thing.name assert ( data["items"][0]["first_visit_date"] @@ -476,9 +469,7 @@ def test_get_springs(spring_thing, location): assert data["items"][0]["release_status"] == spring_thing.release_status assert data["items"][0]["spring_type"] == spring_thing.spring_type expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["items"][0]["current_location"] == expected_location @@ -487,18 +478,16 @@ def test_get_spring_by_id(spring_thing, location): assert response.status_code == 200 data = response.json() assert data["id"] == spring_thing.id - assert data["created_at"] == spring_thing.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == spring_thing.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["name"] == spring_thing.name assert data["first_visit_date"] == spring_thing.first_visit_date.isoformat() assert data["thing_type"] == spring_thing.thing_type assert data["release_status"] == spring_thing.release_status assert data["spring_type"] == spring_thing.spring_type expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location @@ -536,9 +525,9 @@ def test_get_well_screens(well_screen): assert data["items"][0]["screen_type"] == well_screen.screen_type assert data["items"][0]["screen_description"] == well_screen.screen_description assert data["items"][0]["release_status"] == well_screen.release_status - assert data["items"][0]["created_at"] == well_screen.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["items"][0]["created_at"] == well_screen.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") def test_get_well_screen_by_id(well_screen): @@ -552,9 +541,9 @@ def test_get_well_screen_by_id(well_screen): assert data["screen_type"] == well_screen.screen_type assert data["screen_description"] == well_screen.screen_description assert data["release_status"] == well_screen.release_status - assert data["created_at"] == well_screen.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == well_screen.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") def test_get_well_screen_by_id_404_not_found(well_screen): @@ -607,9 +596,9 @@ def test_get_thing_id_links(thing_id_link): data = response.json() assert data["total"] == 1 assert data["items"][0]["id"] == thing_id_link.id - assert data["items"][0][ - "created_at" - ] == thing_id_link.created_at.isoformat().replace("+00:00", "Z") + assert data["items"][0]["created_at"] == thing_id_link.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["items"][0]["release_status"] == thing_id_link.release_status assert data["items"][0]["thing_id"] == thing_id_link.thing_id assert data["items"][0]["relation"] == thing_id_link.relation @@ -625,9 +614,9 @@ def test_get_thing_id_link_by_id(thing_id_link): assert response.status_code == 200 data = response.json() assert data["id"] == thing_id_link.id - assert data["created_at"] == thing_id_link.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == thing_id_link.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["release_status"] == thing_id_link.release_status assert data["thing_id"] == thing_id_link.thing_id assert data["relation"] == thing_id_link.relation @@ -672,9 +661,7 @@ def test_get_things(water_well_thing, spring_thing, location): assert response.status_code == 200 expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime data = response.json() assert data["total"] == 2 @@ -686,9 +673,9 @@ def test_get_thing_by_id(water_well_thing, location): data = response.json() assert data["id"] == water_well_thing.id - assert data["created_at"] == water_well_thing.created_at.isoformat().replace( - "+00:00", "Z" - ) + assert data["created_at"] == water_well_thing.created_at.astimezone( + timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") assert data["release_status"] == water_well_thing.release_status assert data["name"] == water_well_thing.name assert data["first_visit_date"] == water_well_thing.first_visit_date.isoformat() @@ -704,9 +691,7 @@ def test_get_thing_by_id(water_well_thing, location): assert data["spring_type"] is None expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location @@ -833,9 +818,7 @@ def test_patch_water_well(water_well_thing, location): assert data["well_construction_notes"] == payload["well_construction_notes"] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location cleanup_patch_test(Thing, payload, water_well_thing) @@ -895,9 +878,7 @@ def test_patch_spring(spring_thing, location): assert data["spring_type"] == payload["spring_type"] expected_location = LocationResponse.model_validate(location).model_dump() - expected_location["created_at"] = ( - expected_location["created_at"].isoformat().replace("+00:00", "Z") - ) + # created_at is already serialized to UTC format by UTCAwareDatetime assert data["current_location"] == expected_location cleanup_patch_test(Thing, payload, spring_thing) diff --git a/transfers/logger.py b/transfers/logger.py index 2924ab646..aab12079b 100644 --- a/transfers/logger.py +++ b/transfers/logger.py @@ -14,7 +14,6 @@ # limitations under the License. # =============================================================================== import logging -import sys from datetime import datetime from services.gcs_helper import get_storage_bucket diff --git a/transfers/transfer.py b/transfers/transfer.py index 77dd29ef4..ecef2a361 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -20,6 +20,7 @@ load_dotenv() +from sqlalchemy import text from sqlalchemy.orm import Session from core.initializers import init_lexicon, init_parameter from db import Base @@ -71,8 +72,6 @@ def parameter(): @timeit def erase(session: Session): logger.info("Erasing existing data") - from sqlalchemy import text - with session.bind.connect() as conn: conn.execute(text("DROP SCHEMA public CASCADE")) conn.execute(text("CREATE SCHEMA public"))