diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 414203128..c39141b2a 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -19,9 +19,19 @@ WORKDIR /app # copy the full project source COPY . . -# install dependencies using uv +# Define an optional build argument +ARG INSTALL_DEV=false +ENV INSTALL_DEV=${INSTALL_DEV} + +# Install dependencies using uv (conditionally include dev dependencies) ENV UV_PROJECT_ENVIRONMENT="/usr/local/" -RUN uv sync --locked +RUN if [ "$INSTALL_DEV" = "true" ]; then \ + echo "Installing all groups (including dev)..." && \ + uv sync --locked --all-groups; \ + else \ + echo "Installing only production dependencies..." && \ + uv sync --locked; \ + fi # expose FastAPI's default dev port EXPOSE 8000 diff --git a/pyproject.toml b/pyproject.toml index a190da361..b2f625e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,99 +5,99 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.13" dependencies = [ - "aiofiles==24.1.0", - "aiohappyeyeballs==2.6.1", - "aiohttp==3.12.15", - "aiosignal==1.4.0", - "aiosqlite==0.21.0", - "alembic==1.17.0", - "annotated-types==0.7.0", - "anyio==4.10.0", - "asgiref==3.9.1", - "asn1crypto==1.5.1", - "asyncpg==0.30.0", - "attrs==25.4.0", - "authlib>=1.6.0", - "bcrypt==4.3.0", - "cachetools==5.5.2", - "certifi==2025.8.3", - "cffi==1.17.1", - "charset-normalizer==3.4.3", - "click==8.3.0", - "cloud-sql-python-connector==1.18.4", - "cryptography==45.0.6", - "dnspython==2.7.0", - "dotenv>=0.9.9", - "email-validator==2.2.0", - "fastapi==0.116.1", - "fastapi-pagination==0.14.3", - "frozenlist==1.7.0", - "geoalchemy2==0.18.0", - "google-api-core==2.25.1", - "google-auth==2.41.1", - "google-cloud-core==2.4.3", - "google-cloud-storage==3.3.0", - "google-crc32c==1.7.1", - "google-resumable-media==2.7.2", - "googleapis-common-protos==1.70.0", - "greenlet==3.2.4", - "gunicorn==23.0.0", - "h11==0.16.0", - "httpcore==1.0.9", - "httpx==0.28.1", - "idna==3.10", - "iniconfig==2.1.0", - "itsdangerous>=2.2.0", - "jinja2>=3.1.6", - "mako==1.3.10", - "markupsafe==3.0.2", - "multidict==6.6.3", - "numpy==2.3.3", - "packaging==25.0", - "pandas==2.3.2", - "pandas-stubs==2.3.0.250703", - "pg8000==1.31.4", - "phonenumbers==9.0.13", - "pillow==11.3.0", - "pluggy==1.6.0", - "pre-commit==4.3.0", - "propcache==0.3.2", - "proto-plus==1.26.1", - "protobuf==6.32.1", - "psycopg2-binary>=2.9.10", - "pyasn1==0.6.1", - "pyasn1-modules==0.4.2", - "pycparser==2.23", - "pydantic==2.11.7", - "pydantic-core==2.33.2", - "pygments==2.19.2", - "pyjwt==2.10.1", - "pyproj==3.7.2", - "pyshp==2.3.1", - "pytest==8.4.1", - "pytest-cov>=6.2.1", - "python-dateutil==2.9.0.post0", - "python-jose>=3.5.0", - "python-multipart==0.0.20", - "pytz==2025.2", - "requests==2.32.5", - "rsa==4.9.1", - "scramp==1.4.6", - "sentry-sdk[fastapi]>=2.35.0", - "shapely==2.1.1", - "six==1.17.0", - "sniffio==1.3.1", - "sqlalchemy==2.0.43", - "sqlalchemy-continuum==1.4.2", - "sqlalchemy-searchable==2.1.0", - "sqlalchemy-utils==0.42.0", - "starlette==0.47.3", - "typing-extensions==4.15.0", - "typing-inspection==0.4.1", - "tzdata==2025.2", - "urllib3==2.5.0", - "uvicorn==0.38.0", - "yarl==1.20.1", + "aiofiles==24.1.0", + "aiohappyeyeballs==2.6.1", + "aiohttp==3.12.15", + "aiosignal==1.4.0", + "aiosqlite==0.21.0", + "alembic==1.17.0", + "annotated-types==0.7.0", + "anyio==4.10.0", + "asgiref==3.9.1", + "asn1crypto==1.5.1", + "asyncpg==0.30.0", + "attrs==25.4.0", + "authlib>=1.6.0", + "bcrypt==4.3.0", + "cachetools==5.5.2", + "certifi==2025.8.3", + "cffi==1.17.1", + "charset-normalizer==3.4.3", + "click==8.3.0", + "cloud-sql-python-connector==1.18.4", + "cryptography==45.0.6", + "dnspython==2.7.0", + "dotenv>=0.9.9", + "email-validator==2.2.0", + "fastapi==0.116.1", + "fastapi-pagination==0.14.3", + "frozenlist==1.7.0", + "geoalchemy2==0.18.0", + "google-api-core==2.25.1", + "google-auth==2.41.1", + "google-cloud-core==2.4.3", + "google-cloud-storage==3.3.0", + "google-crc32c==1.7.1", + "google-resumable-media==2.7.2", + "googleapis-common-protos==1.70.0", + "greenlet==3.2.4", + "gunicorn==23.0.0", + "h11==0.16.0", + "httpcore==1.0.9", + "httpx==0.28.1", + "idna==3.10", + "iniconfig==2.1.0", + "itsdangerous>=2.2.0", + "jinja2>=3.1.6", + "mako==1.3.10", + "markupsafe==3.0.2", + "multidict==6.6.3", + "numpy==2.3.3", + "packaging==25.0", + "pandas==2.3.2", + "pandas-stubs==2.3.0.250703", + "pg8000==1.31.4", + "phonenumbers==9.0.13", + "pillow==11.3.0", + "pluggy==1.6.0", + "pre-commit==4.3.0", + "propcache==0.3.2", + "proto-plus==1.26.1", + "protobuf==6.32.1", + "psycopg2-binary>=2.9.10", + "pyasn1==0.6.1", + "pyasn1-modules==0.4.2", + "pycparser==2.23", + "pydantic==2.11.7", + "pydantic-core==2.33.2", + "pygments==2.19.2", + "pyjwt==2.10.1", + "pyproj==3.7.2", + "pyshp==2.3.1", + "pytest==8.4.1", + "pytest-cov>=6.2.1", + "python-dateutil==2.9.0.post0", + "python-jose>=3.5.0", + "python-multipart==0.0.20", + "pytz==2025.2", + "requests==2.32.5", + "rsa==4.9.1", + "scramp==1.4.6", + "sentry-sdk[fastapi]>=2.35.0", + "shapely==2.1.1", + "six==1.17.0", + "sniffio==1.3.1", + "sqlalchemy==2.0.43", + "sqlalchemy-continuum==1.4.2", + "sqlalchemy-searchable==2.1.0", + "sqlalchemy-utils==0.42.0", + "starlette==0.47.3", + "typing-extensions==4.15.0", + "typing-inspection==0.4.1", + "tzdata==2025.2", + "urllib3==2.5.0", + "uvicorn==0.38.0", + "yarl==1.20.1", ] [tool.alembic] @@ -114,17 +114,16 @@ script_location = "%(here)s/alembic" # file_template = "%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s" # additional paths to be prepended to sys.path. defaults to the current working directory. -prepend_sys_path = [ - "." -] +prepend_sys_path = ["."] [dependency-groups] dev = [ - "behave>=1.3.3", - "pytest>=8.4.0", - "python-dotenv>=1.1.1", - "requests>=2.32.5", - "pyhamcrest>=2.0.3", + "behave>=1.3.3", + "pytest>=8.4.0", + "python-dotenv>=1.1.1", + "requests>=2.32.5", + "pyhamcrest>=2.0.3", + "faker>=25.0.0", ] # timezone to use when rendering the date within the migration file @@ -183,4 +182,3 @@ dev = [ # type = "exec" # executable = "%(here)s/.venv/bin/ruff" # options = "check --fix REVISION_SCRIPT_FILENAME" - diff --git a/transfers/seed.py b/transfers/seed.py index af30d0c8d..43983dd9c 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -1,29 +1,231 @@ -from db.thing import Thing +""" +Populates the database with interconnected fake data for frontend CI testing. + +Run with: + docker compose exec -T app python -m transfers.seed +""" + +import random +from datetime import datetime, timedelta +from faker import Faker from db.engine import session_ctx +from sqlalchemy import select + +# Core models +from db.contact import Contact, ThingContactAssociation +from db.location import Location, LocationThingAssociation +from db.thing import Thing +from db.sensor import Sensor +from db.deployment import Deployment +from db.sample import Sample +from db.observation import Observation +from db.parameter import Parameter +from db.analysis_method import AnalysisMethod +from db.regulatory_limit import RegulatoryLimit +from db.transducer import TransducerObservation +from db.status_history import StatusHistory + +fake = Faker() +Faker.seed(42) +random.seed(42) + + +def seed_all(n=5): + """Seed roughly `n` of each main entity and connect them.""" + with session_ctx() as s: + contacts = [] + locations = [] + things = [] + sensors = [] + parameters = [] + methods = [] + samples = [] + observations = [] + + # 1. Contacts + for _ in range(n): + c = Contact( + name=fake.name(), + organization=fake.company(), + role=random.choice(["Hydrologist", "Technician", "Geologist"]), + contact_type="Primary", + ) + s.add(c) + contacts.append(c) + + # 2. Locations + for _ in range(n): + loc = Location( + elevation=round(fake.random_number(digits=3), 2), + county=fake.city(), + latitude=round(fake.latitude(), 6), + longitude=round(fake.longitude(), 6), + release_status="public", + ) + s.add(loc) + locations.append(loc) + + # 3. Retrieve existing Parameters & Methods + # + # If the environment variable MODE=development is set + # then it will initialize both the parameter and lexicon tables. + # See core/app.py for details + parameters = s.scalars(select(Parameter)).all() + if not parameters: + raise RuntimeError("No parameters found — ensure init_parameter() ran.") + + method_codes = ["ASTM-D1293", "EPA-150.1", "SM-4500-O"] + for m in method_codes: + am = AnalysisMethod( + analysis_method_code=m, + analysis_method_name=f"Method {m}", + analysis_method_type="Lab", + source_organization="NMED", + ) + s.add(am) + methods.append(am) + + s.flush() + # 4. Things (Water Wells) & ThingContactAssociation & LocationThingAssociation + for i in range(n): + t = Thing( + name=f"WELL-{i + 1:04d}", + thing_type="water well", + first_visit_date=fake.date_between("-2y", "today"), + well_depth=random.uniform(50, 500), + hole_depth=random.uniform(50, 500), + well_construction_notes=fake.sentence(), + well_casing_diameter=random.uniform(4, 8), + well_casing_depth=random.uniform(10, 50), + release_status="public", + ) -def seed(): - """Create a single contact, location, and water well.""" - with session_ctx() as session: - # Create a water well - water_well = Thing( - name="TEST-0001", - thing_type="water well", - release_status="draft", - first_visit_date="2023-03-03", - well_depth=100.0, - hole_depth=100.0, - well_construction_notes="Seed well construction notes", - well_casing_diameter=5.0, - well_casing_depth=10.0, + # link to random location + loc = random.choice(locations) + if hasattr(t, "locations"): + t.locations.append(loc) + s.add(t) + things.append(t) + + s.flush() + + for t in things: + assigned_contacts = random.sample(contacts, k=min(2, len(contacts))) + for c in assigned_contacts: + assoc = ThingContactAssociation( + thing_id=t.id, + contact_id=c.id, + ) + s.add(assoc) + + for loc in locations: + assigned_things = random.sample(things, k=min(2, len(things))) + for t in assigned_things: + assoc = LocationThingAssociation( + location_id=loc.id, + thing_id=t.id, + effective_start=datetime.utcnow(), + effective_end=None, + ) + s.add(assoc) + + # 5. Sensors & Deployments + for i in range(n): + sn = Sensor( + name=f"Sensor-{i + 1}", + sensor_type=random.choice( + ["Pressure Transducer", "Barometer", "Acoustic Sounder"] + ), + serial_no=fake.unique.bothify(text="SN-####"), + ) + sensors.append(sn) + s.add(sn) + + s.flush() + deployments = [] + for t in things: + sn = random.choice(sensors) + d = Deployment( + thing=t, + sensor=sn, + installation_date=datetime.utcnow() + - timedelta(days=random.randint(30, 180)), + removal_date=None, + ) + deployments.append(d) + s.add(d) + + # 6. Samples & Observations + for i in range(n): + samp = Sample( + sample_name=f"SMPL-{fake.random_int(1000, 9999)}", + sample_matrix="water", + sample_method=fake.choice( + ["Electric tape measurement (E-probe)", "Steel-tape measurement"] + ), + sample_date=fake.date_time_this_year(), + ) + t = random.choice(things) + samp.thing_id = t.id + samples.append(samp) + s.add(samp) + + s.flush() + for i in range(n * 2): + obs = Observation( + sample=random.choice(samples), + sensor=random.choice(sensors), + parameter=random.choice(parameters), + analysis_method=random.choice(methods), + observation_datetime=fake.date_time_this_month(), + value=round(random.uniform(0, 500), 2), + unit="mg/L", + ) + observations.append(obs) + s.add(obs) + + # 7. Regulatory Limits + for prm in parameters: + rl = RegulatoryLimit( + parameter=prm, + limit_value=random.uniform(50, 1000), + limit_unit="mg/L", + ) + s.add(rl) + + # 8. Status History (for Things) + for t in things: + st = StatusHistory( + status_type="Use Status", + status_value=random.choice(["Active", "Inactive", "Decommissioned"]), + start_date=datetime.utcnow() - timedelta(days=random.randint(100, 500)), + statusable_id=t.id, + statusable_type="Thing", + reason="Initial test seed status", + ) + s.add(st) + + # 9. Transducer Observations + for d in deployments: + for _ in range(3): + tobs = TransducerObservation( + parameter=random.choice(parameters), + deployment_id=d.id, + observation_datetime=datetime.utcnow() + - timedelta(hours=random.randint(1, 500)), + value=round(random.uniform(10, 100), 2), + ) + s.add(tobs) + + s.commit() + + print( + f"Seed complete: {len(contacts)} contacts, {len(locations)} locations, " + f"{len(things)} things, {len(sensors)} sensors, {len(samples)} samples, " + f"{len(observations)} observations." ) - session.add(water_well) - session.commit() - session.refresh(water_well) - print(f"Created water well: {water_well.id} - {water_well.name}") if __name__ == "__main__": - seed() - -# ============= EOF ============================================= + seed_all(5) diff --git a/uv.lock b/uv.lock index 20e2eed44..61ebbba0d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.13" [[package]] @@ -514,6 +514,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521, upload-time = "2024-06-20T11:30:28.248Z" }, ] +[[package]] +name = "faker" +version = "37.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/84/e95acaa848b855e15c83331d0401ee5f84b2f60889255c2e055cb4fb6bdf/faker-37.12.0.tar.gz", hash = "sha256:7505e59a7e02fa9010f06c3e1e92f8250d4cfbb30632296140c2d6dbef09b0fa", size = 1935741, upload-time = "2025-10-24T15:19:58.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl", hash = "sha256:afe7ccc038da92f2fbae30d8e16d19d91e92e242f8401ce9caf44de892bab4c4", size = 1975461, upload-time = "2025-10-24T15:19:55.739Z" }, +] + [[package]] name = "fastapi" version = "0.116.1" @@ -720,6 +732,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/1c/53/f9c440463b3057485b8594d7a638bed53ba531165ef0ca0e6c364b5cc807/greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b", size = 1564759, upload-time = "2025-11-04T12:42:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/3bb4240abdd0a8d23f4f88adec746a3099f0d86bfedb623f063b2e3b4df0/greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929", size = 1634288, upload-time = "2025-11-04T12:42:21.174Z" }, { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, @@ -727,6 +741,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/343cd760ab2f92bac1845ca07ee3faea9fe52bee65f7bcb19f16ad7de08b/greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681", size = 1680760, upload-time = "2025-11-04T12:42:25.341Z" }, { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, ] @@ -1015,6 +1031,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "behave" }, + { name = "faker" }, { name = "pyhamcrest" }, { name = "pytest" }, { name = "python-dotenv" }, @@ -1121,6 +1138,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "behave", specifier = ">=1.3.3" }, + { name = "faker", specifier = ">=25.0.0" }, { name = "pyhamcrest", specifier = ">=2.0.3" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "python-dotenv", specifier = ">=1.1.1" },