From 22069b703466c41c48338889a3ba50497b73be9c Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 15:26:54 -0600 Subject: [PATCH 01/13] [seed] Add types --- transfers/seed.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 43983dd9c..de92324b3 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -30,17 +30,17 @@ random.seed(42) -def seed_all(n=5): +def seed_all(n: int = 5): """Seed roughly `n` of each main entity and connect them.""" with session_ctx() as s: - contacts = [] - locations = [] - things = [] - sensors = [] - parameters = [] - methods = [] - samples = [] - observations = [] + contacts: list[Contact] = [] + locations: list[Location] = [] + things: list[Thing] = [] + sensors: list[Sensor] = [] + parameters: list[Parameter] = [] + methods: list[AnalysisMethod] = [] + samples: list[Sample] = [] + observations: list[Observation] = [] # 1. Contacts for _ in range(n): @@ -70,7 +70,7 @@ def seed_all(n=5): # If the environment variable MODE=development is set # then it will initialize both the parameter and lexicon tables. # See core/app.py for details - parameters = s.scalars(select(Parameter)).all() + parameters = list(s.scalars(select(Parameter)).all()) if not parameters: raise RuntimeError("No parameters found — ensure init_parameter() ran.") @@ -143,7 +143,7 @@ def seed_all(n=5): s.add(sn) s.flush() - deployments = [] + deployments: list[Deployment] = [] for t in things: sn = random.choice(sensors) d = Deployment( From d531cacc4582dba4580396ebf33954b9fc362aca Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 15:33:16 -0600 Subject: [PATCH 02/13] [seed] Update Location fields --- transfers/seed.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index de92324b3..cdba782cf 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -10,6 +10,7 @@ from faker import Faker from db.engine import session_ctx from sqlalchemy import select +from geoalchemy2.elements import WKTElement # Core models from db.contact import Contact, ThingContactAssociation @@ -55,11 +56,16 @@ def seed_all(n: int = 5): # 2. Locations for _ in range(n): + lat = round(fake.latitude(), 6) + lon = round(fake.longitude(), 6) + loc = Location( + point=WKTElement(f"POINT({lon} {lat})", srid=4326), elevation=round(fake.random_number(digits=3), 2), county=fake.city(), - latitude=round(fake.latitude(), 6), - longitude=round(fake.longitude(), 6), + notes=fake.sentence(), + elevation_accuracy=random.uniform(0.1, 5.0), + coordinate_accuracy=random.uniform(0.1, 10.0), release_status="public", ) s.add(loc) @@ -222,8 +228,8 @@ def seed_all(n: int = 5): print( f"Seed complete: {len(contacts)} contacts, {len(locations)} locations, " - f"{len(things)} things, {len(sensors)} sensors, {len(samples)} samples, " - f"{len(observations)} observations." + + f"{len(things)} things, {len(sensors)} sensors, {len(samples)} samples, " + + f"{len(observations)} observations." ) From 54bac2709f0c5929001901b166d636fd586d4f1c Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 15:49:52 -0600 Subject: [PATCH 03/13] [seed] Add lexicon term lookup --- transfers/seed.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/transfers/seed.py b/transfers/seed.py index cdba782cf..5addc128c 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -25,12 +25,28 @@ from db.regulatory_limit import RegulatoryLimit from db.transducer import TransducerObservation from db.status_history import StatusHistory +from db.lexicon import ( + LexiconTerm, + LexiconCategory, + LexiconTermCategoryAssociation, +) fake = Faker() Faker.seed(42) random.seed(42) +def get_terms_by_category(s, category_name: str) -> list[LexiconTerm]: + return list( + s.scalars( + select(LexiconTerm) + .join(LexiconTermCategoryAssociation) + .join(LexiconCategory) + .where(LexiconCategory.name == category_name) + ) + ) + + def seed_all(n: int = 5): """Seed roughly `n` of each main entity and connect them.""" with session_ctx() as s: @@ -43,11 +59,14 @@ def seed_all(n: int = 5): samples: list[Sample] = [] observations: list[Observation] = [] + # 0. Lexicons + organization_terms = get_terms_by_category(s, "organization") + # 1. Contacts for _ in range(n): c = Contact( name=fake.name(), - organization=fake.company(), + organization=random.choice(organization_terms).term, role=random.choice(["Hydrologist", "Technician", "Geologist"]), contact_type="Primary", ) From 281cc37117852fc54b6cdf64ff50865d5684e0f6 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 15:59:40 -0600 Subject: [PATCH 04/13] [seed] Add more lexicon terms like analysis_method_type --- transfers/seed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 5addc128c..54ca02df5 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -61,6 +61,7 @@ def seed_all(n: int = 5): # 0. Lexicons organization_terms = get_terms_by_category(s, "organization") + analysis_method_type_terms = get_terms_by_category(s, "analysis_method_type") # 1. Contacts for _ in range(n): @@ -104,8 +105,8 @@ def seed_all(n: int = 5): am = AnalysisMethod( analysis_method_code=m, analysis_method_name=f"Method {m}", - analysis_method_type="Lab", - source_organization="NMED", + analysis_method_type=random.choice(analysis_method_type_terms).term, + source_organization=random.choice(organization_terms).term, ) s.add(am) methods.append(am) From 355d2a39f0cf98fbd5f45ec87c7ffa1ec16d5369 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 16:12:15 -0600 Subject: [PATCH 05/13] [seed] Rm duplicate location to thing linking --- transfers/seed.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 54ca02df5..9e8db6aa0 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -126,11 +126,6 @@ def seed_all(n: int = 5): well_casing_depth=random.uniform(10, 50), release_status="public", ) - - # link to random location - loc = random.choice(locations) - if hasattr(t, "locations"): - t.locations.append(loc) s.add(t) things.append(t) From 0904e5dc9b466a0189e6714c62797232d43319d1 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 16:23:09 -0600 Subject: [PATCH 06/13] [seed] Update datetime now func --- transfers/seed.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 9e8db6aa0..669100ea7 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -62,6 +62,7 @@ def seed_all(n: int = 5): # 0. Lexicons organization_terms = get_terms_by_category(s, "organization") analysis_method_type_terms = get_terms_by_category(s, "analysis_method_type") + sample_method_terms = get_terms_by_category(s, "sample_method") # 1. Contacts for _ in range(n): @@ -146,7 +147,7 @@ def seed_all(n: int = 5): assoc = LocationThingAssociation( location_id=loc.id, thing_id=t.id, - effective_start=datetime.utcnow(), + effective_start=datetime.now(datetime.UTC), effective_end=None, ) s.add(assoc) @@ -170,7 +171,7 @@ def seed_all(n: int = 5): d = Deployment( thing=t, sensor=sn, - installation_date=datetime.utcnow() + installation_date=datetime.now(datetime.UTC) - timedelta(days=random.randint(30, 180)), removal_date=None, ) @@ -182,9 +183,7 @@ def seed_all(n: int = 5): samp = Sample( sample_name=f"SMPL-{fake.random_int(1000, 9999)}", sample_matrix="water", - sample_method=fake.choice( - ["Electric tape measurement (E-probe)", "Steel-tape measurement"] - ), + sample_method=random.choice(sample_method_terms).term, sample_date=fake.date_time_this_year(), ) t = random.choice(things) @@ -220,7 +219,8 @@ def seed_all(n: int = 5): st = StatusHistory( status_type="Use Status", status_value=random.choice(["Active", "Inactive", "Decommissioned"]), - start_date=datetime.utcnow() - timedelta(days=random.randint(100, 500)), + start_date=datetime.now(datetime.UTC) + - timedelta(days=random.randint(100, 500)), statusable_id=t.id, statusable_type="Thing", reason="Initial test seed status", @@ -233,7 +233,7 @@ def seed_all(n: int = 5): tobs = TransducerObservation( parameter=random.choice(parameters), deployment_id=d.id, - observation_datetime=datetime.utcnow() + observation_datetime=datetime.now(datetime.UTC) - timedelta(hours=random.randint(1, 500)), value=round(random.uniform(10, 100), 2), ) From cc386ee641512442ff36cee3e5437651dd42d5a9 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 16:29:31 -0600 Subject: [PATCH 07/13] [seed] fix broken timezone import --- transfers/seed.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 669100ea7..8a179e7f4 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -6,7 +6,7 @@ """ import random -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from faker import Faker from db.engine import session_ctx from sqlalchemy import select @@ -147,7 +147,7 @@ def seed_all(n: int = 5): assoc = LocationThingAssociation( location_id=loc.id, thing_id=t.id, - effective_start=datetime.now(datetime.UTC), + effective_start=datetime.now(timezone.utc), effective_end=None, ) s.add(assoc) @@ -171,7 +171,7 @@ def seed_all(n: int = 5): d = Deployment( thing=t, sensor=sn, - installation_date=datetime.now(datetime.UTC) + installation_date=datetime.now(timezone.utc) - timedelta(days=random.randint(30, 180)), removal_date=None, ) @@ -219,7 +219,7 @@ def seed_all(n: int = 5): st = StatusHistory( status_type="Use Status", status_value=random.choice(["Active", "Inactive", "Decommissioned"]), - start_date=datetime.now(datetime.UTC) + start_date=datetime.now(timezone.utc) - timedelta(days=random.randint(100, 500)), statusable_id=t.id, statusable_type="Thing", @@ -233,7 +233,7 @@ def seed_all(n: int = 5): tobs = TransducerObservation( parameter=random.choice(parameters), deployment_id=d.id, - observation_datetime=datetime.now(datetime.UTC) + observation_datetime=datetime.now(timezone.utc) - timedelta(hours=random.randint(1, 500)), value=round(random.uniform(10, 100), 2), ) From a2bbf76126f41eba4aab8cecebc4a3936b61a7ad Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 16:48:58 -0600 Subject: [PATCH 08/13] [seed] Add field events and field activities --- transfers/seed.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 8a179e7f4..4157e2a25 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -18,6 +18,7 @@ from db.thing import Thing from db.sensor import Sensor from db.deployment import Deployment +from db.field import FieldEvent, FieldActivity from db.sample import Sample from db.observation import Observation from db.parameter import Parameter @@ -56,6 +57,8 @@ def seed_all(n: int = 5): sensors: list[Sensor] = [] parameters: list[Parameter] = [] methods: list[AnalysisMethod] = [] + field_events: list[FieldEvent] = [] + field_activities: list[FieldActivity] = [] samples: list[Sample] = [] observations: list[Observation] = [] @@ -63,6 +66,8 @@ def seed_all(n: int = 5): organization_terms = get_terms_by_category(s, "organization") analysis_method_type_terms = get_terms_by_category(s, "analysis_method_type") sample_method_terms = get_terms_by_category(s, "sample_method") + activity_type_terms = get_terms_by_category(s, "activity_type") + sensor_type_terms = get_terms_by_category(s, "sensor_type") # 1. Contacts for _ in range(n): @@ -152,13 +157,36 @@ def seed_all(n: int = 5): ) s.add(assoc) - # 5. Sensors & Deployments + # 5. FieldEvent, FieldActivity, Sensors & Deployments + for t in things: + fe = FieldEvent( + thing_id=t.id, + event_date=datetime.now(timezone.utc), + notes=f"Auto-generated field event for {t.name}", + release_status="public", + ) + s.add(fe) + field_events.append(fe) + + s.flush() + + for fe in field_events: + fa = FieldActivity( + field_event_id=fe.id, + activity_type=random.choice(activity_type_terms).term, + notes=f"Auto-generated activity for event {fe.id}", + release_status="public", + ) + s.add(fa) + field_activities.append(fa) + + s.flush() + for i in range(n): sn = Sensor( + field_activity_id=random.choice(field_activities).id, name=f"Sensor-{i + 1}", - sensor_type=random.choice( - ["Pressure Transducer", "Barometer", "Acoustic Sounder"] - ), + sensor_type=random.choice(sensor_type_terms).term, serial_no=fake.unique.bothify(text="SN-####"), ) sensors.append(sn) From 40fdb10985bf3911811b6dd19b282e8ba2bc2f38 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 16:55:28 -0600 Subject: [PATCH 09/13] [seed] mv field activity to sample --- transfers/seed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/seed.py b/transfers/seed.py index 4157e2a25..332aaaaed 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -184,7 +184,6 @@ def seed_all(n: int = 5): for i in range(n): sn = Sensor( - field_activity_id=random.choice(field_activities).id, name=f"Sensor-{i + 1}", sensor_type=random.choice(sensor_type_terms).term, serial_no=fake.unique.bothify(text="SN-####"), @@ -209,6 +208,7 @@ def seed_all(n: int = 5): # 6. Samples & Observations for i in range(n): samp = Sample( + field_activity_id=random.choice(field_activities).id, sample_name=f"SMPL-{fake.random_int(1000, 9999)}", sample_matrix="water", sample_method=random.choice(sample_method_terms).term, From b5e913bdfcb313fb1207c03360ffb82885f554ac Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 17:07:06 -0600 Subject: [PATCH 10/13] [seed] rm Transducer Observations --- transfers/seed.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index 332aaaaed..c6134bde8 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -254,19 +254,6 @@ def seed_all(n: int = 5): reason="Initial test seed status", ) s.add(st) - - # 9. Transducer Observations - for d in deployments: - for _ in range(3): - tobs = TransducerObservation( - parameter=random.choice(parameters), - deployment_id=d.id, - observation_datetime=datetime.now(timezone.utc) - - timedelta(hours=random.randint(1, 500)), - value=round(random.uniform(10, 100), 2), - ) - s.add(tobs) - s.commit() print( From 0f8d5591b4057d389f2c1c82b698759854e69eee Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 17:14:37 -0600 Subject: [PATCH 11/13] [seed] Rm RegulatoryLimit & StatusHistory --- transfers/seed.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index c6134bde8..9ad81e9b2 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -232,28 +232,6 @@ def seed_all(n: int = 5): ) observations.append(obs) s.add(obs) - - # 7. Regulatory Limits - for prm in parameters: - rl = RegulatoryLimit( - parameter=prm, - limit_value=random.uniform(50, 1000), - limit_unit="mg/L", - ) - s.add(rl) - - # 8. Status History (for Things) - for t in things: - st = StatusHistory( - status_type="Use Status", - status_value=random.choice(["Active", "Inactive", "Decommissioned"]), - start_date=datetime.now(timezone.utc) - - timedelta(days=random.randint(100, 500)), - statusable_id=t.id, - statusable_type="Thing", - reason="Initial test seed status", - ) - s.add(st) s.commit() print( From b98b593bf3d20b84ddc7aaca79738d0a6518af16 Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 18:52:45 -0600 Subject: [PATCH 12/13] [seed] Add thing id link --- transfers/seed.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/transfers/seed.py b/transfers/seed.py index 9ad81e9b2..be6723b3d 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -15,7 +15,7 @@ # Core models from db.contact import Contact, ThingContactAssociation from db.location import Location, LocationThingAssociation -from db.thing import Thing +from db.thing import Thing, ThingIdLink from db.sensor import Sensor from db.deployment import Deployment from db.field import FieldEvent, FieldActivity @@ -64,6 +64,7 @@ def seed_all(n: int = 5): # 0. Lexicons organization_terms = get_terms_by_category(s, "organization") + relation_terms = get_terms_by_category(s, "relation") analysis_method_type_terms = get_terms_by_category(s, "analysis_method_type") sample_method_terms = get_terms_by_category(s, "sample_method") activity_type_terms = get_terms_by_category(s, "activity_type") @@ -157,6 +158,18 @@ def seed_all(n: int = 5): ) s.add(assoc) + for t in things: + for _ in range(random.randint(1, 3)): + chosen_org = random.choice(organization_terms) + link = ThingIdLink( + thing_id=t.id, + relation=random.choice(relation_terms).term, + alternate_id=chosen_org.id, + alternate_organization=chosen_org.term, + release_status="public", + ) + s.add(link) + # 5. FieldEvent, FieldActivity, Sensors & Deployments for t in things: fe = FieldEvent( From 0e31a50f7e9b26e28b0dd53a148f2ed8bc77f02c Mon Sep 17 00:00:00 2001 From: Tyler Adam Martinez Date: Tue, 4 Nov 2025 21:51:49 -0600 Subject: [PATCH 13/13] [seed] Update lat and long to be solely in New Mexico --- transfers/seed.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/transfers/seed.py b/transfers/seed.py index be6723b3d..eb43fc578 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -23,9 +23,6 @@ from db.observation import Observation from db.parameter import Parameter from db.analysis_method import AnalysisMethod -from db.regulatory_limit import RegulatoryLimit -from db.transducer import TransducerObservation -from db.status_history import StatusHistory from db.lexicon import ( LexiconTerm, LexiconCategory, @@ -50,6 +47,14 @@ def get_terms_by_category(s, category_name: str) -> list[LexiconTerm]: def seed_all(n: int = 5): """Seed roughly `n` of each main entity and connect them.""" + new_mexico_bounds = [ + (36.9, -106.6), # Taos area + (35.1, -106.6), # Albuquerque + (32.3, -106.8), # Las Cruces + (34.4, -103.2), # Clovis + (36.7, -108.2), # Farmington + ] + with session_ctx() as s: contacts: list[Contact] = [] locations: list[Location] = [] @@ -83,13 +88,14 @@ def seed_all(n: int = 5): # 2. Locations for _ in range(n): - lat = round(fake.latitude(), 6) - lon = round(fake.longitude(), 6) + # Generate coordinates roughly within New Mexico’s bounding box + base_lat, base_lon = random.choice(new_mexico_bounds) + lat = round(base_lat + random.uniform(-0.3, 0.3), 6) + lon = round(base_lon + random.uniform(-0.3, 0.3), 6) loc = Location( point=WKTElement(f"POINT({lon} {lat})", srid=4326), elevation=round(fake.random_number(digits=3), 2), - county=fake.city(), notes=fake.sentence(), elevation_accuracy=random.uniform(0.1, 5.0), coordinate_accuracy=random.uniform(0.1, 10.0),