From 36122f0c5093b614ba19cae4b64fd9376000a9d0 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:10:50 -0700 Subject: [PATCH 01/12] fix: use lexicon values in well inventory CSV testing data update the CSV files to use values restricted by the lexicon --- tests/features/data/well-inventory-valid-comma-in-quotes.csv | 4 ++-- tests/features/data/well-inventory-valid-extra-columns.csv | 4 ++-- tests/features/data/well-inventory-valid-reordered.csv | 4 ++-- tests/features/data/well-inventory-valid.csv | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/features/data/well-inventory-valid-comma-in-quotes.csv b/tests/features/data/well-inventory-valid-comma-in-quotes.csv index 7c1f2b28a..f347e0aef 100644 --- a/tests/features/data/well-inventory-valid-comma-in-quotes.csv +++ b/tests/features/data/well-inventory-valid-comma-in-quotes.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1D,"""Smith Farm, Domestic Well""",2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith T,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1G,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis E,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1D,"""Smith Farm, Domestic Well""",2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith T,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1G,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis E,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid-extra-columns.csv b/tests/features/data/well-inventory-valid-extra-columns.csv index 160ab9cc4..6b9eee613 100644 --- a/tests/features/data/well-inventory-valid-extra-columns.csv +++ b/tests/features/data/well-inventory-valid-extra-columns.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible,extra_column1,extract_column2 -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1v,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith B,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia V,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,, -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1f,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis B,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,, +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1v,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith B,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia V,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,, +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1f,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis B,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,, diff --git a/tests/features/data/well-inventory-valid-reordered.csv b/tests/features/data/well-inventory-valid-reordered.csv index 034c3c6a4..31427ab20 100644 --- a/tests/features/data/well-inventory-valid-reordered.csv +++ b/tests/features/data/well-inventory-valid-reordered.csv @@ -1,3 +1,3 @@ well_name_point_id,project,site_name,date_time,field_staff,utm_northing,utm_easting,utm_zone,elevation_method,elevation_ft,field_staff_2,measuring_point_height_ft,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -MRG-001_MP12,Middle Rio Grande Groundwater Monitoring,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,4000000,250000,13N,Survey-grade GPS,5250,B Chen,1.5,,John Smith A,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia A,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -MRG-003_MP12,Middle Rio Grande Groundwater Monitoring,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,4000000,250000,13N,Global positioning system (GPS),5320,,1.8,,Emily Davis A,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +MRG-001_MP12,Middle Rio Grande Groundwater Monitoring,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,4000000,250000,13N,Survey-grade GPS,5250,B Chen,1.5,,John Smith A,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia A,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +MRG-003_MP12,Middle Rio Grande Groundwater Monitoring,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,4000000,250000,13N,Global positioning system (GPS),5320,,1.8,,Emily Davis A,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid.csv b/tests/features/data/well-inventory-valid.csv index 7bcb39f71..18cdcddc6 100644 --- a/tests/features/data/well-inventory-valid.csv +++ b/tests/features/data/well-inventory-valid.csv @@ -1,3 +1,3 @@ project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible -Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True -Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False From 339f839ae63cc4175fa86ac6b9d65eb7ba12f079 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:12:03 -0700 Subject: [PATCH 02/12] feat: ignore .DS_Store --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4bf6245e0..9a894e920 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ requirements.txt # VS Code +# macOS +.DS_Store + # local development files development.db .env From 27bb37e5cf98d1d4a1b04f6f6fa2ba0ea624789a Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:13:10 -0700 Subject: [PATCH 03/12] refactor: update origin_source to origin_type in lexicon origin_source is freeform, whereas origin_type is a list of pre-defined values --- core/enums.py | 2 +- core/lexicon.json | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/core/enums.py b/core/enums.py index 91b206cab..a2c73f521 100644 --- a/core/enums.py +++ b/core/enums.py @@ -50,7 +50,7 @@ MonitoringStatus: type[Enum] = build_enum_from_lexicon_category("monitoring_status") ParameterName: type[Enum] = build_enum_from_lexicon_category("parameter_name") Organization: type[Enum] = build_enum_from_lexicon_category("organization") -OriginSource: type[Enum] = build_enum_from_lexicon_category("origin_source") +OriginType: type[Enum] = build_enum_from_lexicon_category("origin_type") ParameterType: type[Enum] = build_enum_from_lexicon_category("parameter_type") PhoneType: type[Enum] = build_enum_from_lexicon_category("phone_type") PublicationType: type[Enum] = build_enum_from_lexicon_category("publication_type") diff --git a/core/lexicon.json b/core/lexicon.json index 0d14be5ac..04c0e5f30 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -53,7 +53,7 @@ {"name": "well_purpose", "description": null}, {"name": "status_type", "description": null}, {"name": "status_value", "description": null}, - {"name": "origin_source", "description": null}, + {"name": "origin_type", "description": null}, {"name": "well_pump_type", "description": null}, {"name": "permission_type", "description": null}, {"name": "formation_code", "description": null}, @@ -1151,18 +1151,19 @@ {"categories": ["lithology"],"term": "Ignesous, intrusive, undifferentiated","definition": "Ignesous, intrusive, undifferentiated"}, {"categories": ["lithology"],"term": "Limestone, sandstone and shale","definition": "Limestone, sandstone and shale"}, {"categories": ["lithology"],"term": "Sand, silt and clay","definition": "Sand, silt and clay"}, - {"categories": ["origin_source"], "term": "Reported by another agency", "definition": "Reported by another agency"}, - {"categories": ["origin_source"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, - {"categories": ["origin_source"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, - {"categories": ["origin_source"], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency"}, - {"categories": ["origin_source"], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller"}, - {"categories": ["origin_source"], "term": "Measured by source agency", "definition": "Measured by source agency"}, - {"categories": ["origin_source"], "term": "Reported by owner of well", "definition": "Reported by owner of well"}, - {"categories": ["origin_source"], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency"}, - {"categories": ["origin_source"], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff"}, - {"categories": ["origin_source"], "term": "Other", "definition": "Other"}, - {"categories": ["origin_source"], "term": "Data Portal", "definition": "Data Portal"}, + {"categories": ["origin_type"], "term": "Reported by another agency", "definition": "Reported by another agency"}, + {"categories": ["origin_type"], "term": "From driller's log or well report", "definition": "From driller's log or well report"}, + {"categories": ["origin_type"], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate"}, + {"categories": ["origin_type"], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency"}, + {"categories": ["origin_type"], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller"}, + {"categories": ["origin_type"], "term": "Measured by source agency", "definition": "Measured by source agency"}, + {"categories": ["origin_type"], "term": "Reported by owner of well", "definition": "Reported by owner of well"}, + {"categories": ["origin_type"], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency"}, + {"categories": ["origin_type"], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff"}, + {"categories": ["origin_type"], "term": "Other", "definition": "Other"}, + {"categories": ["origin_type"], "term": "Data Portal", "definition": "Data Portal"}, {"categories": ["note_type"], "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc."}, + {"categories": ["note_type"], "term": "Directions", "definition": "Notes about directions to a location"}, {"categories": ["note_type"], "term": "Construction", "definition": "Construction details, well development, drilling notes, etc. Could create separate `types` for each of these if needed."}, {"categories": ["note_type"], "term": "Maintenance", "definition": "Maintenance observations and issues."}, {"categories": ["note_type"], "term": "Historical", "definition": "Historical information or context about the well or location."}, From 9c79e8d28fd73d941389e3f01799992b3e48940a Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:18:22 -0700 Subject: [PATCH 04/12] feat: add well inventory csv gherkin file --- tests/features/well-inventory-csv.feature | 452 ++++++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 tests/features/well-inventory-csv.feature diff --git a/tests/features/well-inventory-csv.feature b/tests/features/well-inventory-csv.feature new file mode 100644 index 000000000..f7738960d --- /dev/null +++ b/tests/features/well-inventory-csv.feature @@ -0,0 +1,452 @@ +@backend +@BDMS-TBD +@production +Feature: Bulk upload well inventory from CSV + As a hydrogeologist or data specialist + I want to upload a CSV file containing well inventory data for multiple wells + So that well records can be created efficiently and accurately in the system + + Background: + Given a functioning api + And valid lexicon values exist for: + | lexicon category | + | contact_role | + | contact_type | + | phone_type | + | email_type | + | address_type | + | elevation_method | + | well_pump_type | + | well_purpose | + | well_hole_status | + | monitoring_frequency | + + @positive @happy_path @BDMS-TBD + Scenario: Uploading a valid well inventory CSV containing required and optional fields + Given a valid CSV file for bulk well inventory upload + And my CSV file is encoded in UTF-8 and uses commas as separators + And my CSV file contains multiple rows of well inventory data + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + And each "well_name_point_id" value is unique per row + And "date_time" values are valid ISO 8601 timestamps with timezone offsets (e.g. "2025-02-15T10:30:00-08:00") + And the CSV includes optional fields when available: + | optional field name | + | field_staff_2 | + | field_staff_3 | + | contact_1_name | + | contact_1_organization | + | contact_1_role | + | contact_1_type | + | contact_1_phone_1 | + | contact_1_phone_1_type | + | contact_1_phone_2 | + | contact_1_phone_2_type | + | contact_1_email_1 | + | contact_1_email_1_type | + | contact_1_email_2 | + | contact_1_email_2_type | + | contact_1_address_1_line_1 | + | contact_1_address_1_line_2 | + | contact_1_address_1_type | + | contact_1_address_1_state | + | contact_1_address_1_city | + | contact_1_address_1_postal_code | + | contact_1_address_2_line_1 | + | contact_1_address_2_line_2 | + | contact_1_address_2_type | + | contact_1_address_2_state | + | contact_1_address_2_city | + | contact_1_address_2_postal_code | + | contact_2_name | + | contact_2_organization | + | contact_2_role | + | contact_2_type | + | contact_2_phone_1 | + | contact_2_phone_1_type | + | contact_2_phone_2 | + | contact_2_phone_2_type | + | contact_2_email_1 | + | contact_2_email_1_type | + | contact_2_email_2 | + | contact_2_email_2_type | + | contact_2_address_1_line_1 | + | contact_2_address_1_line_2 | + | contact_2_address_1_type | + | contact_2_address_1_state | + | contact_2_address_1_city | + | contact_2_address_1_postal_code | + | contact_2_address_2_line_1 | + | contact_2_address_2_line_2 | + | contact_2_address_2_type | + | contact_2_address_2_state | + | contact_2_address_2_city | + | contact_2_address_2_postal_code | + | directions_to_site | + | specific_location_of_well | + | repeat_measurement_permission | + | sampling_permission | + | datalogger_installation_permission | + | public_availability_acknowledgement | + | result_communication_preference | + | contact_special_requests_notes | + | ose_well_record_id | + | date_drilled | + | completion_source | + | total_well_depth_ft | + | historic_depth_to_water_ft | + | depth_source | + | well_pump_type | + | well_pump_depth_ft | + | is_open | + | datalogger_possible | + | casing_diameter_ft | + | measuring_point_description | + | well_purpose | + | well_purpose_2 | + | well_hole_status | + | monitoring_frequency | + | sampling_scenario_notes | + | well_measuring_notes | + | sample_possible | +# And all optional lexicon fields contain valid lexicon values when provided +# And all optional numeric fields contain valid numeric values when provided +# And all optional date fields contain valid ISO 8601 timestamps when provided + + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format +# And null values in the response are represented as JSON null + And the response includes a summary containing: + | summary_field | value | + | total_rows_processed | 2 | + | total_rows_imported | 2 | + | validation_errors_or_warnings | 0 | + And the response includes an array of created well objects + + @positive @validation @column_order @BDMS-TBD + Scenario: Upload succeeds when required columns are present but in a different order + Given my CSV file contains all required headers but in a different column order + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @extra_columns @BDMS-TBD + Scenario: Upload succeeds when CSV contains extra, unknown columns + Given my CSV file contains extra columns but is otherwise valid + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @autogenerate_ids @BDMS-TBD + Scenario: Upload succeeds and system auto-generates well_name_point_id when prefixed with "XY- + Given my CSV file contains all valid columns but uses "XY-" prefix for well_name_point_id values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported with system-generated unique well_name_point_id values + + ########################################################################### + # NEGATIVE VALIDATION SCENARIOS + ########################################################################### + @negative @validation @transactional_import @BDMS-TBD + Scenario: No wells are imported when any row fails validation + Given my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the row missing "well_name_point_id" + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has an invalid postal code format + Given my CSV file contains a row that has an invalid postal code format in contact_1_address_1_postal_code + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid postal code format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid phone number format + Given my CSV file contains a row with a contact with a phone number that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid phone number format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid email format + Given my CSV file contains a row with a contact with an email that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid email format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a contact_role + Given my CSV file contains a row with a contact but is missing the required "contact_role" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_role" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a "contact_type" + Given my CSV file contains a row with a contact but is missing the required "contact_type" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an invalid "contact_type" + Given my CSV file contains a row with a contact_type value that is not in the valid lexicon for "contact_type" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an email without an email_type + Given my CSV file contains a row with a contact with an email but is missing the required "email_type" field for that email + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "email_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with a phone without a phone_type + Given my CSV file contains a row with a contact with a phone but is missing the required "phone_type" field for that phone + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "phone_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an address without an address_type + Given my CSV file contains a row with a contact with an address but is missing the required "address_type" field for that address + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "address_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has utm_easting utm_northing and utm_zone values that are not within New Mexico + Given my CSV file contains a row with utm_easting utm_northing and utm_zone values that are not within New Mexico + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid UTM coordinates + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when required fields are missing + Given my CSV file contains rows missing a required field "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes validation errors for all rows missing required fields + And the response identifies the row and field for each error + And no wells are imported + + @negative @validation @required_fields @BDMS-TBD + Scenario Outline: Upload fails when a required field is missing + Given my CSV file contains a row missing the required "" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the "" field + And no wells are imported + + Examples: + | required_field | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + + @negative @validation @boolean_fields @BDMS-TBD + Scenario: Upload fails due to invalid boolean field values + Given my CSV file contains a row with an invalid boolean value "maybe" in the "is_open" field +# And my CSV file contains other boolean fields such as "sample_possible" with valid boolean values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid boolean value for the "is_open" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when duplicate well_name_point_id values are present + Given my CSV file contains one or more duplicate "well_name_point_id" values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors indicating duplicated values + And each error identifies the row and field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid lexicon values + Given my CSV file contains invalid lexicon values for "contact_role" or other lexicon fields + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid date formats + Given my CSV file contains invalid ISO 8601 date values in the "date_time" or "date_drilled" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid numeric fields + Given my CSV file contains values that cannot be parsed as numeric in numeric-required fields such as "utm_easting" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + +# ########################################################################### +# # FILE FORMAT SCENARIOS +# ########################################################################### + + @negative @file_format @limits @BDMS-TBD + Scenario: Upload fails when the CSV exceeds the maximum allowed number of rows + Given my CSV file contains more rows than the configured maximum for bulk upload + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating the row limit was exceeded + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when file type is unsupported + Given I have a non-CSV file + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating unsupported file type + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when the CSV file is empty + Given my CSV file is empty + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating an empty file + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when CSV contains only headers + Given my CSV file contains column headers but no data rows + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error indicating that no data rows were found + And no wells are imported + + ########################################################################### + # HEADER & SCHEMA INTEGRITY SCENARIOS + ########################################################################### + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when a header row is repeated in the middle of the file + Given my CSV file contains a valid but duplicate header row + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating a repeated header row + And no wells are imported + + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when the header row contains duplicate column names + Given my CSV file header row contains the "contact_1_email_1" column name more than once + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating duplicate header names + And no wells are imported + + + ########################################################################### + # DELIMITER & QUOTING / EXCEL-RELATED SCENARIOS + ########################################################################### + + @negative @file_format @delimiter @BDMS-TBD + Scenario Outline: Upload fails when CSV uses an unsupported delimiter + Given my file is named with a .csv extension + And my file uses "" as the field delimiter instead of commas + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating an unsupported delimiter + And no wells are imported + + Examples: + | delimiter_description | + | semicolons | + | tab characters | + + @positive @file_format @quoting @BDMS-TBD + Scenario: Upload succeeds when fields contain commas inside properly quoted values + Given my CSV file header row contains all required columns + And my CSV file contains a data row where the "site_name" field value includes a comma and is enclosed in quotes +# And all other required fields are populated with valid values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported +# +# @negative @validation @numeric @excel @BDMS-TBD +# Scenario: Upload fails when numeric fields are provided in Excel scientific notation format +# Given my CSV file contains a numeric-required field such as "utm_easting" +# And Excel has exported the "utm_easting" value in scientific notation (for example "1.2345E+06") +# When I upload the file to the bulk upload endpoint +# Then the system returns a 422 Unprocessable Entity status code +# And the system should return a response in JSON format +# And the response includes a validation error indicating an invalid numeric format for "utm_easting" +# And no wells are imported \ No newline at end of file From 1b4bfcc5b78e762bb390ef643acd6b7b2c43f1aa Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:21:28 -0700 Subject: [PATCH 05/12] refactor: default engine's port to 54321 to reflect docker The docker compose file was changed to map Postgres to host port 54321. --- db/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/engine.py b/db/engine.py index bc177eb8e..d9e889d2f 100644 --- a/db/engine.py +++ b/db/engine.py @@ -109,7 +109,7 @@ def getconn(): # elif driver == "postgres": password = os.environ.get("POSTGRES_PASSWORD", "") host = os.environ.get("POSTGRES_HOST", "localhost") - port = os.environ.get("POSTGRES_PORT", "5432") + port = os.environ.get("POSTGRES_PORT", "54321") # Default to current OS user if POSTGRES_USER not set or empty user = os.environ.get("POSTGRES_USER", "").strip() or getpass.getuser() name = os.environ.get("POSTGRES_DB", "postgres") From 6b37efa9dc26561a13745c2eabdd5951cc29e689 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:53:46 -0700 Subject: [PATCH 06/12] feat: add well inventory as possible activity_type --- core/lexicon.json | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lexicon.json b/core/lexicon.json index 04c0e5f30..85378e759 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -284,6 +284,7 @@ {"categories": ["relation"], "term": "OSEWellTagID", "definition": "NM OSE well tag ID"}, {"categories": ["relation"], "term": "OSEPOD", "definition": "NM OSE 'Point of Diversion' ID"}, {"categories": ["relation"], "term": "PLSS", "definition": "Public Land Survey System ID"}, + {"categories": ["activity_type"], "term": "well inventory", "definition": "well inventory"}, {"categories": ["activity_type"], "term": "groundwater level", "definition": "groundwater level"}, {"categories": ["activity_type"], "term": "water chemistry", "definition": "water chemistry"}, {"categories": ["participant_role"], "term": "Lead", "definition": "the leader of the field event"}, From 6560b92297cdd51148c90608cc90eaaf4f116f61 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 14:58:34 -0700 Subject: [PATCH 07/12] note: indicate which fields still need a home in the models These fields were noted with the inline comment "TODO: needs a home" --- schemas/well_inventory.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/schemas/well_inventory.py b/schemas/well_inventory.py index fb0d6c76f..0524baea6 100644 --- a/schemas/well_inventory.py +++ b/schemas/well_inventory.py @@ -216,17 +216,17 @@ class WellInventoryRow(BaseModel): repeat_measurement_permission: OptionalBool = None sampling_permission: OptionalBool = None datalogger_installation_permission: OptionalBool = None - public_availability_acknowledgement: OptionalBool = None + public_availability_acknowledgement: OptionalBool = None # TODO: needs a home special_requests: Optional[str] = None ose_well_record_id: Optional[str] = None date_drilled: OptionalDateTime = None completion_source: Optional[str] = None total_well_depth_ft: OptionalFloat = None - historic_depth_to_water_ft: OptionalFloat = None + historic_depth_to_water_ft: OptionalFloat = None # TODO: needs a home depth_source: Optional[str] = None well_pump_type: Optional[str] = None well_pump_depth_ft: OptionalFloat = None - is_open: OptionalBool = None + is_open: OptionalBool = None # TODO: needs a home datalogger_possible: OptionalBool = None casing_diameter_ft: OptionalFloat = None measuring_point_description: Optional[str] = None @@ -235,11 +235,11 @@ class WellInventoryRow(BaseModel): well_hole_status: Optional[str] = None monitoring_frequency: MonitoryFrequencyField = None - result_communication_preference: Optional[str] = None - contact_special_requests_notes: Optional[str] = None - sampling_scenario_notes: Optional[str] = None + result_communication_preference: Optional[str] = None # TODO: needs as home + contact_special_requests_notes: Optional[str] = None # TODO: needs a home + sampling_scenario_notes: Optional[str] = None # TODO: needs a home well_measuring_notes: Optional[str] = None - sample_possible: OptionalBool = None + sample_possible: OptionalBool = None # TODO: needs a home @model_validator(mode="after") def validate_model(self): From 0387409a1d991ffb4644779a66ecc398a6122eda Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 15:00:40 -0700 Subject: [PATCH 08/12] feat: update CreateWell and CreateThing schemas for well inventory CSV import Both optional and required fields have been added to the CreateWell and CreateThing schemas per the well inventory CSV import requirements. The fields added to CreateThing are applicable to all thing types, while the fields added to CreateWell are specific to well things. --- schemas/thing.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/schemas/thing.py b/schemas/thing.py index 0ccf80376..eae9191d3 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -27,6 +27,7 @@ WellConstructionMethod, WellPumpType, FormationCode, + OriginType, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate from schemas.group import GroupResponse @@ -43,6 +44,7 @@ class ValidateWell(BaseModel): hole_depth: float | None = None # in feet well_casing_depth: float | None = None # in feet measuring_point_height: float | None = None + well_pump_depth: float | None = None # in feet @model_validator(mode="after") def validate_values(self): @@ -59,6 +61,12 @@ def validate_values(self): "well casing depth must be less than or equal to hole depth" ) + if self.well_pump_depth is not None: + if self.well_depth is not None and self.well_pump_depth > self.well_depth: + raise ValueError("well pump depth must be less than well depth") + elif self.hole_depth is not None and self.well_pump_depth > self.hole_depth: + raise ValueError("well pump depth must be less than hole depth") + # if self.measuring_point_height is not None: # if ( # self.hole_depth is not None @@ -107,6 +115,21 @@ class CreateBaseThing(BaseCreateModel): group_id: int | None = None # Optional group ID for the thing name: str # Name of the thing first_visit_date: PastOrTodayDate | None = None # Date of NMBGMR's first visit + notes: list[CreateNote] | None = None + alternate_ids: list[CreateThingIdLink] | None = None + monitoring_frequencies: list[MonitoringFrequency] | None = None + + @field_validator("alternate_ids", mode="before") + def use_dummy_values(cls, v): + """ + When alternate IDs are provided they are assumed to be the same as + the thing being created. This gets handled in the function services/thing_helper.py::add_thing. + By using dummy values here we can avoid validation errors and then use the + thing's id when creating the actual links. + """ + for alternate_id in v: + alternate_id.thing_id = -1 # dummy value + return v class CreateWell(CreateBaseThing, ValidateWell): @@ -118,6 +141,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_depth: float | None = Field( default=None, gt=0, description="Well depth in feet" ) + well_depth_source: OriginType | None = None hole_depth: float | None = Field( default=None, gt=0, description="Hole depth in feet" ) @@ -128,16 +152,15 @@ class CreateWell(CreateBaseThing, ValidateWell): default=None, gt=0, description="Well casing depth in feet" ) well_casing_materials: list[CasingMaterial] | None = None - measuring_point_height: float = Field(description="Measuring point height in feet") measuring_point_description: str | None = None - notes: list[CreateNote] | None = None well_completion_date: PastOrTodayDate | None = None well_completion_date_source: str | None = None well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None + well_pump_depth: float | None = None is_suitable_for_datalogger: bool | None formation_completion_code: FormationCode | None = None From a7a096834cd1c33bce586f6589e81e2b4dd37dc6 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 15:03:50 -0700 Subject: [PATCH 09/12] feat/refactor: move logic for thing tables to add_thing The function add_thing should handle all of the data in CreateWell, so that it can be used in multiple places without duplicating code. --- api/well_inventory.py | 248 ++++++++++++++++++++++++++++----------- services/thing_helper.py | 168 +++++++++++++++++++++----- 2 files changed, 314 insertions(+), 102 deletions(-) diff --git a/api/well_inventory.py b/api/well_inventory.py index 68b9cb323..533ba8f19 100644 --- a/api/well_inventory.py +++ b/api/well_inventory.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import csv +from datetime import date import logging import re from collections import Counter @@ -38,17 +39,14 @@ from core.dependencies import session_dependency, amp_editor_dependency from db import ( Group, - ThingIdLink, - GroupThingAssociation, Location, - LocationThingAssociation, - MeasuringPointHistory, DataProvenance, FieldEvent, FieldEventParticipant, Contact, + PermissionHistory, + Thing, ) -from db.thing import Thing, WellPurpose, MonitoringFrequencyHistory from schemas.thing import CreateWell from schemas.well_inventory import WellInventoryRow from services.contact_helper import add_contact @@ -59,7 +57,7 @@ router = APIRouter(prefix="/well-inventory-csv") -def _add_location(model, well) -> Location: +def _make_location(model) -> Location: point = Point(model.utm_easting, model.utm_northing) # TODO: this needs to be more sophisticated in the future. Likely more than 13N and 12N will be used @@ -79,11 +77,8 @@ def _add_location(model, well) -> Location: point=transformed_point.wkt, elevation=elevation_m, ) - date_time = model.date_time - assoc = LocationThingAssociation(location=loc, thing=well) - assoc.effective_start = date_time - return loc, assoc + return loc def _make_contact(model: WellInventoryRow, well: Thing, idx) -> dict: @@ -133,6 +128,43 @@ def _make_contact(model: WellInventoryRow, well: Thing, idx) -> dict: } +def _make_well_permission( + well: Thing, + contact: Contact | None, + permission_type: str, + permission_allowed: bool, + start_date: date, +) -> PermissionHistory: + """ + Makes a PermissionHistory record for the given well and contact. + If the contact has not been provided, but a permission is to be created, + no PermissionHistory record is created and a 400 error is raised. + """ + if contact is None: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "At least one contact required for permission", + "type": "Contact required for permission", + "input": None, + } + ], + ) + + permission = PermissionHistory( + target_table="thing", + target_id=well.id, + contact=contact, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=None, + ) + return permission + + AUTOGEN_REGEX = re.compile(r"^[A-Za-z]{2}-$") @@ -414,32 +446,130 @@ def _add_field_staff( def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) -> str: name = model.well_name_point_id date_time = model.date_time - site_name = model.site_name + + # -------------------- + # Location and associated tables + # -------------------- + + # add Location + loc = _make_location(model) + session.add(loc) + session.flush() + + # add location notes + if model.directions_to_site: + directions_note = loc.add_note( + content=model.directions_to_site, note_type="Directions" + ) + session.add(directions_note) + + # add data provenance records + dp = DataProvenance( + target_id=loc.id, + target_table="location", + field_name="elevation", + collection_method=model.elevation_method, + ) + session.add(dp) + + # -------------------- + # Thing and associated tables + # -------------------- # add Thing + well_notes = [] + for note_content, note_type in ( + (model.specific_location_of_well, "Access"), + (model.special_requests, "General"), + (model.well_measuring_notes, "Measuring"), + ): + if note_content is not None: + well_notes.append({"content": note_content, "note_type": note_type}) + + alternate_ids = [] + for alternate_id, alternate_organization in ( + (model.site_name, "NMBGMR"), + (model.ose_well_record_id, "NMOSE"), + ): + if alternate_id is not None: + alternate_ids.append( + { + "alternate_id": alternate_id, + "alternate_organization": alternate_organization, + "relation": "same_as", + } + ) + + well_purposes = [] + if model.well_purpose: + well_purposes.append(model.well_purpose) + if model.well_purpose_2: + well_purposes.append(model.well_purpose_2) + + monitoring_frequencies = [] + if model.monitoring_frequency: + monitoring_frequencies.append( + { + "monitoring_frequency": model.monitoring_frequency, + "start_date": date_time.date(), + } + ) + data = CreateWell( + location_id=loc.id, + group_id=group.id, name=name, first_visit_date=date_time.date(), well_depth=model.total_well_depth_ft, + well_depth_source=model.depth_source, well_casing_diameter=model.casing_diameter_ft, measuring_point_height=model.measuring_point_height_ft, measuring_point_description=model.measuring_point_description, + well_completion_date=model.date_drilled, + well_completion_date_source=model.completion_source, + well_pump_type=model.well_pump_type, + well_pump_depth=model.well_pump_depth_ft, + is_suitable_for_datalogger=model.datalogger_possible, + notes=well_notes, + well_purposes=well_purposes, ) well_data = data.model_dump( exclude=[ - "location_id", - "group_id", "well_purposes", "well_casing_materials", - "measuring_point_height", - "measuring_point_description", ] ) + + """ + Developer's notes + + the add_thing function also handles: + - MeasuringPointHistory + - GroupThingAssociation + - LocationThingAssociation + - DataProvenance for well_completion_date + - DataProvenance for well_construction_method + - DataProvenance for well_depth + - Notes + - WellPurpose + - MonitoringFrequencyHistory + """ well = add_thing( session=session, data=well_data, user=user, thing_type="water well" ) session.refresh(well) + # ------------------ + # Field Events and related tables + # ------------------ + """ + Developer's notes + + These tables are not handled in add_thing because they are only relevant if + the well has been inventoried in the field, not if the well is added from + another source like a report, database, or map. + """ + # add field event fe = FieldEvent( event_date=date_time, @@ -459,64 +589,40 @@ def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) _add_field_staff(session, fsi, fe, role, user) - # add MonitoringFrequency - if model.monitoring_frequency: - mfh = MonitoringFrequencyHistory( - thing=well, - monitoring_frequency=model.monitoring_frequency, - start_date=date_time.date(), - ) - session.add(mfh) - - # add WellPurpose - for p in (model.well_purpose, model.well_purpose_2): - if not p: - continue - wp = WellPurpose(purpose=p, thing=well) - session.add(wp) - - # BDMS-221 adds MeasuringPointHistory model - measuring_point_height_ft = model.measuring_point_height_ft - if measuring_point_height_ft: - mph = MeasuringPointHistory( - thing=well, - measuring_point_height=measuring_point_height_ft, - measuring_point_description=model.measuring_point_description, - start_date=date_time.date(), - ) - session.add(mph) - - # add Location - loc, assoc = _add_location(model, well) - session.add(loc) - session.add(assoc) - session.flush() - - dp = DataProvenance( - target_id=loc.id, - target_table="location", - field_name="elevation", - collection_method=model.elevation_method, - ) - session.add(dp) - - gta = GroupThingAssociation(group=group, thing=well) - session.add(gta) - group.thing_associations.append(gta) - - # add alternate ids - well.links.append( - ThingIdLink( - alternate_id=site_name, - alternate_organization="NMBGMR", - relation="same_as", - ) - ) + # ------------------ + # Contacts + # ------------------ + # add contacts + contact_for_permissions = None for idx in (1, 2): - contact = _make_contact(model, well, idx) - if contact: - add_contact(session, contact, user=user) + contact_dict = _make_contact(model, well, idx) + if contact_dict: + contact = add_contact(session, contact_dict, user=user) + + # Use the first created contact for permissions if available + if contact_for_permissions is None: + contact_for_permissions = contact + + # ------------------ + # Permissions + # ------------------ + + # add permissions + for permission_type, permission_allowed in ( + ("Water Level Sample", model.repeat_measurement_permission), + ("Water Chemistry Sample", model.sampling_permission), + ("Datalogger Installation", model.datalogger_installation_permission), + ): + if permission_allowed is not None: + permission = _make_well_permission( + well=well, + contact=contact_for_permissions, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=model.date_time.date(), + ) + session.add(permission) return model.well_name_point_id diff --git a/services/thing_helper.py b/services/thing_helper.py index 100b49994..ec4e330d5 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -35,6 +35,9 @@ ThingAquiferAssociation, GroupThingAssociation, MeasuringPointHistory, + DataProvenance, + ThingIdLink, + MonitoringFrequencyHistory, ) from services.audit_helper import audit_add @@ -48,7 +51,7 @@ "well_casing_materials": (WellCasingMaterial, "material"), } -WELL_LOADER_OPTIONS = [ +WATER_WELL_LOADER_OPTIONS = [ selectinload(Thing.location_associations).selectinload( LocationThingAssociation.location ), @@ -62,7 +65,7 @@ ), ] -WELL_THING_TYPE = "water well" +WATER_WELL_THING_TYPE = "water well" def wkb_to_geojson(wkb_element): @@ -91,11 +94,11 @@ def get_db_things( if thing_type: sql = sql.where(Thing.thing_type == thing_type) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) else: # add all eager loads for generic thing query until/unless GET /thing is deprecated - sql = sql.options(*WELL_LOADER_OPTIONS) + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) if name: sql = sql.where(Thing.name == name) @@ -160,8 +163,8 @@ def get_thing_of_a_thing_type_by_id(session: Session, request: Request, thing_id thing_type = get_thing_type_from_request(request) sql = select(Thing).where(Thing.id == thing_id) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) thing = session.execute(sql).scalar_one_or_none() @@ -186,21 +189,44 @@ def add_thing( if request is not None: thing_type = get_thing_type_from_request(request) - if isinstance(data, BaseModel): - well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) - data = data.model_dump(exclude=well_descriptor_table_list) + # Extract data for related tables - notes = None - if "notes" in data: - notes = data.pop("notes") + # --------- + # BEGIN UNIVERSAL THING RELATED TABLES + # --------- + notes = data.pop("notes", None) + alternate_ids = data.pop("alternate_ids", None) location_id = data.pop("location_id", None) + effective_start = data.get("first_visit_date") group_id = data.pop("group_id", None) + monitoring_frequencies = data.pop("monitoring_frequencies", None) - # Extract measuring point data (stored in separate history table, not as Thing columns) + # ---------- + # END UNIVERSAL THING RELATED TABLES + # ---------- + + # ---------- + # BEGIN WATER WELL SPECIFIC RELATED TABLES + # ---------- + + # measuring point info measuring_point_height = data.pop("measuring_point_height", None) measuring_point_description = data.pop("measuring_point_description", None) + # data provenance info + well_completion_date_source = data.pop("well_completion_date_source", None) + well_construction_method_source = data.pop("well_construction_method_source", None) + well_depth_source = data.pop("well_depth_source", None) + + # descriptor tables + well_purposes = data.pop("well_purposes", None) + well_casing_materials = data.pop("well_casing_materials", None) + + # ---------- + # END WATER WELL SPECIFIC RELATED TABLES + # ---------- + try: thing = Thing(**data) thing.thing_type = thing_type @@ -211,17 +237,73 @@ def add_thing( session.flush() session.refresh(thing) - # Create MeasuringPointHistory record if measuring_point_height provided - if measuring_point_height is not None: - measuring_point_history = MeasuringPointHistory( - thing_id=thing.id, - measuring_point_height=measuring_point_height, - measuring_point_description=measuring_point_description, - start_date=datetime.now(tz=ZoneInfo("UTC")), - end_date=None, - ) - audit_add(user, measuring_point_history) - session.add(measuring_point_history) + # ---------- + # BEING WATER WELL SPECIFIC LOGIC + # ---------- + + if thing_type == WATER_WELL_THING_TYPE: + + # Create MeasuringPointHistory record if measuring_point_height provided + if measuring_point_height is not None: + measuring_point_history = MeasuringPointHistory( + thing_id=thing.id, + measuring_point_height=measuring_point_height, + measuring_point_description=measuring_point_description, + start_date=datetime.now(tz=ZoneInfo("UTC")), + end_date=None, + ) + audit_add(user, measuring_point_history) + session.add(measuring_point_history) + + if well_completion_date_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_completion_date", + origin_type=well_completion_date_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_depth_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_depth", + origin_type=well_depth_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_construction_method_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_construction_method", + origin_source=well_construction_method_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_purposes: + for purpose in well_purposes: + wp = WellPurpose(thing_id=thing.id, purpose=purpose) + audit_add(user, wp) + session.add(wp) + + if well_casing_materials: + for material in well_casing_materials: + wcm = WellCasingMaterial(thing_id=thing.id, material=material) + audit_add(user, wcm) + session.add(wcm) + + # ---------- + # END WATER WELL SPECIFIC LOGIC + # ---------- + + # ---------- + # BEGIN UNIVERSAL THING RELATED LOGIC + # ---------- # endpoint catches ProgrammingError if location_id or group_id do not exist if group_id: @@ -232,23 +314,47 @@ def add_thing( session.add(assoc) if location_id is not None: - # TODO: how do we want to handle effective_start? is it the date it gets entered? assoc = LocationThingAssociation() audit_add(user, assoc) assoc.location_id = location_id assoc.thing_id = thing.id + assoc.effective_start = effective_start session.add(assoc) - session.commit() - session.refresh(thing) - if notes: for n in notes: - nn = thing.add_note(n["content"], n["note_type"]) - session.add(nn) + thing_note = thing.add_note(n["content"], n["note_type"]) + session.add(thing_note) session.commit() session.refresh(thing) + if alternate_ids: + for aid in alternate_ids: + id_link = ThingIdLink( + thing_id=thing.id, + relation=aid["relation"], + alternate_id=aid["alternate_id"], + alternate_organization=aid["alternate_organization"], + ) + session.add(id_link) + + if monitoring_frequencies: + for mf in monitoring_frequencies: + mfh = MonitoringFrequencyHistory( + thing_id=thing.id, + monitoring_frquency=mf["monitoring_frequency"], + start_date=mf["start_date"], + end_date=mf.get("end_date", None), + ) + session.add(mfh) + + # ---------- + # END UNIVERSAL THING RELATED LOGIC + # ---------- + + session.commit() + session.refresh(thing) + except Exception as e: session.rollback() raise e From a70b71ca52b0d5d3d968ee736b233bc6090c33c5 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 15:08:38 -0700 Subject: [PATCH 10/12] feat: update well transfer script to account for updated CreateWell schema --- transfers/well_transfer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index aaa2eb0bd..91c388fb1 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -339,12 +339,18 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): "measuring_point_description", "well_completion_date_source", "well_construction_method_source", + "well_depth_source", + "alternate_ids", + "monitoring_frequencies", + "notes", + "well_depth_source", + "well_completion_date_source", + "well_construction_method_source", ] ) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID - well_data.pop("notes") well = Thing(**well_data) session.add(well) From 8577af420cca00fa92a82bdb816fa78e5c68baa5 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 15:41:07 -0700 Subject: [PATCH 11/12] feat: add field activity record for the well inventory there can be multiple activities per field event, one of which is the well inventory --- api/well_inventory.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/well_inventory.py b/api/well_inventory.py index 533ba8f19..5f4b072ab 100644 --- a/api/well_inventory.py +++ b/api/well_inventory.py @@ -43,6 +43,7 @@ DataProvenance, FieldEvent, FieldEventParticipant, + FieldActivity, Contact, PermissionHistory, Thing, @@ -589,6 +590,14 @@ def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) _add_field_staff(session, fsi, fe, role, user) + # add field activity + fa = FieldActivity( + field_event=fe, + activity_type="well inventory", + notes="Well inventory conducted during field event.", + ) + session.add(fa) + # ------------------ # Contacts # ------------------ From efa3af4320a5333c0b48c819da906ad00ffc782c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Tue, 9 Dec 2025 17:03:22 -0700 Subject: [PATCH 12/12] fix: reset default POSTGRES_PORT to 5432 and update POSTGRES_PORT in docker-compose.yml Inside Docker the app needs to use port 5432 to connect to Postgres, but on the host machine we want to use 54321. This can be set in the .env file, but to prevent 54321 from being used within Docker we set POSTGRES_PORT to 5432. --- .env.example | 4 ++++ db/engine.py | 2 +- docker-compose.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 227db2d9d..cbf54e954 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ DB_DRIVER=postgres POSTGRES_USER=admin POSTGRES_PASSWORD=password POSTGRES_DB= +POSTGRES_PORT=54321 # asset storage GCS_BUCKET_NAME= @@ -14,6 +15,9 @@ MODE=development # disable authentication (for development only) AUTHENTIK_DISABLE_AUTHENTICATION=1 +# erase and rebuild the database for step tests +REBUILD_DB=1 + # authentik AUTHENTIK_URL= AUTHENTIK_CLIENT_ID= diff --git a/db/engine.py b/db/engine.py index d9e889d2f..bc177eb8e 100644 --- a/db/engine.py +++ b/db/engine.py @@ -109,7 +109,7 @@ def getconn(): # elif driver == "postgres": password = os.environ.get("POSTGRES_PASSWORD", "") host = os.environ.get("POSTGRES_HOST", "localhost") - port = os.environ.get("POSTGRES_PORT", "54321") + port = os.environ.get("POSTGRES_PORT", "5432") # Default to current OS user if POSTGRES_USER not set or empty user = os.environ.get("POSTGRES_USER", "").strip() or getpass.getuser() name = os.environ.get("POSTGRES_DB", "postgres") diff --git a/docker-compose.yml b/docker-compose.yml index 1c6dec4ef..30d22b9d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_HOST=db + - POSTGRES_PORT=5432 - MODE=${MODE} - AUTHENTIK_DISABLE_AUTHENTICATION=${AUTHENTIK_DISABLE_AUTHENTICATION} ports: