diff --git a/api/search.py b/api/search.py index 319a09719..a3ff05b73 100644 --- a/api/search.py +++ b/api/search.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== from fastapi import APIRouter -from sqlalchemy import select +from sqlalchemy import select, func, text from sqlalchemy.orm import Session from api.pagination import CustomPage from fastapi_pagination import paginate @@ -39,14 +39,14 @@ def _get_contact_results(session: Session, q: str, limit: int) -> list[dict]: vector = ( - Contact.search_vector - | Email.search_vector - | Phone.search_vector - | Address.search_vector + func.coalesce(Contact.search_vector, text("''::tsvector")) + .op("||")(func.coalesce(Email.search_vector, text("''::tsvector"))) + .op("||")(func.coalesce(Phone.search_vector, text("''::tsvector"))) + .op("||")(func.coalesce(Address.search_vector, text("''::tsvector"))) ) query = search( - select(Contact).join(Email).join(Phone).join(Address), + select(Contact).outerjoin(Email).outerjoin(Phone).outerjoin(Address), q, vector=vector, limit=limit, @@ -66,7 +66,6 @@ def _get_contact_results(session: Session, q: str, limit: int) -> list[dict]: } for c in contacts ] - return results diff --git a/core/app.py b/core/app.py index 2b018614b..3498b20ab 100644 --- a/core/app.py +++ b/core/app.py @@ -24,7 +24,7 @@ ) from fastapi.openapi.utils import get_openapi -from .initializers import init_db, init_lexicon +from .initializers import init_db, init_lexicon, init_parameter from .settings import settings @@ -36,6 +36,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: if settings.get_enum("MODE") == "development": init_db() init_lexicon() + init_parameter() yield diff --git a/core/initializers.py b/core/initializers.py index 233365c93..3d43e5787 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -20,6 +20,7 @@ from db import Base from db.engine import engine, session_ctx +from db.parameter import Parameter from services.lexicon_helper import add_lexicon_term, add_lexicon_category @@ -50,6 +51,36 @@ def init_hypertables(): # session.close() +def init_parameter(path: str = None) -> None: + """ + Populate the parameter table to allow their use in creating and editing + observations + """ + if path is None: + path = Path(__file__).parent / "parameter.json" + + with open(path) as f: + import json + + default_parameter = json.load(f) + + with session_ctx() as session: + for param in default_parameter: + try: + parameter_obj = Parameter( + parameter_name=param["parameter_name"], + matrix=param["matrix"], + parameter_type=param["parameter_type"], + cas_number=param["cas_number"], + default_unit=param["default_unit"], + ) + session.add(parameter_obj) + session.commit() + except DatabaseError as e: + print(f"Failed to add parameter {param['parameter_name']}: error: {e}") + session.rollback() + + def init_lexicon(path: str = None) -> None: if path is None: path = Path(__file__).parent / "lexicon.json" diff --git a/core/lexicon.json b/core/lexicon.json index fecc14419..3fc46d82f 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -20,11 +20,14 @@ {"name": "participant_role", "description": null}, {"name": "geochronology", "description": null}, {"name": "horizontal_datum", "description": null}, + {"name": "value_reason", "description": null}, + {"name": "limit_type", "description": null}, {"name": "groundwater_level_reason", "description": null}, {"name": "measurement_method", "description": null}, {"name": "monitoring_status", "description": null}, - {"name": "observed_property", "description": null}, + {"name": "parameter_name", "description": null}, {"name": "organization", "description": null}, + {"name": "parameter_type", "description": null}, {"name": "phone_type", "description": null}, {"name": "publication_type", "description": null}, {"name": "qc_type", "description": null}, @@ -131,123 +134,123 @@ {"categories": ["unit"], "term": "second", "definition": "second"}, {"categories": ["unit"], "term": "minute", "definition": "minute"}, {"categories": ["unit"], "term": "hour", "definition": "hour"}, - {"categories": ["observed_property"], "term": "groundwater level", "definition": "groundwater level measurement"}, - {"categories": ["observed_property"], "term": "temperature", "definition": "Temperature measurement"}, - {"categories": ["observed_property"], "term": "pH", "definition": "pH"}, - {"categories": ["observed_property"], "term": "Alkalinity, Total", "definition": "Alkalinity, Total"}, - {"categories": ["observed_property"], "term": "Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, - {"categories": ["observed_property"], "term": "Alkalinity as OH-", "definition": "Alkalinity as OH-"}, - {"categories": ["observed_property"], "term": "Calcium", "definition": "Calcium"}, - {"categories": ["observed_property"], "term": "Calcium, total, unfiltered", "definition": "Calcium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Chloride", "definition": "Chloride"}, - {"categories": ["observed_property"], "term": "Carbonate", "definition": "Carbonate"}, - {"categories": ["observed_property"], "term": "Conductivity, laboratory", "definition": "Conductivity, laboratory"}, - {"categories": ["observed_property"], "term": "Bicarbonate", "definition": "Bicarbonate"}, - {"categories": ["observed_property"], "term": "Hardness (CaCO3)", "definition": "Hardness (CaCO3)"}, - {"categories": ["observed_property"], "term": "Ion Balance", "definition": "Ion Balance"}, - {"categories": ["observed_property"], "term": "Potassium", "definition": "Potassium"}, - {"categories": ["observed_property"], "term": "Potassium, total, unfiltered", "definition": "Potassium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Magnesium", "definition": "Magnesium"}, - {"categories": ["observed_property"], "term": "Magnesium, total, unfiltered", "definition": "Magnesium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Sodium", "definition": "Sodium"}, - {"categories": ["observed_property"], "term": "Sodium, total, unfiltered", "definition": "Sodium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Sodium and Potassium combined", "definition": "Sodium and Potassium combined"}, - {"categories": ["observed_property"], "term": "Sulfate", "definition": "Sulfate"}, - {"categories": ["observed_property"], "term": "Total Anions", "definition": "Total Anions"}, - {"categories": ["observed_property"], "term": "Total Cations", "definition": "Total Cations"}, - {"categories": ["observed_property"], "term": "Total Dissolved Solids", "definition": "Total Dissolved Solids"}, - {"categories": ["observed_property"], "term": "Tritium", "definition": "Tritium"}, - {"categories": ["observed_property"], "term": "Age of Water using dissolved gases", "definition": "Age of Water using dissolved gases"}, - {"categories": ["observed_property"], "term": "Silver", "definition": "Silver"}, - {"categories": ["observed_property"], "term": "Silver, total, unfiltered", "definition": "Silver, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Aluminum", "definition": "Aluminum"}, - {"categories": ["observed_property"], "term": "Aluminum, total, unfiltered", "definition": "Aluminum, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Arsenic", "definition": "Arsenic"}, - {"categories": ["observed_property"], "term": "Arsenic, total, unfiltered", "definition": "Arsenic, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Boron", "definition": "Boron"}, - {"categories": ["observed_property"], "term": "Boron, total, unfiltered", "definition": "Boron, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Barium", "definition": "Barium"}, - {"categories": ["observed_property"], "term": "Barium, total, unfiltered", "definition": "Barium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Beryllium", "definition": "Beryllium"}, - {"categories": ["observed_property"], "term": "Beryllium, total, unfiltered", "definition": "Beryllium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Bromide", "definition": "Bromide"}, - {"categories": ["observed_property"], "term": "13C:12C ratio", "definition": "13C:12C ratio"}, - {"categories": ["observed_property"], "term": "14C content, pmc", "definition": "14C content, pmc"}, - {"categories": ["observed_property"], "term": "Uncorrected C14 age", "definition": "Uncorrected C14 age"}, - {"categories": ["observed_property"], "term": "Cadmium", "definition": "Cadmium"}, - {"categories": ["observed_property"], "term": "Cadmium, total, unfiltered", "definition": "Cadmium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Chlorofluorocarbon-11 avg age", "definition": "Chlorofluorocarbon-11 avg age"}, - {"categories": ["observed_property"], "term": "Chlorofluorocarbon-113 avg age", "definition": "Chlorofluorocarbon-113 avg age"}, - {"categories": ["observed_property"], "term": "Chlorofluorocarbon-113/12 avg RATIO age", "definition": "Chlorofluorocarbon-113/12 avg RATIO age"}, - {"categories": ["observed_property"], "term": "Chlorofluorocarbon-12 avg age", "definition": "Chlorofluorocarbon-12 avg age"}, - {"categories": ["observed_property"], "term": "Cobalt", "definition": "Cobalt"}, - {"categories": ["observed_property"], "term": "Cobalt, total, unfiltered", "definition": "Cobalt, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Chromium", "definition": "Chromium"}, - {"categories": ["observed_property"], "term": "Chromium, total, unfiltered", "definition": "Chromium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Copper", "definition": "Copper"}, - {"categories": ["observed_property"], "term": "Copper, total, unfiltered", "definition": "Copper, total, unfiltered"}, - {"categories": ["observed_property"], "term": "delta O18 sulfate", "definition": "delta O18 sulfate"}, - {"categories": ["observed_property"], "term": "Sulfate 34 isotope ratio", "definition": "Sulfate 34 isotope ratio"}, - {"categories": ["observed_property"], "term": "Fluoride", "definition": "Fluoride"}, - {"categories": ["observed_property"], "term": "Iron", "definition": "Iron"}, - {"categories": ["observed_property"], "term": "Iron, total, unfiltered", "definition": "Iron, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Deuterium:Hydrogen ratio", "definition": "Deuterium:Hydrogen ratio"}, - {"categories": ["observed_property"], "term": "Mercury", "definition": "Mercury"}, - {"categories": ["observed_property"], "term": "Mercury, total, unfiltered", "definition": "Mercury, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Lithium", "definition": "Lithium"}, - {"categories": ["observed_property"], "term": "Lithium, total, unfiltered", "definition": "Lithium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Manganese", "definition": "Manganese"}, - {"categories": ["observed_property"], "term": "Manganese, total, unfiltered", "definition": "Manganese, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Molybdenum", "definition": "Molybdenum"}, - {"categories": ["observed_property"], "term": "Molybdenum, total, unfiltered", "definition": "Molybdenum, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Nickel", "definition": "Nickel"}, - {"categories": ["observed_property"], "term": "Nickel, total, unfiltered", "definition": "Nickel, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Nitrite (as NO2)", "definition": "Nitrite (as NO2)"}, - {"categories": ["observed_property"], "term": "Nitrite (as N)", "definition": "Nitrite (as N)"}, - {"categories": ["observed_property"], "term": "Nitrate (as NO3)", "definition": "Nitrate (as NO3)"}, - {"categories": ["observed_property"], "term": "Nitrate (as N)", "definition": "Nitrate (as N)"}, - {"categories": ["observed_property"], "term": "18O:16O ratio", "definition": "18O:16O ratio"}, - {"categories": ["observed_property"], "term": "Lead", "definition": "Lead"}, - {"categories": ["observed_property"], "term": "Lead, total, unfiltered", "definition": "Lead, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Phosphate", "definition": "Phosphate"}, - {"categories": ["observed_property"], "term": "Antimony", "definition": "Antimony"}, - {"categories": ["observed_property"], "term": "Antimony, total, unfiltered", "definition": "Antimony, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Selenium", "definition": "Selenium"}, - {"categories": ["observed_property"], "term": "Selenium, total, unfiltered", "definition": "Selenium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Sulfur hexafluoride", "definition": "Sulfur hexafluoride"}, - {"categories": ["observed_property"], "term": "Silicon", "definition": "Silicon"}, - {"categories": ["observed_property"], "term": "Silicon, total, unfiltered", "definition": "Silicon, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Silica", "definition": "Silica"}, - {"categories": ["observed_property"], "term": "Tin", "definition": "Tin"}, - {"categories": ["observed_property"], "term": "Tin, total, unfiltered", "definition": "Tin, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Strontium", "definition": "Strontium"}, - {"categories": ["observed_property"], "term": "Strontium, total, unfiltered", "definition": "Strontium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Strontium 87:86 ratio", "definition": "Strontium 87:86 ratio"}, - {"categories": ["observed_property"], "term": "Thorium", "definition": "Thorium"}, - {"categories": ["observed_property"], "term": "Thorium, total, unfiltered", "definition": "Thorium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Titanium", "definition": "Titanium"}, - {"categories": ["observed_property"], "term": "Titanium, total, unfiltered", "definition": "Titanium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Thallium", "definition": "Thallium"}, - {"categories": ["observed_property"], "term": "Thallium, total, unfiltered", "definition": "Thallium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Uranium (total, by ICP-MS)", "definition": "Uranium (total, by ICP-MS)"}, - {"categories": ["observed_property"], "term": "Uranium, total, unfiltered", "definition": "Uranium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Vanadium", "definition": "Vanadium"}, - {"categories": ["observed_property"], "term": "Vanadium, total, unfiltered", "definition": "Vanadium, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Zinc", "definition": "Zinc"}, - {"categories": ["observed_property"], "term": "Zinc, total, unfiltered", "definition": "Zinc, total, unfiltered"}, - {"categories": ["observed_property"], "term": "Corrected C14 in years", "definition": "Corrected C14 in years"}, - {"categories": ["observed_property"], "term": "Arsenite (arsenic species)", "definition": "Arsenite (arsenic species)"}, - {"categories": ["observed_property"], "term": "Arsenate (arsenic species)", "definition": "Arsenate (arsenic species)"}, - {"categories": ["observed_property"], "term": "Cyanide", "definition": "Cyanide"}, - {"categories": ["observed_property"], "term": "Estimated recharge temperature", "definition": "Estimated recharge temperature"}, - {"categories": ["observed_property"], "term": "Hydrogen sulfide", "definition": "Hydrogen sulfide"}, - {"categories": ["observed_property"], "term": "Ammonia", "definition": "Ammonia"}, - {"categories": ["observed_property"], "term": "Ammonium", "definition": "Ammonium"}, - {"categories": ["observed_property"], "term": "Total nitrogen", "definition": "Total nitrogen"}, - {"categories": ["observed_property"], "term": "Total Kjeldahl nitrogen", "definition": "Total Kjeldahl nitrogen"}, - {"categories": ["observed_property"], "term": "Dissolved organic carbon", "definition": "Dissolved organic carbon"}, - {"categories": ["observed_property"], "term": "Total organic carbon", "definition": "Total organic carbon"}, - {"categories": ["observed_property"], "term": "delta C13 of dissolved inorganic carbon", "definition": "delta C13 of dissolved inorganic carbon"}, + {"categories": ["parameter_name"], "term": "groundwater level", "definition": "groundwater level measurement"}, + {"categories": ["parameter_name"], "term": "temperature", "definition": "Temperature measurement"}, + {"categories": ["parameter_name"], "term": "pH", "definition": "pH"}, + {"categories": ["parameter_name"], "term": "Alkalinity, Total", "definition": "Alkalinity, Total"}, + {"categories": ["parameter_name"], "term": "Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, + {"categories": ["parameter_name"], "term": "Alkalinity as OH-", "definition": "Alkalinity as OH-"}, + {"categories": ["parameter_name"], "term": "Calcium", "definition": "Calcium"}, + {"categories": ["parameter_name"], "term": "Calcium, total, unfiltered", "definition": "Calcium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Chloride", "definition": "Chloride"}, + {"categories": ["parameter_name"], "term": "Carbonate", "definition": "Carbonate"}, + {"categories": ["parameter_name"], "term": "Conductivity, laboratory", "definition": "Conductivity, laboratory"}, + {"categories": ["parameter_name"], "term": "Bicarbonate", "definition": "Bicarbonate"}, + {"categories": ["parameter_name"], "term": "Hardness (CaCO3)", "definition": "Hardness (CaCO3)"}, + {"categories": ["parameter_name"], "term": "Ion Balance", "definition": "Ion Balance"}, + {"categories": ["parameter_name"], "term": "Potassium", "definition": "Potassium"}, + {"categories": ["parameter_name"], "term": "Potassium, total, unfiltered", "definition": "Potassium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Magnesium", "definition": "Magnesium"}, + {"categories": ["parameter_name"], "term": "Magnesium, total, unfiltered", "definition": "Magnesium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Sodium", "definition": "Sodium"}, + {"categories": ["parameter_name"], "term": "Sodium, total, unfiltered", "definition": "Sodium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Sodium and Potassium combined", "definition": "Sodium and Potassium combined"}, + {"categories": ["parameter_name"], "term": "Sulfate", "definition": "Sulfate"}, + {"categories": ["parameter_name"], "term": "Total Anions", "definition": "Total Anions"}, + {"categories": ["parameter_name"], "term": "Total Cations", "definition": "Total Cations"}, + {"categories": ["parameter_name"], "term": "Total Dissolved Solids", "definition": "Total Dissolved Solids"}, + {"categories": ["parameter_name"], "term": "Tritium", "definition": "Tritium"}, + {"categories": ["parameter_name"], "term": "Age of Water using dissolved gases", "definition": "Age of Water using dissolved gases"}, + {"categories": ["parameter_name"], "term": "Silver", "definition": "Silver"}, + {"categories": ["parameter_name"], "term": "Silver, total, unfiltered", "definition": "Silver, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Aluminum", "definition": "Aluminum"}, + {"categories": ["parameter_name"], "term": "Aluminum, total, unfiltered", "definition": "Aluminum, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Arsenic", "definition": "Arsenic"}, + {"categories": ["parameter_name"], "term": "Arsenic, total, unfiltered", "definition": "Arsenic, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Boron", "definition": "Boron"}, + {"categories": ["parameter_name"], "term": "Boron, total, unfiltered", "definition": "Boron, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Barium", "definition": "Barium"}, + {"categories": ["parameter_name"], "term": "Barium, total, unfiltered", "definition": "Barium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Beryllium", "definition": "Beryllium"}, + {"categories": ["parameter_name"], "term": "Beryllium, total, unfiltered", "definition": "Beryllium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Bromide", "definition": "Bromide"}, + {"categories": ["parameter_name"], "term": "13C:12C ratio", "definition": "13C:12C ratio"}, + {"categories": ["parameter_name"], "term": "14C content, pmc", "definition": "14C content, pmc"}, + {"categories": ["parameter_name"], "term": "Uncorrected C14 age", "definition": "Uncorrected C14 age"}, + {"categories": ["parameter_name"], "term": "Cadmium", "definition": "Cadmium"}, + {"categories": ["parameter_name"], "term": "Cadmium, total, unfiltered", "definition": "Cadmium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Chlorofluorocarbon-11 avg age", "definition": "Chlorofluorocarbon-11 avg age"}, + {"categories": ["parameter_name"], "term": "Chlorofluorocarbon-113 avg age", "definition": "Chlorofluorocarbon-113 avg age"}, + {"categories": ["parameter_name"], "term": "Chlorofluorocarbon-113/12 avg RATIO age", "definition": "Chlorofluorocarbon-113/12 avg RATIO age"}, + {"categories": ["parameter_name"], "term": "Chlorofluorocarbon-12 avg age", "definition": "Chlorofluorocarbon-12 avg age"}, + {"categories": ["parameter_name"], "term": "Cobalt", "definition": "Cobalt"}, + {"categories": ["parameter_name"], "term": "Cobalt, total, unfiltered", "definition": "Cobalt, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Chromium", "definition": "Chromium"}, + {"categories": ["parameter_name"], "term": "Chromium, total, unfiltered", "definition": "Chromium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Copper", "definition": "Copper"}, + {"categories": ["parameter_name"], "term": "Copper, total, unfiltered", "definition": "Copper, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "delta O18 sulfate", "definition": "delta O18 sulfate"}, + {"categories": ["parameter_name"], "term": "Sulfate 34 isotope ratio", "definition": "Sulfate 34 isotope ratio"}, + {"categories": ["parameter_name"], "term": "Fluoride", "definition": "Fluoride"}, + {"categories": ["parameter_name"], "term": "Iron", "definition": "Iron"}, + {"categories": ["parameter_name"], "term": "Iron, total, unfiltered", "definition": "Iron, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Deuterium:Hydrogen ratio", "definition": "Deuterium:Hydrogen ratio"}, + {"categories": ["parameter_name"], "term": "Mercury", "definition": "Mercury"}, + {"categories": ["parameter_name"], "term": "Mercury, total, unfiltered", "definition": "Mercury, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Lithium", "definition": "Lithium"}, + {"categories": ["parameter_name"], "term": "Lithium, total, unfiltered", "definition": "Lithium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Manganese", "definition": "Manganese"}, + {"categories": ["parameter_name"], "term": "Manganese, total, unfiltered", "definition": "Manganese, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Molybdenum", "definition": "Molybdenum"}, + {"categories": ["parameter_name"], "term": "Molybdenum, total, unfiltered", "definition": "Molybdenum, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Nickel", "definition": "Nickel"}, + {"categories": ["parameter_name"], "term": "Nickel, total, unfiltered", "definition": "Nickel, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Nitrite (as NO2)", "definition": "Nitrite (as NO2)"}, + {"categories": ["parameter_name"], "term": "Nitrite (as N)", "definition": "Nitrite (as N)"}, + {"categories": ["parameter_name"], "term": "Nitrate (as NO3)", "definition": "Nitrate (as NO3)"}, + {"categories": ["parameter_name"], "term": "Nitrate (as N)", "definition": "Nitrate (as N)"}, + {"categories": ["parameter_name"], "term": "18O:16O ratio", "definition": "18O:16O ratio"}, + {"categories": ["parameter_name"], "term": "Lead", "definition": "Lead"}, + {"categories": ["parameter_name"], "term": "Lead, total, unfiltered", "definition": "Lead, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Phosphate", "definition": "Phosphate"}, + {"categories": ["parameter_name"], "term": "Antimony", "definition": "Antimony"}, + {"categories": ["parameter_name"], "term": "Antimony, total, unfiltered", "definition": "Antimony, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Selenium", "definition": "Selenium"}, + {"categories": ["parameter_name"], "term": "Selenium, total, unfiltered", "definition": "Selenium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Sulfur hexafluoride", "definition": "Sulfur hexafluoride"}, + {"categories": ["parameter_name"], "term": "Silicon", "definition": "Silicon"}, + {"categories": ["parameter_name"], "term": "Silicon, total, unfiltered", "definition": "Silicon, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Silica", "definition": "Silica"}, + {"categories": ["parameter_name"], "term": "Tin", "definition": "Tin"}, + {"categories": ["parameter_name"], "term": "Tin, total, unfiltered", "definition": "Tin, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Strontium", "definition": "Strontium"}, + {"categories": ["parameter_name"], "term": "Strontium, total, unfiltered", "definition": "Strontium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Strontium 87:86 ratio", "definition": "Strontium 87:86 ratio"}, + {"categories": ["parameter_name"], "term": "Thorium", "definition": "Thorium"}, + {"categories": ["parameter_name"], "term": "Thorium, total, unfiltered", "definition": "Thorium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Titanium", "definition": "Titanium"}, + {"categories": ["parameter_name"], "term": "Titanium, total, unfiltered", "definition": "Titanium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Thallium", "definition": "Thallium"}, + {"categories": ["parameter_name"], "term": "Thallium, total, unfiltered", "definition": "Thallium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Uranium (total, by ICP-MS)", "definition": "Uranium (total, by ICP-MS)"}, + {"categories": ["parameter_name"], "term": "Uranium, total, unfiltered", "definition": "Uranium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Vanadium", "definition": "Vanadium"}, + {"categories": ["parameter_name"], "term": "Vanadium, total, unfiltered", "definition": "Vanadium, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Zinc", "definition": "Zinc"}, + {"categories": ["parameter_name"], "term": "Zinc, total, unfiltered", "definition": "Zinc, total, unfiltered"}, + {"categories": ["parameter_name"], "term": "Corrected C14 in years", "definition": "Corrected C14 in years"}, + {"categories": ["parameter_name"], "term": "Arsenite (arsenic species)", "definition": "Arsenite (arsenic species)"}, + {"categories": ["parameter_name"], "term": "Arsenate (arsenic species)", "definition": "Arsenate (arsenic species)"}, + {"categories": ["parameter_name"], "term": "Cyanide", "definition": "Cyanide"}, + {"categories": ["parameter_name"], "term": "Estimated recharge temperature", "definition": "Estimated recharge temperature"}, + {"categories": ["parameter_name"], "term": "Hydrogen sulfide", "definition": "Hydrogen sulfide"}, + {"categories": ["parameter_name"], "term": "Ammonia", "definition": "Ammonia"}, + {"categories": ["parameter_name"], "term": "Ammonium", "definition": "Ammonium"}, + {"categories": ["parameter_name"], "term": "Total nitrogen", "definition": "Total nitrogen"}, + {"categories": ["parameter_name"], "term": "Total Kjeldahl nitrogen", "definition": "Total Kjeldahl nitrogen"}, + {"categories": ["parameter_name"], "term": "Dissolved organic carbon", "definition": "Dissolved organic carbon"}, + {"categories": ["parameter_name"], "term": "Total organic carbon", "definition": "Total organic carbon"}, + {"categories": ["parameter_name"], "term": "delta C13 of dissolved inorganic carbon", "definition": "delta C13 of dissolved inorganic carbon"}, {"categories": ["release_status"], "term": "draft", "definition": "draft version"}, {"categories": ["release_status"], "term": "provisional", "definition": "provisional version"}, {"categories": ["release_status"], "term": "final", "definition": "final version"}, @@ -267,6 +270,7 @@ {"categories": ["participant_role"], "term": "Observer", "definition": "a person observing the field event"}, {"categories": ["participant_role"], "term": "Visitor", "definition": "a person visiting the field event"}, {"categories": ["sample_matrix"], "term": "water", "definition": "water"}, + {"categories": ["sample_matrix"], "term": "groundwater", "definition": "groundwater"}, {"categories": ["sample_matrix"], "term": "soil", "definition": "soil"}, {"categories": ["thing_type"], "term": "observation well", "definition": "a well used to monitor groundwater levels"}, {"categories": ["thing_type"], "term": "piezometer", "definition": "a type of observation well that measures pressure head in the aquifer"}, @@ -486,6 +490,19 @@ {"categories": ["sample_type"], "term": "Repeat sample", "definition": "Repeat sample"}, {"categories": ["sample_type"], "term": "Standard field sample", "definition": "Standard field sample"}, {"categories": ["sample_type"], "term": "Soil or Rock sample", "definition": "Soil or Rock sample"}, - {"categories": ["sample_type"], "term": "Source water blank", "definition": "Source water blank"} + {"categories": ["sample_type"], "term": "Source water blank", "definition": "Source water blank"}, + {"categories": ["limit_type"], "term": "MCL", "definition": "Maximum Contaminant Level. The highest level of a contaminant that is legally allowed in public drinking water systems under the Safe Drinking Water Act. This is an enforceable standard."}, + {"categories": ["limit_type"], "term": "SMCL", "definition": "Secondary Maximum Contaminant Level. Non-enforceable guidelines regulating contaminants that may cause cosmetic or aesthetic effects in drinking water."}, + {"categories": ["limit_type"], "term": "GWQS", "definition": "Groundwater Quality Standard. State-specific standards that define acceptable levels of various contaminants in groundwater, often used for regulatory and remediation purposes. These can be stricter than or in addition to federal standards."}, + {"categories": ["limit_type"], "term": "MRL", "definition": "Method Reporting Level. The lowest concentration of an analyte that a laboratory can reliably quantify within specified limits of precision and accuracy for a given analytical method. This is the most common 'limit of detection' you will see on a final lab report. Often used interchangeably with PQL."}, + {"categories": ["limit_type"], "term": "PQL", "definition": "Practical Quantitation Limit. Similar to the MRL, this is the lowest concentration achievable by a lab during routine operating conditions. It represents the practical, real-world limit of quantification."}, + {"categories": ["limit_type"], "term": "MDL", "definition": "Method Detection Limit. The minimum measured concentration of a substance that can be reported with 99% confidence that the analyte concentration is greater than zero. It is a statistical value determined under ideal lab conditions and is typically lower than the MRL/PQL."}, + {"categories": ["limit_type"], "term": "RL", "definition": "Reporting Limit. A generic term often used by labs to mean their MRL or PQL. It is the lowest concentration they are willing to report as a quantitative result."}, + {"categories": ["parameter_type"], "term": "Field Parameter", "definition": "Field Parameter"}, + {"categories": ["parameter_type"], "term": "Metal", "definition": "Metal"}, + {"categories": ["parameter_type"], "term": "Radionuclide", "definition": "Radionuclide"}, + {"categories": ["parameter_type"], "term": "Major Element", "definition": "Major Element"}, + {"categories": ["parameter_type"], "term": "Minor Element", "definition": "Minor Element"}, + {"categories": ["parameter_type"], "term": "Physical property", "definition": "Physical property"} ] } \ No newline at end of file diff --git a/core/parameter.json b/core/parameter.json new file mode 100644 index 000000000..5d9b7db7a --- /dev/null +++ b/core/parameter.json @@ -0,0 +1,16 @@ +[ + { + "parameter_name": "groundwater level", + "matrix": "groundwater", + "parameter_type": "Field Parameter", + "cas_number": null, + "default_unit": "ft" + }, + { + "parameter_name": "pH", + "matrix": "groundwater", + "parameter_type": "Field Parameter", + "cas_number": null, + "default_unit": "dimensionless" + } +] \ No newline at end of file diff --git a/db/__init__.py b/db/__init__.py index 3cda51bd5..7a7b20fa6 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -31,8 +31,10 @@ from db.lexicon import * from db.location import * from db.observation import * +from db.parameter import * from db.permission import * from db.publication import * +from db.regulatory_limit import * from db.sample import * from db.sensor import * from db.status_history import * diff --git a/db/lexicon.py b/db/lexicon.py index ba03cec5c..a632661c5 100644 --- a/db/lexicon.py +++ b/db/lexicon.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== from sqlalchemy import String, ForeignKey, Integer -from sqlalchemy.orm import mapped_column, relationship +from sqlalchemy.orm import mapped_column, relationship, Mapped from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from db.base import AutoBaseMixin, Base, lexicon_term @@ -27,8 +27,8 @@ class LexiconTerm(Base, AutoBaseMixin): """ __tablename__ = "lexicon_term" - term = mapped_column(String(100), unique=True, nullable=False) - definition = mapped_column(String(255), nullable=False) + term: Mapped[str] = mapped_column(unique=True, nullable=False) + definition: Mapped[str] = mapped_column(nullable=False) category_associations = relationship( "LexiconTermCategoryAssociation", diff --git a/db/observation.py b/db/observation.py index ded414d28..90971bc52 100644 --- a/db/observation.py +++ b/db/observation.py @@ -29,6 +29,7 @@ from db.sample import Sample from db.sensor import Sensor from db.analysis_method import AnalysisMethod + from db.parameter import Parameter class Observation(Base, AutoBaseMixin, ReleaseMixin): @@ -50,11 +51,14 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): Integer, ForeignKey("analysis_method.id"), nullable=True ) + parameter_id: Mapped[int] = mapped_column( + Integer, ForeignKey("parameter.id"), nullable=False + ) + # --- Columns --- observation_datetime: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, doc="Timestamp of the observation" ) - observed_property: Mapped[str] = lexicon_term(nullable=False) value: Mapped[float] = mapped_column( nullable=True, ) @@ -88,5 +92,10 @@ class Observation(Base, AutoBaseMixin, ReleaseMixin): "AnalysisMethod", back_populates="observations" ) + # Many-To-One: An Observation measures one Parameter. + parameter: Mapped["Parameter"] = relationship( + "Parameter", back_populates="observations", lazy="joined" + ) + # ============= EOF ============================================= diff --git a/db/parameter.py b/db/parameter.py new file mode 100644 index 000000000..a128bacdc --- /dev/null +++ b/db/parameter.py @@ -0,0 +1,67 @@ +""" +This table is a controlled vocabulary for all analytes, properties, and +characteristics that can be measured or observed. +""" + +from typing import List, TYPE_CHECKING + +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.observation import Observation + from db.regulatory_limit import RegulatoryLimit + + +class Parameter(Base, AutoBaseMixin, ReleaseMixin): + """ + + Represents an analyte or property that can be measured (e.g., Chloride). + """ + + __versioned__ = {} + + # --- Columns --- + # TODO: Parameter names are currently associated with the 'observed_property' category in the lexicon. Should we update the lexicon category name to 'parameter_name'? + parameter_name: Mapped[str] = lexicon_term( + nullable=False, + comment="The official, full name of the parameter (e.g., 'Arsenic, Dissolved').", + ) + matrix: Mapped[str] = lexicon_term( + nullable=False, + comment="A controlled vocabulary field defining the physical medium the analyte is measured in (e.g., 'Water', 'Soil', 'Air').", + ) + parameter_type: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary field defining the category of the parameter (e.g., 'Metals', 'Nutrients', 'Field Parameter'). Used for grouping and filtering.", + ) + cas_number: Mapped[str] = mapped_column( + nullable=True, + comment="The Chemical Abstracts Service (CAS) registry number, a globally unique identifier for a chemical substance.", + ) + default_unit: Mapped[str] = lexicon_term( + nullable=False, + comment="The standard, preferred unit for reporting this parameter (e.g., 'ug/L', 'mg/L', 'pH units').", + ) + + # --- Relationships --- + # One-To-Many: A Parameter can have many Observations. + observations: Mapped[List["Observation"]] = relationship( + "Observation", back_populates="parameter" + ) + + # One-To-Many: A Parameter can have many associated RegulatoryLimits. + # If a Parameter is deleted, all its associated limits are deleted as well. + regulatory_limits: Mapped[List["RegulatoryLimit"]] = relationship( + "RegulatoryLimit", back_populates="parameter", cascade="all, delete-orphan" + ) + + # --- Table Arguments --- + # An analyte is defined by its name and matrix. This constraint + # ensures a single, specific analyte can only be defined once. + from sqlalchemy import UniqueConstraint + + __table_args__ = ( + UniqueConstraint("parameter_name", "matrix", name="uq_parameter_name_matrix"), + ) diff --git a/db/regulatory_limit.py b/db/regulatory_limit.py new file mode 100644 index 000000000..f72643c60 --- /dev/null +++ b/db/regulatory_limit.py @@ -0,0 +1,50 @@ +""" +This table stores the various regulatory or health-based limits for a given +parameter, sourced from different agencies or standards. + +The purpose of this table is to solve the real-world problem where a single +chemical (`Parameter`) can have multiple different limits set by various agencies +(e.g., a federal EPA limit and a state-level NMED limit). +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, Numeric, ForeignKey +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.parameter import Parameter + + +class RegulatoryLimit(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a single, citable regulatory or health-based limit for a + specific Parameter. + """ + + __versioned__ = {} + + # --- Foreign Keys --- + parameter_id: Mapped[int] = mapped_column( + Integer, ForeignKey("parameter.id"), nullable=False + ) + + # --- Columns --- + limit_source: Mapped[str] = lexicon_term( + nullable=False, + comment="The official source of the limit (e.g., 'EPA', 'NMED', 'EPA').", + ) + limit_value: Mapped[float] = mapped_column(Numeric, nullable=False) + limit_unit: Mapped[str] = lexicon_term(nullable=False) + limit_type: Mapped[str] = lexicon_term( + nullable=True, + comment="A controlled vocabulary field to categorize the limit (e.g., 'MCL', 'PQL', 'MDL', etc.).", + ) + + # --- Relationships --- + # Many-To-One: A RegulatoryLimit is for one Parameter. + parameter: Mapped["Parameter"] = relationship( + "Parameter", back_populates="regulatory_limits" + ) diff --git a/schemas/observation.py b/schemas/observation.py index 67fc247c2..06d5cb4fa 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -25,6 +25,7 @@ from typing_extensions import Self from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel +from schemas.parameter import ParameterResponse # class GeothermalMixin: @@ -36,7 +37,7 @@ class ValidateObservation(BaseModel): - observed_property: str + parameter_id: int observation_datetime: AwareDatetime @field_validator("observation_datetime", check_fields=False) @@ -60,7 +61,7 @@ class CreateBaseObservation(BaseCreateModel, ValidateObservation): observation_datetime: Annotated[AwareDatetime, PastDatetime()] sample_id: int sensor_id: int - observed_property: str + parameter_id: int release_status: str value: float | None unit: str | None @@ -82,7 +83,7 @@ class UpdateBaseObservation(BaseUpdateModel, ValidateObservation): observation_datetime: Annotated[AwareDatetime, PastDatetime()] | None = None sample_id: int | None = None sensor_id: int | None = None - observed_property: str | None = None + parameter_id: int | None = None release_status: str | None = None value: float | None | None = None unit: str | None = None @@ -98,11 +99,12 @@ class UpdateWaterChemistryObservation(UpdateBaseObservation): # -------- RESPONSE ---------- +# TODO: Return full sample and sensor objects class BaseObservationResponse(BaseResponseModel): sample_id: int sensor_id: int observation_datetime: AwareDatetime - observed_property: str + parameter: ParameterResponse release_status: str value: float | None unit: str diff --git a/schemas/parameter.py b/schemas/parameter.py new file mode 100644 index 000000000..4e3cea140 --- /dev/null +++ b/schemas/parameter.py @@ -0,0 +1,15 @@ +from schemas import BaseResponseModel + + +# -------- RESPONSE ------- +class ParameterResponse(BaseResponseModel): + """ + Pydantic model for the response of a parameter. + This model can be extended to include additional fields as needed. + """ + + parameter_name: str + matrix: str + parameter_type: str | None + cas_number: str | None + default_unit: str diff --git a/tests/__init__.py b/tests/__init__.py index 985e4ae10..02566806e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -15,8 +15,8 @@ # =============================================================================== from fastapi.testclient import TestClient -from core.initializers import init_lexicon -from db import Base +from core.initializers import init_lexicon, init_parameter +from db import Base, Parameter from db.engine import engine, session_ctx from main import app @@ -25,9 +25,23 @@ Base.metadata.create_all(engine) init_lexicon() +init_parameter() client = TestClient(app) +# map (name, type) to id for easy lookup in tests +parameter_map = {} +with session_ctx() as session: + for param in session.query(Parameter).all(): + if ( + param.parameter_name in ["groundwater level", "pH"] + and param.parameter_type == "Field Parameter" + ): + parameter_map[(param.parameter_name, param.parameter_type)] = param.id + +groundwater_level_parameter_id = parameter_map[("groundwater level", "Field Parameter")] +pH_parameter_id = parameter_map[("pH", "Field Parameter")] + def override_authentication(default=True): """ diff --git a/tests/conftest.py b/tests/conftest.py index 8009883bb..6373148f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,9 +3,10 @@ from db import * from db.engine import session_ctx +from tests import groundwater_level_parameter_id, pH_parameter_id -@pytest.fixture(scope="session") +@pytest.fixture() def location(): with session_ctx() as session: loc = Location( @@ -26,8 +27,8 @@ def location(): session.commit() session.refresh(loc) yield loc - - session.close() + session.delete(loc) + session.commit() @pytest.fixture(scope="function") @@ -46,7 +47,7 @@ def second_location(): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def water_well_thing(location): with session_ctx() as session: water_well = Thing( @@ -70,9 +71,12 @@ def water_well_thing(location): session.add(assoc) session.commit() yield water_well + session.delete(water_well) + session.delete(assoc) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def well_screen(water_well_thing): with session_ctx() as session: screen = WellScreen( @@ -86,6 +90,8 @@ def well_screen(water_well_thing): session.add(screen) session.commit() yield screen + session.delete(screen) + session.commit() @pytest.fixture(scope="function") @@ -106,7 +112,7 @@ def second_well_screen(water_well_thing): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def thing_id_link(water_well_thing): with session_ctx() as session: id_link = ThingIdLink( @@ -119,6 +125,8 @@ def thing_id_link(water_well_thing): session.add(id_link) session.commit() yield id_link + session.delete(id_link) + session.commit() @pytest.fixture(scope="function") @@ -138,7 +146,7 @@ def second_thing_id_link(water_well_thing): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def spring_thing(location): with session_ctx() as session: spring = Thing( @@ -158,6 +166,9 @@ def spring_thing(location): session.add(assoc) session.commit() yield spring + session.delete(spring) + session.delete(assoc) + session.commit() @pytest.fixture(scope="function") @@ -181,10 +192,11 @@ def second_spring_thing(location): session.commit() yield spring session.delete(spring) + session.delete(assoc) session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def sensor(): with session_ctx() as session: sensor = Sensor( @@ -200,6 +212,8 @@ def sensor(): session.add(sensor) session.commit() yield sensor + session.delete(sensor) + session.commit() @pytest.fixture(scope="function") @@ -222,7 +236,7 @@ def second_sensor(): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def sensor_to_water_well_thing_deployment(sensor, water_well_thing): with session_ctx() as session: deployment = Deployment( @@ -240,9 +254,11 @@ def sensor_to_water_well_thing_deployment(sensor, water_well_thing): session.add(deployment) session.commit() yield deployment + session.delete(deployment) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def contact(water_well_thing): with session_ctx() as session: contact = Contact( @@ -264,9 +280,12 @@ def contact(water_well_thing): session.refresh(association) yield contact + session.delete(contact) + session.delete(association) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def address(contact): with session_ctx() as session: address = Address( @@ -284,9 +303,11 @@ def address(contact): session.commit() session.refresh(address) yield address + session.delete(address) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def email(contact): with session_ctx() as session: email = Email( @@ -299,9 +320,11 @@ def email(contact): session.commit() session.refresh(email) yield email + session.delete(email) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def phone(contact): with session_ctx() as session: phone = Phone( @@ -314,6 +337,8 @@ def phone(contact): session.commit() session.refresh(phone) yield phone + session.delete(phone) + session.commit() @pytest.fixture(scope="function") @@ -412,7 +437,7 @@ def third_contact(): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def asset(): with session_ctx() as session: asset = Asset( @@ -429,6 +454,8 @@ def asset(): session.commit() session.refresh(asset) yield asset + session.delete(asset) + session.commit() @pytest.fixture(scope="function") @@ -481,7 +508,7 @@ def second_asset(): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def field_event(water_well_thing): with session_ctx() as session: field_event = FieldEvent( @@ -493,9 +520,11 @@ def field_event(water_well_thing): session.add(field_event) session.commit() yield field_event + session.delete(field_event) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def field_event_participant(field_event, contact): with session_ctx() as session: field_event_participant = FieldEventParticipant( @@ -506,9 +535,11 @@ def field_event_participant(field_event, contact): session.add(field_event_participant) session.commit() yield field_event_participant + session.delete(field_event_participant) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def groundwater_level_field_activity(field_event): with session_ctx() as session: field_activity = FieldActivity( @@ -520,9 +551,11 @@ def groundwater_level_field_activity(field_event): session.add(field_activity) session.commit() yield field_activity + session.delete(field_activity) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def water_chemistry_field_activity(field_event): with session_ctx() as session: field_activity = FieldActivity( @@ -534,9 +567,11 @@ def water_chemistry_field_activity(field_event): session.add(field_activity) session.commit() yield field_activity + session.delete(field_activity) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def groundwater_level_sample(groundwater_level_field_activity, field_event_participant): with session_ctx() as session: sample = Sample( @@ -555,9 +590,11 @@ def groundwater_level_sample(groundwater_level_field_activity, field_event_parti session.add(sample) session.commit() yield sample + session.delete(sample) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def water_chemistry_sample(water_chemistry_field_activity, field_event_participant): with session_ctx() as session: sample = Sample( @@ -576,6 +613,8 @@ def water_chemistry_sample(water_chemistry_field_activity, field_event_participa session.add(sample) session.commit() yield sample + session.delete(sample) + session.commit() @pytest.fixture(scope="function") @@ -601,14 +640,14 @@ def sample_to_delete(water_chemistry_field_activity, field_event_participant): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def groundwater_level_observation(sensor, groundwater_level_sample): with session_ctx() as session: observation = Observation( observation_datetime="2025-01-01T00:04:00Z", sample_id=groundwater_level_sample.id, sensor_id=sensor.id, - observed_property="groundwater level", + parameter_id=groundwater_level_parameter_id, release_status="draft", value=10.0, unit="ft", @@ -618,16 +657,18 @@ def groundwater_level_observation(sensor, groundwater_level_sample): session.add(observation) session.commit() yield observation + session.delete(observation) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def water_chemistry_observation(sensor, water_chemistry_sample): with session_ctx() as session: observation = Observation( observation_datetime="2025-01-01T00:03:00Z", sample_id=water_chemistry_sample.id, sensor_id=sensor.id, - observed_property="pH", + parameter_id=pH_parameter_id, release_status="draft", value=4.0, unit="dimensionless", @@ -635,6 +676,28 @@ def water_chemistry_observation(sensor, water_chemistry_sample): session.add(observation) session.commit() yield observation + session.delete(observation) + session.commit() + + +@pytest.fixture() +def geothermal_observation(sensor, geothermal_sample): + with session_ctx() as session: + observation = Observation( + observation_datetime="2025-01-01T00:02:00Z", + sample_id=geothermal_sample.id, + sensor_id=sensor.id, + observed_property="temperature", + release_status="draft", + value=20.0, + unit="deg C", + observation_depth=200.0, + ) + session.add(observation) + session.commit() + yield observation + session.delete(observation) + session.commit() @pytest.fixture(scope="function") @@ -644,7 +707,7 @@ def observation_to_delete(water_chemistry_sample, sensor): observation_datetime="2019-01-01T00:03:00Z", sample_id=water_chemistry_sample.id, sensor_id=sensor.id, - observed_property="pH", + parameter_id=pH_parameter_id, release_status="draft", value=4.0, unit="dimensionless", @@ -652,9 +715,53 @@ def observation_to_delete(water_chemistry_sample, sensor): session.add(observation) session.commit() yield observation - - -@pytest.fixture(scope="session") + session.delete(observation) + session.commit() + + +# @pytest.fixture() +# def parameter_water_chemistry(): +# """ +# Fixture to create a Parameter for testing. +# """ +# with session_ctx() as session: +# parameter = Parameter( +# parameter_name="pH", +# parameter_type="Field Parameter", +# matrix="groundwater", +# cas_number="7440-38-2", +# default_unit="dimensionless", +# release_status="draft", +# ) +# session.add(parameter) +# session.commit() +# yield parameter +# session.delete(parameter) +# session.commit() + + +# @pytest.fixture() +# def parameter_groundwater(): +# """ +# Fixture to create a Parameter for testing. +# """ +# with session_ctx() as session: +# parameter = Parameter( +# parameter_name="groundwater level", +# parameter_type="Field Parameter", +# matrix="groundwater", +# cas_number=None, +# default_unit="ft", +# release_status="draft", +# ) +# session.add(parameter) +# session.commit() +# yield parameter +# session.delete(parameter) +# session.commit() + + +@pytest.fixture() def group(water_well_thing): with session_ctx() as session: group = Group( @@ -676,6 +783,9 @@ def group(water_well_thing): session.refresh(group_thing_association) yield group + session.delete(group) + session.delete(group_thing_association) + session.commit() @pytest.fixture(scope="function") @@ -701,8 +811,12 @@ def second_group(water_well_thing): yield group + session.delete(group) + session.delete(group_thing_association) + session.commit() + -@pytest.fixture(scope="session") +@pytest.fixture() def lexicon_category(): with session_ctx() as session: category = LexiconCategory( @@ -712,6 +826,8 @@ def lexicon_category(): session.commit() session.refresh(category) yield category + session.delete(category) + session.commit() @pytest.fixture(scope="function") @@ -729,7 +845,7 @@ def second_lexicon_category(): session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def lexicon_term(lexicon_category): with session_ctx() as session: term = LexiconTerm( @@ -748,9 +864,12 @@ def lexicon_term(lexicon_category): session.refresh(term_category_association) yield term + session.delete(term) + session.delete(term_category_association) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def second_lexicon_term(lexicon_category): with session_ctx() as session: term = LexiconTerm( @@ -770,8 +889,12 @@ def second_lexicon_term(lexicon_category): yield term + session.delete(term) + session.delete(term_category_association) + session.commit() -@pytest.fixture(scope="session") + +@pytest.fixture() def third_lexicon_term(lexicon_category): with session_ctx() as session: term = LexiconTerm( @@ -791,8 +914,12 @@ def third_lexicon_term(lexicon_category): yield term + session.delete(term) + session.delete(term_category_association) + session.commit() + -@pytest.fixture(scope="session") +@pytest.fixture() def fourth_lexicon_term(lexicon_category): with session_ctx() as session: term = LexiconTerm( @@ -812,8 +939,12 @@ def fourth_lexicon_term(lexicon_category): yield term + session.delete(term) + session.delete(term_category_association) + session.commit() -@pytest.fixture(scope="session") + +@pytest.fixture() def lexicon_triple(lexicon_term, second_lexicon_term): with session_ctx() as session: triple = LexiconTriple( @@ -825,9 +956,11 @@ def lexicon_triple(lexicon_term, second_lexicon_term): session.commit() session.refresh(triple) yield triple + session.delete(triple) + session.commit() -@pytest.fixture(scope="session") +@pytest.fixture() def second_lexicon_triple(third_lexicon_term, fourth_lexicon_term): with session_ctx() as session: triple = LexiconTriple( @@ -839,3 +972,5 @@ def second_lexicon_triple(third_lexicon_term, fourth_lexicon_term): session.commit() session.refresh(triple) yield triple + session.delete(triple) + session.commit() diff --git a/tests/test_collabnet.py b/tests/test_collabnet.py index 96712e081..ce57f07c1 100644 --- a/tests/test_collabnet.py +++ b/tests/test_collabnet.py @@ -41,6 +41,9 @@ def well(): yield wt + session.delete(wt) + session.commit() + @pytest.mark.skip def test_add_collabnet_well(well): diff --git a/tests/test_observation.py b/tests/test_observation.py index 12ff56e5c..22717744b 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -22,7 +22,14 @@ viewer_function, ) from main import app -from tests import client, cleanup_post_test, override_authentication, cleanup_patch_test +from tests import ( + client, + cleanup_post_test, + override_authentication, + cleanup_patch_test, + groundwater_level_parameter_id, + pH_parameter_id, +) import pytest @@ -54,6 +61,8 @@ def test_add_water_chemistry_observation(water_chemistry_sample, sensor): "unit": "dimensionless", "sample_id": water_chemistry_sample.id, "sensor_id": sensor.id, + "parameter_id": pH_parameter_id, + "value_reason": "Observed value not affected", "observed_property": "pH", } response = client.post("/observation/water-chemistry", json=payload) @@ -68,7 +77,7 @@ def test_add_water_chemistry_observation(water_chemistry_sample, sensor): assert data["unit"] == payload["unit"] assert data["sample_id"] == payload["sample_id"] assert data["sensor_id"] == payload["sensor_id"] - assert data["observed_property"] == payload["observed_property"] + assert data["parameter"]["id"] == pH_parameter_id cleanup_post_test(Observation, data["id"]) @@ -80,9 +89,9 @@ def test_add_groundwater_level_observation(groundwater_level_sample, sensor): "value": 101, "measuring_point_height": 53, "sample_id": groundwater_level_sample.id, + "parameter_id": groundwater_level_parameter_id, "sensor_id": sensor.id, "groundwater_level_reason": "Water level not affected", - "observed_property": "groundwater level", "unit": "ft", } response = client.post("/observation/groundwater-level", json=payload) @@ -96,8 +105,8 @@ def test_add_groundwater_level_observation(groundwater_level_sample, sensor): assert data["value"] == payload["value"] assert data["measuring_point_height"] == payload["measuring_point_height"] assert data["sensor_id"] == payload["sensor_id"] + assert data["parameter"]["id"] == groundwater_level_parameter_id assert data["groundwater_level_reason"] == payload["groundwater_level_reason"] - assert data["observed_property"] == payload["observed_property"] assert ( data["depth_to_water_bgs"] == payload["value"] - payload["measuring_point_height"] @@ -109,6 +118,7 @@ def test_add_groundwater_level_observation(groundwater_level_sample, sensor): # PATCH tests ================================================================== +# TODO update patch test to test every single field def test_patch_groundwater_level_observation(groundwater_level_observation): payload = {"measuring_point_height": 3, "release_status": "private"} response = client.patch( @@ -213,7 +223,7 @@ def test_get_all_observations( assert "sample_id" in item assert "sensor_id" in item assert "observation_datetime" in item - assert "observed_property" in item + assert "parameter" in item assert "value" in item assert "unit" in item @@ -232,7 +242,7 @@ def test_get_observation_by_id( assert data["id"] == obs.id assert data["created_at"] == obs.created_at.isoformat().replace("+00:00", "Z") assert data["release_status"] == obs.release_status - if obs.observed_property == "groundwater level": + if obs.parameter.id == groundwater_level_parameter_id: assert data["depth_to_water_bgs"] == obs.value - obs.measuring_point_height else: assert data["depth_to_water_bgs"] is None @@ -263,11 +273,7 @@ def test_get_groundwater_level_observations(groundwater_level_observation): data["items"][0]["observation_datetime"] == groundwater_level_observation.observation_datetime ) - colon_index = groundwater_level_observation.observed_property.find(":") - assert ( - data["items"][0]["observed_property"] - == groundwater_level_observation.observed_property[colon_index + 1 :] - ) + assert data["items"][0]["parameter"]["id"] == groundwater_level_parameter_id assert ( data["items"][0]["release_status"] == groundwater_level_observation.release_status @@ -305,11 +311,7 @@ def test_get_groundwater_level_observation_by_id(groundwater_level_observation): data["observation_datetime"] == groundwater_level_observation.observation_datetime ) - colon_index = groundwater_level_observation.observed_property.find(":") - assert ( - data["observed_property"] - == groundwater_level_observation.observed_property[colon_index + 1 :] - ) + assert data["parameter"]["id"] == groundwater_level_parameter_id assert data["release_status"] == groundwater_level_observation.release_status assert ( data["groundwater_level_reason"] @@ -361,12 +363,13 @@ def test_get_groundwater_level_observation_by_id_404_wrong_activity_type( assert data["detail"][0]["loc"] == ["path", "observation_id"] -def test_get_groundwater_observation_by_sample(groundwater_level_sample): +def test_get_groundwater_observation_by_sample( + groundwater_level_observation, groundwater_level_sample +): response = client.get( "/observation/groundwater-level", params={ "sample_id": groundwater_level_sample.id, - "observed_property": "groundwater level", }, ) assert response.status_code == 200 @@ -376,12 +379,13 @@ def test_get_groundwater_observation_by_sample(groundwater_level_sample): assert len(items) > 0, "Expected at least one groundwater observation for the thing" -def test_get_groundwater_observation_by_thing(water_well_thing): +def test_get_groundwater_observation_by_thing( + groundwater_level_observation, water_well_thing +): response = client.get( "/observation/groundwater-level", params={ "thing_id": water_well_thing.id, - "observed_property": "groundwater level", }, ) assert response.status_code == 200 @@ -402,7 +406,7 @@ def test_get_groundwater_observation_by_thing_nonexistent(): ), "Expected no groundwater observations for a non-existent thing" -def test_get_groundwater_observation_by_time_range(): +def test_get_groundwater_observation_by_time_range(groundwater_level_observation): response = client.get( "/observation/groundwater-level", params={ @@ -452,11 +456,7 @@ def test_get_water_chemistry_observations(water_chemistry_observation): data["items"][0]["observation_datetime"] == water_chemistry_observation.observation_datetime ) - colon_index = water_chemistry_observation.observed_property.find(":") - assert ( - data["items"][0]["observed_property"] - == water_chemistry_observation.observed_property[colon_index + 1 :] - ) + assert data["items"][0]["parameter"]["id"] == pH_parameter_id assert data["items"][0]["value"] == water_chemistry_observation.value assert data["items"][0]["unit"] == water_chemistry_observation.unit @@ -477,11 +477,8 @@ def test_get_water_chemistry_observation_by_id(water_chemistry_observation): assert ( data["observation_datetime"] == water_chemistry_observation.observation_datetime ) - colon_index = water_chemistry_observation.observed_property.find(":") - assert ( - data["observed_property"] - == water_chemistry_observation.observed_property[colon_index + 1 :] - ) + + assert data["parameter"]["id"] == pH_parameter_id assert data["value"] == water_chemistry_observation.value assert data["unit"] == water_chemistry_observation.unit @@ -505,10 +502,7 @@ def test_get_water_chemistry_observation_by_id_404_wrong_activity_type( assert response.status_code == 404 data = response.json() - if groundwater_level_observation.observed_property == "groundwater level": - actual_activity_type = "groundwater level" - else: - actual_activity_type = "geothermal" + actual_activity_type = "groundwater level" assert ( data["detail"][0]["msg"] diff --git a/transfers/transfer.py b/transfers/transfer.py index 56977e9bd..61b88ef9d 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -20,7 +20,7 @@ load_dotenv() from sqlalchemy.orm import Session -from core.initializers import init_lexicon +from core.initializers import init_lexicon, init_parameter from db import Base from db.engine import session_ctx @@ -36,9 +36,12 @@ def erase_and_initalize(session: Session) -> None: - logger.info("Erasing existing data and initializing lexicon and sensors") + logger.info( + "Erasing existing data and initializing lexicon, parameter, and sensors" + ) erase(session) lexicon() + parameter() sensor(session) @@ -54,6 +57,12 @@ def lexicon(): init_lexicon() +@timeit +def parameter(): + logger.info("Initializing parameter") + init_parameter() + + @timeit def erase(session: Session): logger.info("Erasing existing data")