From 28b4c74c8d202fc611a46e1e39515bf8e8a74e30 Mon Sep 17 00:00:00 2001 From: Jeremy Zilar Date: Mon, 27 Apr 2026 14:57:26 -0400 Subject: [PATCH 1/3] query_helper: apply all Refine filter params and more operators order_sort_filter now walks every filter JSON in the request (optional filters list plus legacy single filter_). _apply_json_filter_clause supports contains, ncontains, startswith, endswith, eq, ne, comparison operators, null checks, and in, in addition to the old contains-only behavior. Any list or report that sends filter= more than once or uses operators other than contains will now get real SQL instead of silent no-ops. --- services/query_helper.py | 160 +++++++++++++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 22 deletions(-) diff --git a/services/query_helper.py b/services/query_helper.py index 379e27919..b197045b4 100644 --- a/services/query_helper.py +++ b/services/query_helper.py @@ -18,7 +18,7 @@ from fastapi import HTTPException from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy import select, Float, Integer, Column, Select, func, String +from sqlalchemy import Column, Float, Integer, Select, String, Text, func, not_, select from sqlalchemy.orm import DeclarativeBase, Session from sqlalchemy.sql.elements import OperatorExpression from starlette.status import HTTP_404_NOT_FOUND @@ -110,8 +110,130 @@ def simple_all_getter(session, table) -> list[object]: return session.scalars(sql).all() +def _python_type(column: Any): + try: + return column.type.python_type + except Exception: + return None + + +def _apply_json_filter_clause( + sql: Select[Any], table: DeclarativeBase, f: dict +) -> Select[Any]: + """Apply one Refine logical filter dict (field / operator / value) to a SELECT.""" + required_keys = {"field", "value", "operator"} + missing = required_keys - f.keys() + if missing: + raise HTTPException( + status_code=422, + detail=f"Missing required filter keys: {', '.join(sorted(missing))}", + ) + + field = f["field"] + value = f["value"] + operator = f["operator"] + + try: + column = getattr(table, field) + except AttributeError as exc: + raise HTTPException( + status_code=400, + detail=f"Unknown filter field {field!r} for {table.__name__}", + ) from exc + + py_t = _python_type(column) + is_string = py_t is str or isinstance(column.type, (String, Text)) + + if operator == "contains": + if not is_string: + raise HTTPException( + status_code=400, + detail=f"Operator contains is not supported for field {field!r}", + ) + return sql.where(column.ilike(f"%{value}%")) + + if operator == "ncontains": + if not is_string: + raise HTTPException( + status_code=400, + detail=f"Operator ncontains is not supported for field {field!r}", + ) + return sql.where(not_(column.ilike(f"%{value}%"))) + + if operator == "startswith": + if not is_string: + raise HTTPException( + status_code=400, + detail=f"Operator startswith is not supported for field {field!r}", + ) + return sql.where(column.ilike(f"{value}%")) + + if operator == "endswith": + if not is_string: + raise HTTPException( + status_code=400, + detail=f"Operator endswith is not supported for field {field!r}", + ) + return sql.where(column.ilike(f"%{value}")) + + if operator == "eq": + if py_t is float: + return sql.where(column == float(value)) + if py_t is int: + return sql.where(column == int(value)) + if is_string: + return sql.where(column == str(value)) + return sql.where(column == value) + + if operator == "ne": + if py_t is float: + return sql.where(column != float(value)) + if py_t is int: + return sql.where(column != int(value)) + if is_string: + return sql.where(column != str(value)) + return sql.where(column != value) + + if operator == "gt": + return sql.where(column > float(value) if py_t is float else column > value) + + if operator == "gte": + return sql.where(column >= float(value) if py_t is float else column >= value) + + if operator == "lt": + return sql.where(column < float(value) if py_t is float else column < value) + + if operator == "lte": + return sql.where(column <= float(value) if py_t is float else column <= value) + + if operator == "null": + return sql.where(column.is_(None)) + + if operator == "nnull": + return sql.where(column.is_not(None)) + + if operator == "in": + if not isinstance(value, (list, tuple)): + raise HTTPException( + status_code=400, + detail="Operator in requires an array value", + ) + return sql.where(column.in_(list(value))) + + raise HTTPException( + status_code=400, + detail=f"Unsupported filter operator {operator!r}", + ) + + def order_sort_filter( - sql: Select[Any], table: DeclarativeBase, sort: str, order: str, filter_: str + sql: Select[Any], + table: DeclarativeBase, + sort: str | None, + order: str | None, + filter_: str | None = None, + *, + filters: list[str] | None = None, ) -> Select[Any]: if order: if not sort: @@ -132,27 +254,21 @@ def order_sort_filter( else: raise ValueError("Invalid order parameter. Use 'asc' or 'desc'.") + filter_jsons: list[str] = [] + if filters: + filter_jsons.extend([x for x in filters if x]) if filter_: - required_keys = {"field", "value", "operator"} - if filter_ is not None: - try: - f = json.loads(filter_) - except Exception: - raise HTTPException(status_code=400, detail="Invalid JSON in filter") - - missing = required_keys - f.keys() - if missing: - raise HTTPException( - status_code=422, - detail=f"Missing required filter keys: {', '.join(missing)}", - ) - - field = f["field"] - value = f["value"] - operator = f["operator"] - column = getattr(table, field) - if operator == "contains": - sql = sql.where(column.ilike(f"%{value}%")) + filter_jsons.append(filter_) + + for raw in filter_jsons: + try: + f = json.loads(raw) + except Exception as exc: + raise HTTPException( + status_code=400, detail="Invalid JSON in filter" + ) from exc + + sql = _apply_json_filter_clause(sql, table, f) return sql From 748f43f96ce8cc4243c2458fe01d2ea4fef09161 Mon Sep 17 00:00:00 2001 From: Jeremy Zilar Date: Mon, 27 Apr 2026 14:57:29 -0400 Subject: [PATCH 2/3] thing_helper: merge multiple filter blobs and optional name_contains get_db_things accepts filters as a list of JSON strings (merged with legacy single filter_) and passes them to order_sort_filter. Adds name_contains to narrow rows with Thing.name ILIKE for substring search without changing the full-text query path. --- services/thing_helper.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/thing_helper.py b/services/thing_helper.py index 652a893cb..16fdd9a6a 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -121,6 +121,8 @@ def get_db_things( within: Optional[str] = None, name: Optional[str] = None, include_contacts: bool = False, + filters: Optional[list[str]] = None, + name_contains: Optional[str] = None, ) -> list: if query: @@ -151,6 +153,9 @@ def get_db_things( if name: sql = sql.where(Thing.name == name) + if name_contains and name_contains.strip(): + sql = sql.where(Thing.name.ilike(f"%{name_contains.strip()}%")) + if within: latest_assoc = ( select( @@ -173,7 +178,13 @@ def get_db_things( ) sql = make_within_wkt(sql, within) - sql = order_sort_filter(sql, Thing, sort, order, filter_) + merged_filters: list[str] | None = None + if filters: + merged_filters = list(filters) + elif filter_: + merged_filters = [filter_] + + sql = order_sort_filter(sql, Thing, sort, order, filters=merged_filters) return paginate(query=sql, conn=session) From 054c4c269b2ca685ef11686192970b5bead814ef Mon Sep 17 00:00:00 2001 From: Jeremy Zilar Date: Mon, 27 Apr 2026 14:57:31 -0400 Subject: [PATCH 3/3] api thing: repeated filter query param and name_contains on thing lists GET /thing/water-well, GET /thing/spring, and GET /thing now bind filter as a list so multiple Refine filters are forwarded. Same routes accept optional name_contains for substring matching on thing name alongside existing query. --- api/thing.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/api/thing.py b/api/thing.py index 6beb474e6..baeed59e7 100644 --- a/api/thing.py +++ b/api/thing.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from typing import Optional +from typing import Annotated, Optional from fastapi import APIRouter, Query, Request from fastapi_pagination.ext.sqlalchemy import paginate from sqlalchemy import select @@ -151,9 +151,10 @@ def get_water_wells( request: Request, sort: Optional[str] = None, order: Optional[str] = None, - filter_: str = Query(alias="filter", default=None), + filter_params: Annotated[list[str] | None, Query(alias="filter")] = None, query: Optional[str] = None, name: Optional[str] = None, + name_contains: Optional[str] = None, include_contacts: bool = False, ) -> CustomPage[WellResponse]: """ @@ -161,7 +162,7 @@ def get_water_wells( """ thing_type = request.url.path.split("/")[2].replace("-", " ") return get_db_things( - filter_, + None, order, query, session, @@ -169,6 +170,8 @@ def get_water_wells( name=name, thing_type=thing_type, include_contacts=include_contacts, + filters=filter_params, + name_contains=name_contains, ) @@ -293,14 +296,24 @@ def get_springs( request: Request, sort: str = None, order: str = None, - filter_: str = Query(alias="filter", default=None), + filter_params: Annotated[list[str] | None, Query(alias="filter")] = None, query: str = None, + name_contains: Optional[str] = None, ) -> CustomPage[SpringResponse]: """ Retrieve all springs from the database. """ thing_type = request.url.path.split("/")[2].replace("-", " ") - return get_db_things(filter_, order, query, session, sort, thing_type=thing_type) + return get_db_things( + None, + order, + query, + session, + sort, + thing_type=thing_type, + filters=filter_params, + name_contains=name_contains, + ) @router.get("/spring/{thing_id}", summary="Get spring by ID", status_code=HTTP_200_OK) @@ -359,23 +372,23 @@ def get_things( sort: Optional[str] = None, order: Optional[str] = None, include_contacts: bool = False, - filter_: str = Query( - default=None, - alias="filter", - ), + filter_params: Annotated[list[str] | None, Query(alias="filter")] = None, + name_contains: Optional[str] = None, ) -> CustomPage[ThingResponse]: """ Retrieve all things or filter by type. """ return get_db_things( - filter_, + None, order, query, session, sort, within=within, include_contacts=include_contacts, + filters=filter_params, + name_contains=name_contains, )