From 1874ac19f475645f099d0d1efd6f8cf82ebfc831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Pardou?= <571533+jrmi@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:41:59 +0200 Subject: [PATCH] feat: port grouped aggregations data source to builder (#5546) Co-authored-by: Tsering Paljor --- backend/src/baserow/api/serializers.py | 18 +- .../automation/automation_dispatch_context.py | 1 + .../builder/api/data_sources/serializers.py | 4 +- .../contrib/builder/api/data_sources/views.py | 4 +- .../data_sources/builder_dispatch_context.py | 3 + .../contrib/builder/data_sources/handler.py | 5 +- .../contrib/builder/data_sources/service.py | 13 +- .../contrib/builder/elements/element_types.py | 16 +- .../contrib/builder/elements/mixins.py | 2 +- .../dashboard/api/data_sources/views.py | 10 +- .../contrib/dashboard/application_types.py | 3 + .../data_sources/dispatch_context.py | 14 +- .../contrib/database/fields/field_types.py | 62 + .../contrib/database/fields/registries.py | 11 + .../integrations/local_baserow/mixins.py | 26 +- backend/src/baserow/core/formula/validator.py | 33 +- .../baserow/core/services/dispatch_context.py | 9 +- .../src/baserow/core/services/registries.py | 38 +- .../src/baserow/test_utils/pytest_conftest.py | 4 +- .../test_data_source_serializers.py | 15 +- .../data_sources/test_data_source_views.py | 11 + .../test_data_provider_types.py | 36 +- .../data_sources/test_dispatch_context.py | 22 +- .../test_dashboard_data_source_views.py | 84 + .../test_dashboard_data_source_handler.py | 8 +- .../test_dashboard_data_source_service.py | 16 +- .../test_list_rows_service_filter_groups.py | 68 + .../baserow/core/formula/test_validator.py | 42 + .../baserow/core/service/test_service_type.py | 25 +- ..._group_and_summarize_rows_data_source.json | 9 + .../src/baserow_enterprise/features.py | 1 + .../integrations/core/service_types.py | 12 +- .../src/baserow_enterprise/license_types.py | 2 + .../builder/elements/test_element_types.py | 21 +- .../core/test_core_xls_file_reader_license.py | 26 + .../test_core_xls_file_reader_service_type.py | 9 + .../test_grouped_aggregate_rows_license.py | 157 ++ .../modules/baserow_enterprise/features.js | 1 + .../baserow_enterprise/licenseTypes.js | 1 + .../local_baserow/service_types.py | 476 +++- ...grouped_aggregate_rows_data_source_type.py | 199 +- ...est_grouped_aggregate_rows_service_type.py | 2151 +++++++++++++---- .../components/dashboard_chart_widget.scss | 8 + .../dashboard/chartFieldFormatting.js | 4 + .../GroupedAggregateRowsDataSourceForm.vue | 49 +- .../dashboard/components/widget/Chart.vue | 180 +- .../components/widget/ChartWidget.vue | 51 +- .../components/widget/PieChartWidget.vue | 51 +- .../services}/AggregationGroupByForm.vue | 36 +- .../services}/AggregationSeriesForm.vue | 10 +- .../services}/AggregationSortByForm.vue | 0 .../LocalBaserowGroupedAggregateRowsForm.vue | 251 ++ .../integrations/localBaserow/serviceTypes.js | 176 ++ .../modules/baserow_premium/locales/en.json | 45 +- .../modules/baserow_premium/paidFeatures.js | 28 + .../modules/baserow_premium/plugin.js | 10 + .../components/paidFeaturesModal.spec.js | 53 + .../dashboard/components/widget/Chart.spec.js | 434 ++++ .../components/widget/ChartWidget.spec.js | 84 + .../components/widget/WidgetSettings.spec.js | 122 + .../groupedAggregateRowsForm.spec.js.snap | 306 +++ .../__snapshots__/serviceTypes.spec.js.snap | 77 + .../aggregationGroupByForm.spec.js | 57 + .../groupedAggregateRowsForm.spec.js | 226 ++ .../premium/integrations/serviceTypes.spec.js | 357 +++ .../components/dataSource/DataSourceForm.vue | 30 + .../components/dataSource/DataSourceItem.vue | 11 +- .../components/FormContainerElement.vue | 5 +- .../components/RecordSelectorElement.vue | 51 +- .../forms/general/TableElementForm.vue | 1 + .../modules/builder/dataProviderTypes.js | 2 +- web-frontend/modules/builder/elementTypes.js | 53 +- .../builder/mixins/collectionElementForm.js | 18 +- .../modules/builder/mixins/formElement.js | 5 +- .../modules/builder/pageHeaderItemTypes.js | 1 + .../modules/builder/services/dataSource.js | 13 +- .../modules/builder/store/dataSource.js | 6 + .../components/services/service_form.scss | 1 + web-frontend/modules/core/serviceTypes.js | 38 +- web-frontend/modules/core/utils/validator.js | 30 +- .../components/DashboardFormulaInput.vue | 46 + .../components/widget/WidgetSettings.vue | 7 + .../dashboard/store/dashboardApplication.js | 16 +- .../modules/integrations/core/serviceTypes.js | 2 +- .../services/ServiceRefinementForms.vue | 105 +- .../integrations/localBaserow/serviceTypes.js | 140 +- .../modules/integrations/locales/en.json | 2 + web-frontend/package.json | 2 +- .../components/RecordSelectorElement.spec.js | 209 +- .../RecordSelectorElement.spec.js.snap | 10 + .../test/unit/builder/elementTypes.spec.js | 93 + .../mixins/collectionElementForm.spec.js | 59 + .../unit/builder/store/dataSource.spec.js | 20 + .../test/unit/core/utils/validator.spec.js | 26 +- .../dashboardApplication.spec.js.snap | 35 + .../store/dashboardApplication.spec.js | 54 + .../dataSync/configureDataSyncHistory.spec.js | 5 +- .../dataSync/syncTableModal.spec.js | 4 +- .../__snapshots__/serviceTypes.spec.js.snap | 88 + .../localBaserow/serviceTypes.spec.js | 29 + 100 files changed, 6736 insertions(+), 764 deletions(-) create mode 100644 changelog/entries/unreleased/feature/add_group_and_summarize_rows_data_source.json create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/integrations/local_baserow/test_grouped_aggregate_rows_license.py rename premium/web-frontend/modules/baserow_premium/{dashboard/components/data_source => integrations/localBaserow/components/services}/AggregationGroupByForm.vue (72%) rename premium/web-frontend/modules/baserow_premium/{dashboard/components/data_source => integrations/localBaserow/components/services}/AggregationSeriesForm.vue (97%) rename premium/web-frontend/modules/baserow_premium/{dashboard/components/data_source => integrations/localBaserow/components/services}/AggregationSortByForm.vue (100%) create mode 100644 premium/web-frontend/modules/baserow_premium/integrations/localBaserow/components/services/LocalBaserowGroupedAggregateRowsForm.vue create mode 100644 premium/web-frontend/modules/baserow_premium/integrations/localBaserow/serviceTypes.js create mode 100644 premium/web-frontend/test/unit/premium/components/paidFeaturesModal.spec.js create mode 100644 premium/web-frontend/test/unit/premium/dashboard/components/widget/Chart.spec.js create mode 100644 premium/web-frontend/test/unit/premium/dashboard/components/widget/ChartWidget.spec.js create mode 100644 premium/web-frontend/test/unit/premium/dashboard/components/widget/WidgetSettings.spec.js create mode 100644 premium/web-frontend/test/unit/premium/integrations/__snapshots__/groupedAggregateRowsForm.spec.js.snap create mode 100644 premium/web-frontend/test/unit/premium/integrations/__snapshots__/serviceTypes.spec.js.snap create mode 100644 premium/web-frontend/test/unit/premium/integrations/aggregationGroupByForm.spec.js create mode 100644 premium/web-frontend/test/unit/premium/integrations/groupedAggregateRowsForm.spec.js create mode 100644 premium/web-frontend/test/unit/premium/integrations/serviceTypes.spec.js create mode 100644 web-frontend/modules/dashboard/components/DashboardFormulaInput.vue create mode 100644 web-frontend/test/unit/builder/mixins/collectionElementForm.spec.js create mode 100644 web-frontend/test/unit/dashboard/store/__snapshots__/dashboardApplication.spec.js.snap create mode 100644 web-frontend/test/unit/integrations/localBaserow/__snapshots__/serviceTypes.spec.js.snap diff --git a/backend/src/baserow/api/serializers.py b/backend/src/baserow/api/serializers.py index 5e3c9d01cd..6b939780f1 100644 --- a/backend/src/baserow/api/serializers.py +++ b/backend/src/baserow/api/serializers.py @@ -6,7 +6,10 @@ from baserow.core.context import clear_current_workspace_id, set_current_workspace_id from baserow.core.storage import get_default_storage -from baserow.core.utils import split_comma_separated_string +from baserow.core.utils import ( + list_to_comma_separated_string, + split_comma_separated_string, +) class PrefetchedManyToManyListSerializer(serializers.ListSerializer): @@ -135,6 +138,19 @@ def to_internal_value(self, data): return record_ids +class CommaSeparatedValuesField(serializers.Field): + """A serializer field that accepts a CSV string containing a list of values.""" + + def to_representation(self, value): + return list_to_comma_separated_string(value) + + def to_internal_value(self, data): + try: + return split_comma_separated_string(data) + except ValueError as e: + raise serializers.ValidationError(str(e), code="invalid") from e + + class FileURLSerializerMixin(serializers.Serializer): url = serializers.SerializerMethodField() diff --git a/backend/src/baserow/contrib/automation/automation_dispatch_context.py b/backend/src/baserow/contrib/automation/automation_dispatch_context.py index 8dc8a12902..ca3c43edf7 100644 --- a/backend/src/baserow/contrib/automation/automation_dispatch_context.py +++ b/backend/src/baserow/contrib/automation/automation_dispatch_context.py @@ -64,6 +64,7 @@ def __init__( use_sample_data=bool(self.simulate_until_node), force_outputs=force_outputs, event_payload=event_payload, + workspace=workflow.get_original().automation.workspace, ) def clone(self, **kwargs): diff --git a/backend/src/baserow/contrib/builder/api/data_sources/serializers.py b/backend/src/baserow/contrib/builder/api/data_sources/serializers.py index bf4d7cd3a7..ad54ed4304 100644 --- a/backend/src/baserow/contrib/builder/api/data_sources/serializers.py +++ b/backend/src/baserow/contrib/builder/api/data_sources/serializers.py @@ -6,7 +6,7 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from baserow.api.serializers import CommaSeparatedIntegerValuesField +from baserow.api.serializers import CommaSeparatedValuesField from baserow.api.services.serializers import ( CreateServiceSerializer, ServiceSerializer, @@ -163,7 +163,7 @@ class MoveDataSourceSerializer(serializers.Serializer): class GetRecordIdsSerializer(serializers.Serializer): - record_ids = CommaSeparatedIntegerValuesField() + record_ids = CommaSeparatedValuesField() class DynamicMetadataSerializer(serializers.Serializer): diff --git a/backend/src/baserow/contrib/builder/api/data_sources/views.py b/backend/src/baserow/contrib/builder/api/data_sources/views.py index acb06625b3..c5adf0056f 100644 --- a/backend/src/baserow/contrib/builder/api/data_sources/views.py +++ b/backend/src/baserow/contrib/builder/api/data_sources/views.py @@ -700,7 +700,9 @@ def get(self, request, data_source_id: int): query = GetRecordIdsSerializer(data=request.query_params) if query.is_valid(raise_exception=True): - record_ids = query.validated_data["record_ids"] + record_ids = service_type.prepare_record_ids( + query.validated_data["record_ids"] + ) record_names = service_type.get_record_names( service, record_ids, dispatch_context ) diff --git a/backend/src/baserow/contrib/builder/data_sources/builder_dispatch_context.py b/backend/src/baserow/contrib/builder/data_sources/builder_dispatch_context.py index 5a484cdd01..f6879ba917 100644 --- a/backend/src/baserow/contrib/builder/data_sources/builder_dispatch_context.py +++ b/backend/src/baserow/contrib/builder/data_sources/builder_dispatch_context.py @@ -39,6 +39,7 @@ class BuilderDispatchContext(DispatchContext): "count", "only_record_id", "only_expose_public_allowed_properties", + "workspace", ] def __init__( @@ -79,6 +80,8 @@ def __init__( only_expose_public_allowed_properties ) + kwargs["workspace"] = page.builder.get_workspace() + super().__init__(**kwargs) # Early call to quickly trigger a validation error diff --git a/backend/src/baserow/contrib/builder/data_sources/handler.py b/backend/src/baserow/contrib/builder/data_sources/handler.py index c6522c47bb..8d5f7d5bbd 100644 --- a/backend/src/baserow/contrib/builder/data_sources/handler.py +++ b/backend/src/baserow/contrib/builder/data_sources/handler.py @@ -558,8 +558,11 @@ def dispatch_data_source( cloned_dispatch_context.add_call(data_source.id) if data_source.id not in cache.setdefault("data_source_contents", {}): + service = data_source.service.specific + service.get_type().raise_if_deactivated(dispatch_context.workspace) + service_dispatch = self.service_handler.dispatch_service( - data_source.service.specific, cloned_dispatch_context + service, cloned_dispatch_context ) # Cache the dispatch in the formula cache if we have formulas that need diff --git a/backend/src/baserow/contrib/builder/data_sources/service.py b/backend/src/baserow/contrib/builder/data_sources/service.py index 596566831f..f90a13c28d 100644 --- a/backend/src/baserow/contrib/builder/data_sources/service.py +++ b/backend/src/baserow/contrib/builder/data_sources/service.py @@ -45,13 +45,6 @@ class DataSourceService: def __init__(self): self.handler = DataSourceHandler() - def _raise_if_service_type_deactivated( - self, service_type: ServiceType, page: Page - ) -> None: - raise_if_deactivated = getattr(service_type, "raise_if_deactivated", None) - if raise_if_deactivated: - raise_if_deactivated(page.builder.workspace) - def get_data_source(self, user: AbstractUser, data_source_id: int) -> DataSource: """ Returns an data_source instance from the database. Also checks the user @@ -166,7 +159,7 @@ def create_data_source( # Verify the `service_type` is dispatch-able as DATA. if not service_type.can_be_dispatched_as(DispatchTypes.DATA): raise InvalidServiceTypeDispatchSource() - self._raise_if_service_type_deactivated(service_type, page) + service_type.raise_if_deactivated(page.builder.workspace) prepared_values = service_type.prepare_values(kwargs, user) else: prepared_values = kwargs @@ -255,8 +248,8 @@ def update_data_source( service_type_for_preparation = new_service_type if service_type_for_preparation: - self._raise_if_service_type_deactivated( - service_type_for_preparation, page or data_source.page + service_type_for_preparation.raise_if_deactivated( + (page or data_source.page).builder.workspace ) service = data_source.service.specific if data_source.service_id else None prepared_values = service_type_for_preparation.prepare_values( diff --git a/backend/src/baserow/contrib/builder/elements/element_types.py b/backend/src/baserow/contrib/builder/elements/element_types.py index 0aed3b8a45..bc04b6993c 100644 --- a/backend/src/baserow/contrib/builder/elements/element_types.py +++ b/backend/src/baserow/contrib/builder/elements/element_types.py @@ -108,7 +108,6 @@ from baserow.core.formula.validator import ( ensure_array, ensure_boolean, - ensure_integer, ensure_numeric, ensure_string_or_integer, ) @@ -781,17 +780,22 @@ def is_valid( service_type = service.get_type() try: - record_ids = set(map(ensure_integer, ensure_array(value))) + raw_record_ids = ( + ensure_array(value) + if element.multiple + else ([] if value is None or value == "" else [value]) + ) + record_ids = set(service_type.prepare_record_ids(raw_record_ids)) record_names = service_type.get_record_names( service.specific, record_ids, dispatch_context, ) available_record_ids = set(record_names.keys()) - except ValidationError as err: + except (ValidationError, DRFValidationError) as err: msg = ( - "The value must be an array of integers, or convertible to an" - "array of integers" + "The value must be an array of valid record identifiers, or " + "convertible to an array of valid record identifiers" ) raise TypeError(msg) from err @@ -804,7 +808,7 @@ def is_valid( msg = f"{value} is not a valid option" raise ValueError(msg) else: - record_id = value + record_id = next(iter(record_ids), None) if not record_id: if element.required: diff --git a/backend/src/baserow/contrib/builder/elements/mixins.py b/backend/src/baserow/contrib/builder/elements/mixins.py index 70651c4a6a..5e41130ce3 100644 --- a/backend/src/baserow/contrib/builder/elements/mixins.py +++ b/backend/src/baserow/contrib/builder/elements/mixins.py @@ -413,7 +413,7 @@ def prepare_value_for_db( else: max_count = 20 - if values["items_per_page"] > max_count: + if max_count is not None and values["items_per_page"] > max_count: raise RequestBodyValidationException( { "items_per_page": [ diff --git a/backend/src/baserow/contrib/dashboard/api/data_sources/views.py b/backend/src/baserow/contrib/dashboard/api/data_sources/views.py index 12fd1aa762..cfa24778ad 100644 --- a/backend/src/baserow/contrib/dashboard/api/data_sources/views.py +++ b/backend/src/baserow/contrib/dashboard/api/data_sources/views.py @@ -18,6 +18,7 @@ DiscriminatorCustomFieldsMappingSerializer, ) from baserow.contrib.dashboard.api.errors import ERROR_DASHBOARD_DOES_NOT_EXIST +from baserow.contrib.dashboard.application_types import DashboardApplicationType from baserow.contrib.dashboard.data_sources.actions import ( UpdateDashboardDataSourceActionType, ) @@ -183,6 +184,7 @@ def patch(self, request, data_source_id: int): service_type_registry, request.data, base_serializer_class=UpdateDashboardDataSourceSerializer, + serializer_class_context={"application_type": DashboardApplicationType}, return_validated=True, ) @@ -245,8 +247,12 @@ def post(self, request, data_source_id: int): Call the given data source related service dispatch method. """ - dispatch_context = DashboardDispatchContext(request) - response = DashboardDataSourceService().dispatch_data_source( + data_source_service = DashboardDataSourceService() + data_source = data_source_service.handler.get_data_source(data_source_id) + dispatch_context = DashboardDispatchContext( + request, data_source.dashboard.workspace + ) + response = data_source_service.dispatch_data_source( request.user, data_source_id, dispatch_context ) return Response(response) diff --git a/backend/src/baserow/contrib/dashboard/application_types.py b/backend/src/baserow/contrib/dashboard/application_types.py index b8f676e7e3..6a40155d72 100644 --- a/backend/src/baserow/contrib/dashboard/application_types.py +++ b/backend/src/baserow/contrib/dashboard/application_types.py @@ -12,6 +12,7 @@ from baserow.contrib.integrations.local_baserow.integration_types import ( LocalBaserowIntegrationType, ) +from baserow.core.formula.registries import DataProviderTypeRegistry from baserow.core.integrations.handler import IntegrationHandler from baserow.core.integrations.registries import integration_type_registry from baserow.core.models import Application, Workspace @@ -27,6 +28,8 @@ class DashboardApplicationType(ApplicationType): serializer_field_names = ["name", "description"] allowed_fields = ["description"] supports_integrations = True + # Dashboard supports runtime functions, but no data provider references. + data_provider_type_registry = DataProviderTypeRegistry() def get_api_urls(self): from .api import urls as api_urls diff --git a/backend/src/baserow/contrib/dashboard/data_sources/dispatch_context.py b/backend/src/baserow/contrib/dashboard/data_sources/dispatch_context.py index f24f8a5973..489a735f32 100644 --- a/backend/src/baserow/contrib/dashboard/data_sources/dispatch_context.py +++ b/backend/src/baserow/contrib/dashboard/data_sources/dispatch_context.py @@ -6,20 +6,32 @@ if TYPE_CHECKING: from baserow.contrib.dashboard.widgets.models import Widget + from baserow.core.models import Workspace class DashboardDispatchContext(DispatchContext): own_properties = [ "request", "widget", + "workspace", ] def __init__( self, request: HttpRequest, + workspace: Optional["Workspace"] = None, widget: Optional["Widget"] = None, ): + """Create a context for dispatching a dashboard data source.""" + + # Before dashboard data sources needed their workspace directly, the second + # positional argument was the widget. Keep accepting that shape while callers + # move to passing the workspace. + if widget is None and workspace is not None and hasattr(workspace, "dashboard"): + widget = workspace + workspace = widget.dashboard.workspace + self.request = request self.widget = widget - super().__init__() + super().__init__(workspace) diff --git a/backend/src/baserow/contrib/database/fields/field_types.py b/backend/src/baserow/contrib/database/fields/field_types.py index b02283fa7a..0ece8ad902 100755 --- a/backend/src/baserow/contrib/database/fields/field_types.py +++ b/backend/src/baserow/contrib/database/fields/field_types.py @@ -3057,6 +3057,22 @@ def get_response_serializer_field(self, instance, **kwargs): }, ) + def get_query_value_serializer(self, field: LinkRowField): + response_serializer = self.get_response_serializer_field(field) + related_model = field.link_row_table.get_model() + + def serialize(value): + if value is None or value == "OTHER_VALUES": + return value + + rows_by_id = { + row.id: row for row in related_model.objects.filter(id__in=value) + } + rows = [rows_by_id[row_id] for row_id in value if row_id in rows_by_id] + return response_serializer.to_representation(rows) + + return serialize + def get_serializer_help_text(self, instance): return ( "This field accepts an `array` containing the ids or the names of the " @@ -4885,6 +4901,18 @@ def get_group_by_serializer_field(self, field, **kwargs): } ) + def get_query_value_serializer(self, field: SingleSelectField): + select_options = {option.id: option for option in field.select_options.all()} + response_serializer = self.get_response_serializer_field(field) + + def serialize(value): + if value is None or value == "OTHER_VALUES": + return value + + return response_serializer.to_representation(select_options.get(value)) + + return serialize + def get_formula_reference_to_model_field( self, model_field, db_column, already_in_subquery ): @@ -5089,6 +5117,23 @@ def get_response_serializer_field(self, instance, **kwargs): } ) + def get_query_value_serializer(self, field: MultipleSelectField): + select_options = {option.id: option for option in field.select_options.all()} + response_serializer = self.get_response_serializer_field(field) + + def serialize(value): + if value is None or value == "OTHER_VALUES": + return value + + options = [ + select_options[option_id] + for option_id in value + if option_id in select_options + ] + return response_serializer.to_representation(options) + + return serialize + def enhance_queryset(self, queryset, field, name, **kwargs): # It's important that this individual enhance_queryset method exists, even # though the enhance queryset in bulk exists, because the link_row field can @@ -7091,6 +7136,23 @@ def get_response_serializer_field(self, instance, **kwargs): } ) + def get_query_value_serializer(self, field: MultipleCollaboratorsField): + response_serializer = self.get_response_serializer_field(field) + + def serialize(value): + if value is None or value == "OTHER_VALUES": + return value + + users_by_id = { + user.id: user for user in get_user_model().objects.filter(id__in=value) + } + users = [ + users_by_id[user_id] for user_id in value if user_id in users_by_id + ] + return response_serializer.to_representation(users) + + return serialize + def serialize_to_input_value(self, field: Field, value: any) -> any: return [{"id": u.id, "name": u.first_name} for u in value.all()] diff --git a/backend/src/baserow/contrib/database/fields/registries.py b/backend/src/baserow/contrib/database/fields/registries.py index d4861aa614..59d18780df 100644 --- a/backend/src/baserow/contrib/database/fields/registries.py +++ b/backend/src/baserow/contrib/database/fields/registries.py @@ -1,6 +1,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Dict, List, NoReturn, @@ -2278,6 +2279,16 @@ def get_distribution_group_by_value(self, field_name: str): return field_name + def get_query_value_serializer(self, field: Field) -> Callable[[Any], Any]: + """ + Returns a serializer function for raw values produced by queries. + + Field types can override this when the raw query value differs from the + public API representation of the field value. + """ + + return lambda value: value + def to_runtime_formula_value(self, field, value): """ Transform the value to be usable in runtime formula land. diff --git a/backend/src/baserow/contrib/integrations/local_baserow/mixins.py b/backend/src/baserow/contrib/integrations/local_baserow/mixins.py index 341b924ef9..1392a4921e 100644 --- a/backend/src/baserow/contrib/integrations/local_baserow/mixins.py +++ b/backend/src/baserow/contrib/integrations/local_baserow/mixins.py @@ -269,9 +269,11 @@ def create_instance_from_serialized( new_group = LocalBaserowTableServiceFilterGroup.objects.create( service=service, filter_type=filter_group["filter_type"], - parent_group_id=group_id_mapping.get(parent_group_id) - if parent_group_id is not None - else None, + parent_group_id=( + group_id_mapping.get(parent_group_id) + if parent_group_id is not None + else None + ), ) group_id_mapping[filter_group["id"]] = new_group.id @@ -280,9 +282,11 @@ def create_instance_from_serialized( [ LocalBaserowTableServiceFilter( **{k: v for k, v in service_filter.items() if k != "group"}, - group_id=group_id_mapping.get(service_filter.get("group")) - if service_filter.get("group") is not None - else None, + group_id=( + group_id_mapping.get(service_filter.get("group")) + if service_filter.get("group") is not None + else None + ), order=index, service=service, ) @@ -579,6 +583,16 @@ def _invalidate_refinement_prefetch_cache(self, service): prefetch_cache.pop("service_filters", None) prefetch_cache.pop("service_filter_groups", None) + def after_create(self, instance: ServiceSubClass, values: Dict) -> None: + super().after_create(instance, values) + + if "service_filters" in values or "service_filter_groups" in values: + self.update_service_filters( + instance, + values.get("service_filters"), + values.get("service_filter_groups"), + ) + def after_update( self, instance: ServiceSubClass, diff --git a/backend/src/baserow/core/formula/validator.py b/backend/src/baserow/core/formula/validator.py index ba28008a56..e2ecf8f9a5 100644 --- a/backend/src/baserow/core/formula/validator.py +++ b/backend/src/baserow/core/formula/validator.py @@ -105,7 +105,9 @@ def ensure_numeric( ) -def ensure_integer(value: Any, allow_empty: bool = False) -> Optional[int]: +def ensure_integer( + value: Any, allow_empty: bool = False, allow_negative: bool = True +) -> Optional[int]: """ Ensures that the value is an integer or can be converted to an integer. Raises a ValidationError if the value is not a valid integer or convertible to an @@ -113,6 +115,7 @@ def ensure_integer(value: Any, allow_empty: bool = False) -> Optional[int]: :param value: The value to ensure as an integer. :param allow_empty: Whether we should throw an error if `value` is empty. + :param allow_negative: Whether negative integer values are allowed. :return: The value as an integer if conversion is successful. :raises ValidationError: If the value is not a valid integer or convertible to an integer. @@ -123,16 +126,38 @@ def ensure_integer(value: Any, allow_empty: bool = False) -> Optional[int]: raise ValidationError("The value is required") return None - if isinstance(value, timedelta): - return int(value.total_seconds()) + if isinstance(value, bool): + raise ValidationError( + "The value must be an integer or convertible to an integer." + ) + + integer_pattern = r"^[+-]?\d+$" if allow_negative else r"^\d+$" + if isinstance(value, str) and not re.match(integer_pattern, value): + raise ValidationError( + "The value must be an integer or convertible to an integer." + ) + + if isinstance(value, (float, Decimal)) and value != int(value): + raise ValidationError( + "The value must be an integer or convertible to an integer." + ) try: - return int(value) + int_value = ( + int(value.total_seconds()) if isinstance(value, timedelta) else int(value) + ) except (ValueError, TypeError) as exc: raise ValidationError( "The value must be an integer or convertible to an integer." ) from exc + if not allow_negative and int_value < 0: + raise ValidationError( + "The value must be a non-negative integer or convertible to one." + ) + + return int_value + def ensure_string(value: Any, allow_empty: bool = True) -> str: """ diff --git a/backend/src/baserow/core/services/dispatch_context.py b/backend/src/baserow/core/services/dispatch_context.py index f0260032da..3e8e5a73af 100644 --- a/backend/src/baserow/core/services/dispatch_context.py +++ b/backend/src/baserow/core/services/dispatch_context.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from django.contrib.auth.models import AbstractUser @@ -8,6 +8,9 @@ from baserow.core.services.types import RuntimeFormulaContextSubClass from baserow.core.services.utils import ServiceAdhocRefinements +if TYPE_CHECKING: + from baserow.core.models import Workspace + class DispatchContext(RuntimeFormulaContext, ABC): own_properties = [ @@ -17,6 +20,7 @@ class DispatchContext(RuntimeFormulaContext, ABC): "force_outputs", "event_payload", "actor", + "workspace", ] """ @@ -28,6 +32,7 @@ class DispatchContext(RuntimeFormulaContext, ABC): def __init__( self, + workspace: Optional["Workspace"] = None, only_record_id=None, event_payload: Any = None, update_sample_data_for: Optional[List[Service]] = None, @@ -48,6 +53,7 @@ def __init__( outputs. Can be used to force a specific service to be dispatched. :param actor: The user this dispatch acts as, for services that have no integration to supply one. + :param workspace: The workspace this dispatch is running in. """ self.cache = {} # can be used by data providers to save queries @@ -57,6 +63,7 @@ def __init__( self.force_outputs = force_outputs self.event_payload = event_payload self.actor = actor + self.workspace = workspace super().__init__() def range(self, service: Service) -> tuple[int, int | None]: diff --git a/backend/src/baserow/core/services/registries.py b/backend/src/baserow/core/services/registries.py index 149b84d143..c4f3593651 100644 --- a/backend/src/baserow/core/services/registries.py +++ b/backend/src/baserow/core/services/registries.py @@ -8,6 +8,7 @@ from django.db import transaction from loguru import logger +from rest_framework.exceptions import PermissionDenied from rest_framework.exceptions import ValidationError as DRFValidationError from baserow.core.formula import resolve_formula @@ -17,9 +18,11 @@ ) from baserow.core.formula.parser.exceptions import BaserowFormulaException from baserow.core.formula.registries import formula_runtime_function_registry +from baserow.core.formula.validator import ensure_integer from baserow.core.integrations.exceptions import IntegrationDoesNotExist from baserow.core.integrations.handler import IntegrationHandler from baserow.core.integrations.models import Integration +from baserow.core.models import Workspace from baserow.core.registry import ( APIUrlsInstanceMixin, APIUrlsRegistryMixin, @@ -94,6 +97,17 @@ class ServiceType( public_serializer_field_names = [] public_serializer_field_overrides = {} + def is_deactivated(self, workspace: Workspace) -> bool: + """Return whether this service type is unavailable in the workspace.""" + + return False + + def raise_if_deactivated(self, workspace: Workspace) -> None: + """Reject use of a deactivated service type when called by its consumer.""" + + if self.is_deactivated(workspace): + raise PermissionDenied("This service type is deactivated.") + def can_be_dispatched_as(self, dispatch_type: DispatchTypes) -> bool: """ Returns whether this service can be dispatched as the given dispatch type. @@ -596,7 +610,24 @@ class ListServiceTypeMixin: returns_list = True - @abstractmethod + def prepare_record_ids(self, record_ids: List[Any]) -> List[int]: + """ + Convert record selector values into the type used by this service as row IDs. + Regular Baserow row services use integer row IDs. Services with synthetic or + external row identifiers can override this hook. + """ + + try: + prepared_record_ids = [ + ensure_integer(record_id) for record_id in record_ids + ] + except ValidationError as exc: + raise DRFValidationError( + {"record_ids": ["The provided record ids are not valid."]} + ) from exc + + return prepared_record_ids + def get_record_names( self, service: Service, @@ -606,13 +637,16 @@ def get_record_names( """ Return the record name associated with each one of the provided record ids. - Implementation is required for any service that uses this mixin. + By default, the record id itself is used as the display name. Services + that can resolve richer names, like local Baserow row services, can + override this. :param service: The available service to use. :param record_ids: The list containing the record identifiers. :param dispatch_context: The context used for the dispatch. :return: A dictionary mapping each record to its name. """ + return {record_id: str(record_id) for record_id in record_ids} @abstractmethod def get_max_result_limit(self, service: Service): diff --git a/backend/src/baserow/test_utils/pytest_conftest.py b/backend/src/baserow/test_utils/pytest_conftest.py index 50f8fd9251..d6fec6ae64 100755 --- a/backend/src/baserow/test_utils/pytest_conftest.py +++ b/backend/src/baserow/test_utils/pytest_conftest.py @@ -9,6 +9,7 @@ from decimal import Decimal from functools import partial from pathlib import Path +from types import SimpleNamespace from typing import Any, AsyncGenerator, Dict, List, Optional from unittest.mock import patch @@ -995,7 +996,8 @@ class FakeDispatchContext(DispatchContext): ] def __init__(self, **kwargs): - super().__init__() + workspace = kwargs.pop("workspace", SimpleNamespace(id=-1)) + super().__init__(workspace) self.context = kwargs.pop("context", {}) self._public_allowed_properties = kwargs.pop("public_allowed_properties", None) self._searchable_fields = kwargs.pop("searchable_fields", []) diff --git a/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_serializers.py b/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_serializers.py index 0b2d200262..915b51ba99 100644 --- a/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_serializers.py +++ b/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_serializers.py @@ -10,8 +10,19 @@ ({"record_ids": ""}, {"record_ids": []}, True), ({"record_ids": "1"}, {"record_ids": ["1"]}, True), ({"record_ids": "1,2,3"}, {"record_ids": ["1", "2", "3"]}, True), - ({"record_ids": "1,invalid"}, {"record_ids": []}, False), - ({"record_ids": "invalid_1,invalid_2"}, {"record_ids": []}, False), + ({"record_ids": "1,invalid"}, {"record_ids": ["1", "invalid"]}, True), + ({"record_ids": "one\ntwo"}, None, False), + ({"record_ids": "one\rtwo"}, None, False), + ( + {"record_ids": '"one\ntwo",Other'}, + {"record_ids": ["one\ntwo", "Other"]}, + True, + ), + ( + {"record_ids": '"ACME, Inc.",Other'}, + {"record_ids": ["ACME, Inc.", "Other"]}, + True, + ), ], ) def test_get_record_ids_serializer(data, expected, valid): diff --git a/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_views.py b/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_views.py index f4e4af9d43..cef54a9a3a 100644 --- a/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_views.py +++ b/backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_views.py @@ -2429,6 +2429,17 @@ def test_get_record_names(api_client, data_fixture): assert response.status_code == HTTP_400_BAD_REQUEST assert response.json()["record_ids"] == ["The provided record ids are not valid."] + # Malformed CSV must return a validation error instead of a server error. + response = api_client.get( + base_url, + {"record_ids": "one\ntwo"}, + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["record_ids"][0].startswith( + "Could not split comma separated string:" + ) + # If the data source is not a list data source, it should raise an error non_list_data_source = ( data_fixture.create_builder_local_baserow_get_row_data_source( diff --git a/backend/tests/baserow/contrib/builder/data_providers/test_data_provider_types.py b/backend/tests/baserow/contrib/builder/data_providers/test_data_provider_types.py index 605065cff1..d0e7e67be7 100644 --- a/backend/tests/baserow/contrib/builder/data_providers/test_data_provider_types.py +++ b/backend/tests/baserow/contrib/builder/data_providers/test_data_provider_types.py @@ -76,7 +76,9 @@ def test_page_parameter_data_provider_get_data_chunk(): fake_request = MagicMock() fake_request.data = {"metadata": json.dumps({"page_parameter": {"id": 42}})} - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) assert page_parameter_provider.get_data_chunk(dispatch_context, ["id"]) == 42 assert page_parameter_provider.get_data_chunk(dispatch_context, []) is None @@ -97,7 +99,9 @@ def test_form_data_provider_get_data_chunk(mock_validate): "metadata": json.dumps({"form_data": {"1": "hello", "2": ["a", "b"]}}) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) mock_validate.side_effect = lambda x, y, z: y # A single valued form data @@ -1058,7 +1062,9 @@ def test_previous_action_data_provider_get_data_chunk(data_fixture): {"previous_action": {str(workflow_action.id): {"path": {"to": 100}}}} ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) with pytest.raises(InvalidFormulaContext): previous_action_data_provider.get_data_chunk( @@ -1075,7 +1081,9 @@ def test_previous_action_data_provider_get_data_chunk(data_fixture): } ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) assert ( previous_action_data_provider.get_data_chunk( @@ -1118,7 +1126,9 @@ def test_previous_action_data_provider_get_data_chunk_returns_cached_result( } ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) previous_action_data_provider.get_dispatch_action_cache_key = MagicMock( return_value="bar" @@ -1160,7 +1170,9 @@ def test_previous_action_data_provider_get_data_chunk_returns_cached_list_result } ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) previous_action_data_provider.get_dispatch_action_cache_key = MagicMock( return_value="bar" @@ -1201,7 +1213,9 @@ def test_previous_action_data_provider_post_dispatch_caches_result(): } ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) workflow_action = MagicMock() workflow_action.id = 100 @@ -1247,7 +1261,9 @@ def test_previous_action_data_provider_post_dispatch_with_empty_response_cache_r } ) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) workflow_action = MagicMock() workflow_action.id = 100 @@ -1482,7 +1498,9 @@ def test_current_record_provider_get_data_chunk_for_idx(): fake_request.data = { "metadata": json.dumps({"current_record": {"index": 123, "record_id": 123}}) } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, Mock(builder=Mock(get_workspace=Mock(return_value=Mock(id=-1)))) + ) assert current_record_provider.get_data_chunk(dispatch_context, ["__idx__"]) == 123 diff --git a/backend/tests/baserow/contrib/builder/data_sources/test_dispatch_context.py b/backend/tests/baserow/contrib/builder/data_sources/test_dispatch_context.py index 037635f5e8..ea39c75d16 100644 --- a/backend/tests/baserow/contrib/builder/data_sources/test_dispatch_context.py +++ b/backend/tests/baserow/contrib/builder/data_sources/test_dispatch_context.py @@ -17,23 +17,31 @@ from baserow.core.user_sources.user_source_user import UserSourceUser +def fake_workspace(): + return Mock(id=-1) + + +def fake_page(): + return Mock(builder=Mock(get_workspace=Mock(return_value=fake_workspace()))) + + def test_dispatch_context_page_range(): request = HttpRequest() request.GET = {"offset": 42, "count": 42} - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.range(None) == (42, 42) request.GET = {"offset": "foo", "count": "bar"} - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.range(None) == (0, None) request.GET = {"offset": "-20", "count": "-10"} - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.range(None) == (0, 0) @@ -102,7 +110,7 @@ def test_dispatch_context_element_type(data_fixture): def test_dispatch_context_search_query(): request = HttpRequest() request.GET["search_query"] = "foobar" - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.search_query() == "foobar" @@ -214,7 +222,7 @@ def test_dispatch_context_filters(): ], } request.GET["filters"] = filter_data - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.filters() == filter_data @@ -245,7 +253,7 @@ def test_dispatch_context_is_publicly_sortable(collection_element_type, data_fix def test_dispatch_context_sortings(): request = HttpRequest() request.GET["order_by"] = "-field_1,-field_2" - dispatch_context = BuilderDispatchContext(request, None) + dispatch_context = BuilderDispatchContext(request, fake_page()) assert dispatch_context.sortings() == "-field_1,-field_2" @@ -382,7 +390,7 @@ def test_get_element_property_options(data_fixture, django_assert_num_queries): def test_validate_filter_search_sort_fields_without_element(): - dispatch_context = BuilderDispatchContext(HttpRequest(), None) + dispatch_context = BuilderDispatchContext(HttpRequest(), fake_page()) with pytest.raises(DataSourceRefinementForbidden) as exc: dispatch_context.validate_filter_search_sort_fields( ["name"], ServiceAdhocRefinements.FILTER diff --git a/backend/tests/baserow/contrib/dashboard/api/data_sources/test_dashboard_data_source_views.py b/backend/tests/baserow/contrib/dashboard/api/data_sources/test_dashboard_data_source_views.py index b530cb3493..01ba7a3ef1 100644 --- a/backend/tests/baserow/contrib/dashboard/api/data_sources/test_dashboard_data_source_views.py +++ b/backend/tests/baserow/contrib/dashboard/api/data_sources/test_dashboard_data_source_views.py @@ -547,3 +547,87 @@ def test_dispatch_data_source_improperly_configured(api_client, data_fixture): response.json()["detail"] == "The data_source configuration is incorrect: " "No integration selected" ) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "mode,formula,expected_status", + [ + ("raw", "20", HTTP_200_OK), + ("simple", "20", HTTP_200_OK), + ("advanced", "10 + 10", HTTP_200_OK), + ("advanced", "get('data_source.1')", HTTP_400_BAD_REQUEST), + ("advanced", "add(10)", HTTP_400_BAD_REQUEST), + ], +) +def test_update_dashboard_filter_formula( + api_client, data_fixture, mode, formula, expected_status +): + """Validate, reload, and execute filters after switching to formula mode.""" + + user, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=user) + database = data_fixture.create_database_application(workspace=workspace) + table = data_fixture.create_database_table(database=database) + field = data_fixture.create_number_field(table=table) + RowHandler().create_rows( + user, table, [{field.db_column: 10}, {field.db_column: 20}] + ) + dashboard = data_fixture.create_dashboard_application(workspace=workspace) + integration = data_fixture.create_local_baserow_integration( + authorized_user=user, application=dashboard + ) + service = data_fixture.create_local_baserow_aggregate_rows_service( + integration=integration, table=table, field=field, aggregation_type="sum" + ) + data_source = ( + data_fixture.create_dashboard_local_baserow_aggregate_rows_data_source( + dashboard=dashboard, service=service + ) + ) + headers = {"HTTP_AUTHORIZATION": f"JWT {token}"} + response = api_client.patch( + reverse( + "api:dashboard:data_sources:item", kwargs={"data_source_id": data_source.id} + ), + { + "filters": [ + { + "field": field.id, + "type": "equal", + "value": {"formula": formula, "mode": mode}, + } + ] + }, + format="json", + **headers, + ) + assert response.status_code == expected_status, response.json() + if expected_status == HTTP_400_BAD_REQUEST: + assert ( + response.json()["detail"]["value"][0]["code"] == "invalid_formula_argument" + ) + assert not service.service_filters.exists() + return + + saved_formula = response.json()["filters"][0]["value"] + assert saved_formula["formula"] == formula + assert saved_formula["mode"] == mode + response = api_client.get( + reverse( + "api:dashboard:data_sources:list", kwargs={"dashboard_id": dashboard.id} + ), + **headers, + ) + assert response.status_code == HTTP_200_OK + assert response.json()[0]["filters"][0]["value"] == saved_formula + response = api_client.post( + reverse( + "api:dashboard:data_sources:dispatch", + kwargs={"data_source_id": data_source.id}, + ), + format="json", + **headers, + ) + assert response.status_code == HTTP_200_OK, response.json() + assert response.json() == {"result": "20"} diff --git a/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_handler.py b/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_handler.py index a51b16aee5..8d5396db4f 100644 --- a/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_handler.py +++ b/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_handler.py @@ -355,7 +355,9 @@ def test_dispatch_data_source(data_fixture): widget = data_fixture.create_summary_widget( dashboard=dashboard, data_source=data_source ) - dispatch_context = DashboardDispatchContext(HttpRequest(), widget) + dispatch_context = DashboardDispatchContext( + HttpRequest(), widget.dashboard.workspace + ) result = DashboardDataSourceHandler().dispatch_data_source( data_source, dispatch_context @@ -392,7 +394,9 @@ def test_dispatch_data_source_improperly_configured(data_fixture): name="Name 1", user=user, dashboard=dashboard, service=service ) ) - dispatch_context = DashboardDispatchContext(HttpRequest()) + dispatch_context = DashboardDispatchContext( + HttpRequest(), data_source.dashboard.workspace + ) with pytest.raises(ServiceImproperlyConfiguredDispatchException): DashboardDataSourceHandler().dispatch_data_source(data_source, dispatch_context) diff --git a/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_service.py b/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_service.py index 1fdf18f6f5..b1087319a1 100644 --- a/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_service.py +++ b/backend/tests/baserow/contrib/dashboard/data_sources/test_dashboard_data_source_service.py @@ -346,7 +346,9 @@ def test_dispatch_data_source(data_fixture): widget = data_fixture.create_summary_widget( dashboard=dashboard, data_source=data_source ) - dispatch_context = DashboardDispatchContext(HttpRequest(), widget) + dispatch_context = DashboardDispatchContext( + HttpRequest(), widget.dashboard.workspace + ) result = DashboardDataSourceService().dispatch_data_source( user, data_source.id, dispatch_context @@ -359,7 +361,9 @@ def test_dispatch_data_source(data_fixture): def test_dispatch_data_source_permissions_denied(data_fixture): user = data_fixture.create_user() data_source = data_fixture.create_dashboard_data_source(name="Name 1") - dispatch_context = DashboardDispatchContext(HttpRequest()) + dispatch_context = DashboardDispatchContext( + HttpRequest(), data_source.dashboard.workspace + ) with pytest.raises(PermissionException): DashboardDataSourceService().dispatch_data_source( @@ -374,7 +378,9 @@ def test_dispatch_data_source_trashed(data_fixture): data_source = data_fixture.create_dashboard_data_source( name="Name 1", dashboard=dashboard, trashed=True ) - dispatch_context = DashboardDispatchContext(HttpRequest()) + dispatch_context = DashboardDispatchContext( + HttpRequest(), data_source.dashboard.workspace + ) with pytest.raises(DashboardDataSourceDoesNotExist): DashboardDataSourceService().dispatch_data_source( @@ -389,7 +395,9 @@ def test_dispatch_data_source_dashboard_trashed(data_fixture): data_source = data_fixture.create_dashboard_data_source( name="Name 1", dashboard=dashboard ) - dispatch_context = DashboardDispatchContext(HttpRequest()) + dispatch_context = DashboardDispatchContext( + HttpRequest(), data_source.dashboard.workspace + ) with pytest.raises(DashboardDataSourceDoesNotExist): DashboardDataSourceService().dispatch_data_source( diff --git a/backend/tests/baserow/contrib/integrations/local_baserow/service_types/test_list_rows_service_filter_groups.py b/backend/tests/baserow/contrib/integrations/local_baserow/service_types/test_list_rows_service_filter_groups.py index 96d3a58c8c..ff31c6b7a2 100644 --- a/backend/tests/baserow/contrib/integrations/local_baserow/service_types/test_list_rows_service_filter_groups.py +++ b/backend/tests/baserow/contrib/integrations/local_baserow/service_types/test_list_rows_service_filter_groups.py @@ -15,6 +15,7 @@ BASEROW_FORMULA_MODE_RAW, BaserowFormulaObject, ) +from baserow.core.services.handler import ServiceHandler from baserow.core.services.registries import service_type_registry from baserow.test_utils.pytest_conftest import FakeDispatchContext, fake_import_formula @@ -56,6 +57,73 @@ def _build_ingredient_cost_table(data_fixture, user): return integration, table, ingredient, cost, rows +@pytest.mark.django_db +@pytest.mark.parametrize( + "with_filters,with_groups,nested", + [ + pytest.param(True, True, False, id="grouped-filters"), + pytest.param(False, True, False, id="groups-only"), + pytest.param(True, True, True, id="nested-child-first"), + pytest.param(True, False, False, id="ungrouped-filters"), + ], +) +def test_create_service_with_filter_groups( + data_fixture, with_filters, with_groups, nested +): + """Creation preserves groups, parent links, and filter membership.""" + + user = data_fixture.create_user() + integration, table, ingredient, _cost, _rows = _build_ingredient_cost_table( + data_fixture, user + ) + values = {} + if with_groups: + values["service_filter_groups"] = [ + {"id": "outer", "filter_type": "OR", "parent_group_id": None} + ] + if nested: + values["service_filter_groups"].insert( + 0, + {"id": "inner", "filter_type": "AND", "parent_group_id": "outer"}, + ) + if with_filters: + values["service_filters"] = [ + { + "field": ingredient, + "type": "equal", + "value": raw_formula("Duck"), + "value_is_formula": False, + "group_id": ("inner" if nested else "outer") if with_groups else None, + } + ] + + service = ServiceHandler().create_service( + service_type_registry.get("local_baserow_list_rows"), + integration=integration, + table=table, + **values, + ) + service.refresh_from_db() + + assert service.service_filter_groups.count() == (2 if nested else int(with_groups)) + expected_group_id = None + if with_groups: + outer = service.service_filter_groups.get(parent_group__isnull=True) + assert outer.filter_type == "OR" + expected_group_id = outer.id + if nested: + inner = service.service_filter_groups.get(parent_group=outer) + assert inner.filter_type == "AND" + expected_group_id = inner.id + + assert service.service_filters.count() == int(with_filters) + if with_filters: + service_filter = service.service_filters.get() + assert service_filter.field_id == ingredient.id + assert service_filter.value == raw_formula("Duck") + assert service_filter.group_id == expected_group_id + + @pytest.mark.django_db def test_dispatch_with_filter_groups(data_fixture): """ diff --git a/backend/tests/baserow/core/formula/test_validator.py b/backend/tests/baserow/core/formula/test_validator.py index a9d9952781..32266cf022 100644 --- a/backend/tests/baserow/core/formula/test_validator.py +++ b/backend/tests/baserow/core/formula/test_validator.py @@ -328,3 +328,45 @@ def test_ensure_string_with_timedelta(value, expected): ) def test_ensure_integer_with_timedelta(value, expected): assert ensure_integer(value) == expected + + +@pytest.mark.parametrize( + "value,expected", + [ + (-1, -1), + ("-1", -1), + ("+1", 1), + (1.0, 1), + ("001", 1), + ], +) +def test_ensure_integer(value, expected): + assert ensure_integer(value) == expected + + +@pytest.mark.parametrize("value", [1.2, "1.2", True, False, "a"]) +def test_ensure_integer_throws_exception_for_invalid_value(value): + with pytest.raises(ValidationError): + ensure_integer(value) + + +@pytest.mark.parametrize( + "value,expected", + [ + (0, 0), + (1, 1), + ("0", 0), + ("1", 1), + ("001", 1), + ], +) +def test_ensure_integer_disallowing_negative_values(value, expected): + assert ensure_integer(value, allow_negative=False) == expected + + +@pytest.mark.parametrize( + "value", [None, "", -1, "-1", "+1", 1.2, "1.2", True, False, "a"] +) +def test_ensure_integer_disallowing_negative_values_throws_exception(value): + with pytest.raises(ValidationError): + ensure_integer(value, allow_negative=False) diff --git a/backend/tests/baserow/core/service/test_service_type.py b/backend/tests/baserow/core/service/test_service_type.py index 5aab77dbe5..d3b9a0bd03 100644 --- a/backend/tests/baserow/core/service/test_service_type.py +++ b/backend/tests/baserow/core/service/test_service_type.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, Mock, PropertyMock import pytest -from rest_framework.exceptions import ValidationError +from rest_framework.exceptions import PermissionDenied, ValidationError from baserow.core.services.models import Service from baserow.core.services.registries import ServiceType @@ -9,6 +9,29 @@ from baserow_premium.integrations.local_baserow.service_types import DispatchResult +def test_service_type_is_active_by_default(mocker): + mocker.patch.object(ServiceType, "model_class", Service, create=True) + service_type = ServiceType() + workspace = Mock() + + assert service_type.is_deactivated(workspace) is False + service_type.raise_if_deactivated(workspace) + + +def test_service_type_raise_if_deactivated(mocker): + mocker.patch.object(ServiceType, "model_class", Service, create=True) + service_type = ServiceType() + workspace = Mock() + is_deactivated = mocker.patch.object( + service_type, "is_deactivated", return_value=True + ) + + with pytest.raises(PermissionDenied, match="This service type is deactivated"): + service_type.raise_if_deactivated(workspace) + + is_deactivated.assert_called_once_with(workspace) + + def test_service_type_get_schema_name(): mock_service = Mock(id=123) service_type_cls = ServiceType diff --git a/changelog/entries/unreleased/feature/add_group_and_summarize_rows_data_source.json b/changelog/entries/unreleased/feature/add_group_and_summarize_rows_data_source.json new file mode 100644 index 0000000000..7a6d877d13 --- /dev/null +++ b/changelog/entries/unreleased/feature/add_group_and_summarize_rows_data_source.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Add Group and summarize rows data source", + "issue_origin": "github", + "issue_number": null, + "domain": "builder", + "bullet_points": [], + "created_at": "2026-06-30" +} diff --git a/enterprise/backend/src/baserow_enterprise/features.py b/enterprise/backend/src/baserow_enterprise/features.py index de1efda46a..eb1dbc3e8a 100644 --- a/enterprise/backend/src/baserow_enterprise/features.py +++ b/enterprise/backend/src/baserow_enterprise/features.py @@ -14,6 +14,7 @@ BUILDER_NO_BRANDING = "application_no_branding" BUILDER_FILE_INPUT = "builder_file_input" BUILDER_CUSTOM_CODE = "builder_custom_code" +BUILDER_GROUPED_AGGREGATE_ROWS = "builder_grouped_aggregate_rows" CODE_RUNNER = "code_runner" XLS_FILE_READER = "xls_file_reader" diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py b/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py index 77742b04fb..33e01c877a 100644 --- a/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py @@ -264,15 +264,6 @@ class SerializedDict(ServiceDict): "sheet_name", ] - def _get_dispatch_workspace(self, dispatch_context: DispatchContext): - if page := getattr(dispatch_context, "page", None): - return page.builder.workspace - - if workflow := getattr(dispatch_context, "workflow", None): - return workflow.automation.workspace - - return None - def is_deactivated(self, workspace) -> bool: return not LicenseHandler.workspace_has_feature(XLS_FILE_READER, workspace) @@ -286,8 +277,7 @@ def dispatch( service: CoreXLSFileReaderService, dispatch_context: DispatchContext, ) -> DispatchResult: - if workspace := self._get_dispatch_workspace(dispatch_context): - self.raise_if_deactivated(workspace) + self.raise_if_deactivated(dispatch_context.workspace) return super().dispatch(service, dispatch_context) diff --git a/enterprise/backend/src/baserow_enterprise/license_types.py b/enterprise/backend/src/baserow_enterprise/license_types.py index 2844c889bc..e14e840a8d 100755 --- a/enterprise/backend/src/baserow_enterprise/license_types.py +++ b/enterprise/backend/src/baserow_enterprise/license_types.py @@ -6,6 +6,7 @@ AUDIT_LOG, BUILDER_CUSTOM_CODE, BUILDER_FILE_INPUT, + BUILDER_GROUPED_AGGREGATE_ROWS, BUILDER_NO_BRANDING, BUILDER_SSO, CODE_RUNNER, @@ -46,6 +47,7 @@ BUILDER_CUSTOM_CODE, CODE_RUNNER, XLS_FILE_READER, + BUILDER_GROUPED_AGGREGATE_ROWS, # only self-hosted SSO, ] diff --git a/enterprise/backend/tests/baserow_enterprise_tests/builder/elements/test_element_types.py b/enterprise/backend/tests/baserow_enterprise_tests/builder/elements/test_element_types.py index c38e634017..f40e7f6e22 100644 --- a/enterprise/backend/tests/baserow_enterprise_tests/builder/elements/test_element_types.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/builder/elements/test_element_types.py @@ -125,7 +125,12 @@ def test_file_input_element_is_valid(fake): "3c913094-c69a-4fd3-b19d-c35322f7d5c5": fake_file, } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, + MagicMock( + builder=MagicMock(get_workspace=MagicMock(return_value=MagicMock(id=-1))) + ), + ) value = { "__file__": True, @@ -175,7 +180,12 @@ def test_file_input_element_is_valid_invalid_filetype(fake, allowed, should_rais "3c913094-c69a-4fd3-b19d-c35322f7d5c5": fake_file, } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, + MagicMock( + builder=MagicMock(get_workspace=MagicMock(return_value=MagicMock(id=-1))) + ), + ) value = { "__file__": True, @@ -217,7 +227,12 @@ def test_file_input_element_is_valid_invalid_size(fake): "3c913094-c69a-4fd3-b19d-c35322f7d5c5": fake_file, } - dispatch_context = BuilderDispatchContext(fake_request, None) + dispatch_context = BuilderDispatchContext( + fake_request, + MagicMock( + builder=MagicMock(get_workspace=MagicMock(return_value=MagicMock(id=-1))) + ), + ) value = { "__file__": True, diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_license.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_license.py index 7711f0a0d5..ec2217002f 100644 --- a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_license.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_license.py @@ -12,6 +12,10 @@ from baserow.contrib.builder.workflow_actions.service import ( BuilderWorkflowActionService, ) +from baserow.contrib.dashboard.data_sources.dispatch_context import ( + DashboardDispatchContext, +) +from baserow.contrib.dashboard.data_sources.service import DashboardDataSourceService from baserow.core.services.exceptions import ( ServiceImproperlyConfiguredDispatchException, ) @@ -84,6 +88,28 @@ def test_core_xls_file_reader_data_source_dispatch_requires_enterprise_license( DataSourceService().dispatch_data_source(user, data_source, dispatch_context) +@pytest.mark.django_db +def test_core_xls_file_reader_dashboard_data_source_dispatch_requires_enterprise_license( + enterprise_data_fixture, +): + user = enterprise_data_fixture.create_user() + dashboard = enterprise_data_fixture.create_dashboard_application(user=user) + service = enterprise_data_fixture.create_enterprise_core_xls_file_reader_service( + integration_args={"application": dashboard} + ) + data_source = enterprise_data_fixture.create_dashboard_data_source( + dashboard=dashboard, service=service + ) + + enterprise_data_fixture.delete_all_licenses() + + dispatch_context = DashboardDispatchContext(HttpRequest(), dashboard.workspace) + with pytest.raises(FeaturesNotAvailableError): + DashboardDataSourceService().dispatch_data_source( + user, data_source.id, dispatch_context + ) + + @pytest.mark.django_db def test_core_xls_file_reader_data_source_update_requires_enterprise_license( enterprise_data_fixture, diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_service_type.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_service_type.py index a9416aaf9a..adbe2bbe61 100644 --- a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_service_type.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_xls_file_reader_service_type.py @@ -2,6 +2,7 @@ from io import BytesIO from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import override_settings import pytest from openpyxl import Workbook @@ -37,9 +38,12 @@ def service_file(name, content): @pytest.mark.django_db +@override_settings(DEBUG=True) def test_core_xls_file_reader_service_type_dispatch_data_with_xlsx( enterprise_data_fixture, ): + enterprise_data_fixture.enable_enterprise() + workspace = enterprise_data_fixture.create_workspace() service = enterprise_data_fixture.create_enterprise_core_xls_file_reader_service( file="get('file')" ) @@ -47,6 +51,7 @@ def test_core_xls_file_reader_service_type_dispatch_data_with_xlsx( dispatch_result = service.get_type().dispatch( service, FakeDispatchContext( + workspace=workspace, context={ "file": service_file( "people.xlsx", @@ -126,9 +131,12 @@ def test_core_xls_file_reader_service_type_dispatch_data_without_header( @pytest.mark.django_db +@override_settings(DEBUG=True) def test_core_xls_file_reader_service_type_dispatch_data_with_sheet_name( enterprise_data_fixture, ): + enterprise_data_fixture.enable_enterprise() + workspace = enterprise_data_fixture.create_workspace() service = enterprise_data_fixture.create_enterprise_core_xls_file_reader_service( file="get('file')", sheet_name="'People'" ) @@ -136,6 +144,7 @@ def test_core_xls_file_reader_service_type_dispatch_data_with_sheet_name( dispatch_result = service.get_type().dispatch( service, FakeDispatchContext( + workspace=workspace, context={ "file": service_file( "people.xlsx", diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/local_baserow/test_grouped_aggregate_rows_license.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/local_baserow/test_grouped_aggregate_rows_license.py new file mode 100644 index 0000000000..f44f259aec --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/local_baserow/test_grouped_aggregate_rows_license.py @@ -0,0 +1,157 @@ +from django.http import HttpRequest +from django.test import override_settings + +import pytest + +from baserow.contrib.builder.data_sources.builder_dispatch_context import ( + BuilderDispatchContext, +) +from baserow.contrib.builder.data_sources.service import DataSourceService +from baserow.contrib.dashboard.data_sources.dispatch_context import ( + DashboardDispatchContext, +) +from baserow.contrib.dashboard.data_sources.service import DashboardDataSourceService +from baserow.core.services.registries import service_type_registry +from baserow_enterprise.features import BUILDER_GROUPED_AGGREGATE_ROWS +from baserow_enterprise.license_types import ( + AdvancedLicenseType, + EnterpriseLicenseType, + EnterpriseWithoutSupportLicenseType, +) +from baserow_premium.integrations.local_baserow.models import ( + LocalBaserowGroupedAggregateRows, + LocalBaserowTableServiceAggregationSeries, +) +from baserow_premium.license.exceptions import FeaturesNotAvailableError +from baserow_premium.license.license_types import PremiumLicenseType + + +def test_grouped_aggregate_rows_data_source_feature_is_available_in_advanced_and_enterprise(): + assert BUILDER_GROUPED_AGGREGATE_ROWS not in PremiumLicenseType.features + assert BUILDER_GROUPED_AGGREGATE_ROWS in AdvancedLicenseType.features + assert ( + BUILDER_GROUPED_AGGREGATE_ROWS in EnterpriseWithoutSupportLicenseType.features + ) + assert BUILDER_GROUPED_AGGREGATE_ROWS in EnterpriseLicenseType.features + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_requires_enterprise_license( + enterprise_data_fixture, +): + user = enterprise_data_fixture.create_user() + page = enterprise_data_fixture.create_builder_page(user=user) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + enterprise_data_fixture.delete_all_licenses() + + assert service_type.is_deactivated(page.builder.workspace) + + with pytest.raises(FeaturesNotAvailableError): + DataSourceService().create_data_source( + user, page=page, service_type=service_type + ) + + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + assert not service_type.is_deactivated(page.builder.workspace) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_update_requires_enterprise_license( + enterprise_data_fixture, +): + user = enterprise_data_fixture.create_user() + page = enterprise_data_fixture.create_builder_page(user=user) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + data_source = DataSourceService().create_data_source( + user, page=page, service_type=service_type + ) + + enterprise_data_fixture.delete_all_licenses() + + with pytest.raises(FeaturesNotAvailableError): + DataSourceService().update_data_source( + user, data_source, service_type=service_type + ) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_dispatch_requires_enterprise_license( + enterprise_data_fixture, +): + user = enterprise_data_fixture.create_user() + page = enterprise_data_fixture.create_builder_page(user=user) + + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + service = enterprise_data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration_args={"application": page.builder}, + ) + data_source = enterprise_data_fixture.create_builder_data_source( + page=page, service=service + ) + + enterprise_data_fixture.delete_all_licenses() + + dispatch_context = BuilderDispatchContext( + HttpRequest(), page, only_expose_public_allowed_properties=False + ) + with pytest.raises(FeaturesNotAvailableError): + DataSourceService().dispatch_data_source(user, data_source, dispatch_context) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_dashboard_data_source_dispatch_after_license_loss( + enterprise_data_fixture, +): + """An existing dashboard returns the same aggregation after losing its license.""" + user = enterprise_data_fixture.create_user() + dashboard = enterprise_data_fixture.create_dashboard_application(user=user) + + database = enterprise_data_fixture.create_database_application( + workspace=dashboard.workspace + ) + table = enterprise_data_fixture.create_database_table(user=user, database=database) + field = enterprise_data_fixture.create_number_field(table=table) + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + service = enterprise_data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration_args={"application": dashboard, "authorized_user": user}, + table=table, + ) + data_source = enterprise_data_fixture.create_dashboard_data_source( + dashboard=dashboard, service=service + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + model = table.get_model() + model.objects.create(**{f"field_{field.id}": 42}) + expected = DashboardDataSourceService().dispatch_data_source( + user, + data_source.id, + DashboardDispatchContext(HttpRequest(), dashboard.workspace), + ) + + enterprise_data_fixture.delete_all_licenses() + assert service.get_type().is_deactivated(dashboard.workspace) + + result = DashboardDataSourceService().dispatch_data_source( + user, + data_source.id, + DashboardDispatchContext(HttpRequest(), dashboard.workspace), + ) + assert ( + result + == expected + == { + "results": [{"id": "Result", f"{field.name} sum": 42.0}], + "has_next_page": False, + } + ) diff --git a/enterprise/web-frontend/modules/baserow_enterprise/features.js b/enterprise/web-frontend/modules/baserow_enterprise/features.js index 07a91e7612..0ad2fefe63 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/features.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/features.js @@ -10,6 +10,7 @@ const EnterpriseFeatures = { BUILDER_NO_BRANDING: 'BUILDER_NO_BRANDING', BUILDER_FILE_INPUT: 'BUILDER_FILE_INPUT', BUILDER_CUSTOM_CODE: 'BUILDER_CUSTOM_CODE', + BUILDER_GROUPED_AGGREGATE_ROWS: 'builder_grouped_aggregate_rows', CODE_RUNNER: 'CODE_RUNNER', XLS_FILE_READER: 'XLS_FILE_READER', ADVANCED_WEBHOOKS: 'ADVANCED_WEBHOOKS', diff --git a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js index ccd79a7865..667f60c065 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js @@ -20,6 +20,7 @@ const commonAdvancedFeatures = [ EnterpriseFeaturesObject.BUILDER_CUSTOM_CODE, EnterpriseFeaturesObject.CODE_RUNNER, EnterpriseFeaturesObject.XLS_FILE_READER, + EnterpriseFeaturesObject.BUILDER_GROUPED_AGGREGATE_ROWS, // Only self-hosted EnterpriseFeaturesObject.SSO, ] diff --git a/premium/backend/src/baserow_premium/integrations/local_baserow/service_types.py b/premium/backend/src/baserow_premium/integrations/local_baserow/service_types.py index 272a12cfef..6f1c0de76c 100644 --- a/premium/backend/src/baserow_premium/integrations/local_baserow/service_types.py +++ b/premium/backend/src/baserow_premium/integrations/local_baserow/service_types.py @@ -1,8 +1,11 @@ import re +from collections import Counter +from typing import Any from django.conf import settings from django.db.models import F +from rest_framework import serializers from rest_framework.exceptions import ValidationError as DRFValidationError from baserow.contrib.database.api.fields.serializers import FieldSerializer @@ -25,7 +28,11 @@ from baserow.core.services.exceptions import ( ServiceImproperlyConfiguredDispatchException, ) -from baserow.core.services.registries import DispatchTypes +from baserow.core.services.registries import ( + DispatchTypes, + ListServiceTypeMixin, + ServiceType, +) from baserow.core.services.types import DispatchResult from baserow.core.utils import atomic_if_not_already from baserow_premium.api.integrations.local_baserow.serializers import ( @@ -43,15 +50,20 @@ grouped_aggregation_group_by_registry, grouped_aggregation_registry, ) +from baserow_premium.license.handler import LicenseHandler from baserow_premium.services.types import ( ServiceAggregationGroupByDict, ServiceAggregationSeriesDict, ServiceAggregationSortByDict, ) +BUILDER_GROUPED_AGGREGATE_ROWS = "builder_grouped_aggregate_rows" +GROUPED_AGGREGATE_ROW_ID = "id" + class LocalBaserowGroupedAggregateRowsUserServiceType( LocalBaserowTableServiceFilterableMixin, + ListServiceTypeMixin, LocalBaserowViewServiceType, ): """ @@ -65,6 +77,16 @@ class LocalBaserowGroupedAggregateRowsUserServiceType( dispatch_types = [DispatchTypes.DATA] serializer_mixins = LocalBaserowTableServiceFilterableMixin.mixin_serializer_mixins + def is_deactivated(self, workspace) -> bool: + return not LicenseHandler.workspace_has_feature( + BUILDER_GROUPED_AGGREGATE_ROWS, workspace + ) + + def raise_if_deactivated(self, workspace) -> None: + LicenseHandler.raise_if_workspace_doesnt_have_feature( + BUILDER_GROUPED_AGGREGATE_ROWS, workspace + ) + def get_schema_name(self, service: LocalBaserowGroupedAggregateRows) -> str: return f"GroupedAggregation{service.id}Schema" @@ -73,6 +95,276 @@ def get_context_data_schema( ) -> dict | None: return None + def prepare_record_ids(self, record_ids: list[Any]) -> list[str]: + return [str(record_id) for record_id in record_ids if record_id != ""] + + def get_id_property(self, service: Service) -> str: + return GROUPED_AGGREGATE_ROW_ID + + def get_name_property(self, service: Service) -> str | None: + group_by = service.service_aggregation_group_bys.first() + if group_by is None: + return None + + if group_by.field_id is None: + return service.table.get_model().get_primary_field().db_column + + return group_by.field.db_column + + def _get_name_field_object(self, service: Service) -> dict | None: + group_by = service.service_aggregation_group_bys.first() + if group_by is None: + return None + + field_id = group_by.field_id + if field_id is None: + field_id = service.table.get_model().get_primary_field().id + + for field_object in self.get_table_field_objects(service) or []: + if field_object["field"].id == field_id: + return field_object + + return None + + def _can_group_by_row_id(self, model) -> bool: + """Row grouping currently requires a scalar primary-field value.""" + primary_field = model.get_primary_field() + serializer = primary_field.get_type().get_response_serializer_field( + primary_field + ) + return not isinstance( + serializer, (serializers.ListField, serializers.ListSerializer) + ) + + def _get_human_readable_result_value( + self, + value: Any, + field_object: dict | None, + model, + ) -> str: + if field_object is None: + return "" + if value == "OTHER_VALUES": + return value + + model_field = model._meta.get_field(field_object["name"]) + related_model = getattr(model_field.remote_field, "model", None) + if related_model is not None and value is not None: + try: + value = related_model.objects.get(pk=value) + except (TypeError, ValueError, related_model.DoesNotExist): + value = None + + human_readable_value = field_object["type"].get_human_readable_value( + value, field_object + ) + if human_readable_value is None: + return "" + + return str(human_readable_value) + + def _set_unique_record_ids(self, results: list[dict]) -> None: + """Keep readable labels, suffixing duplicates without colliding with real labels.""" + + counts = Counter(row[GROUPED_AGGREGATE_ROW_ID] for row in results) + used = set(counts) + occurrences = Counter() + for row in results: + name = row[GROUPED_AGGREGATE_ROW_ID] + if counts[name] == 1: + continue + while True: + occurrences[name] += 1 + candidate = f"{name} ({occurrences[name]})" + if candidate not in used: + break + row[GROUPED_AGGREGATE_ROW_ID] = candidate + used.add(candidate) + + def generate_schema( + self, + service: LocalBaserowGroupedAggregateRows, + allowed_fields: list[str] | None = None, + ) -> dict | None: + if not service.table_id: + return None + + result_properties = self._get_result_properties(service) + if result_properties is None: + return None + + if allowed_fields is not None: + allowed_fields = set(allowed_fields) + result_properties = { + field: value + for field, value in result_properties.items() + if field in allowed_fields + } + + return { + "type": "array", + "title": self.get_schema_name(service), + "items": { + "type": "object", + "properties": result_properties, + }, + } + + def _get_result_properties( + self, + service: LocalBaserowGroupedAggregateRows, + ) -> dict | None: + table_properties = self._get_table_properties(service) + if table_properties is None: + return None + + def grouped_field_property(field_db_column): + return { + **table_properties[field_db_column], + "sortable": False, + "filterable": False, + "searchable": False, + } + + properties = { + GROUPED_AGGREGATE_ROW_ID: { + "title": "Id", + "type": "string", + "sortable": False, + "filterable": False, + "searchable": False, + }, + } + group_bys = service.service_aggregation_group_bys.all() + for group_by in group_bys: + if group_by.field_id is None: + primary_field = service.table.get_model().get_primary_field() + properties[primary_field.db_column] = grouped_field_property( + primary_field.db_column + ) + elif group_by.field.db_column in table_properties: + properties[group_by.field.db_column] = grouped_field_property( + group_by.field.db_column + ) + + for aggregation_series in service.service_aggregation_series.all(): + if not aggregation_series.field or not aggregation_series.aggregation_type: + continue + aggregation_type = grouped_aggregation_registry.get( + aggregation_series.aggregation_type + ) + properties[ + f"{aggregation_series.field.db_column}_{aggregation_series.aggregation_type}" + ] = { + "title": ( + f"{aggregation_series.field.name} " + f"{aggregation_series.aggregation_type}" + ), + **aggregation_type.get_result_schema(aggregation_series.field.specific), + "metadata": self.get_aggregation_result_metadata(aggregation_series), + } + + self._disambiguate_result_property_names(properties) + return properties + + def _disambiguate_result_property_names(self, properties: dict) -> None: + """ + Add result-name overrides for colliding labels, reserving the record ID. + + Reserve all original labels before assigning suffixes so that generated + names cannot collide with another field's label. Technical property keys + make the mapping independent of series order. Compute this on the full + schema before filtering allowed fields so permissions cannot change names. + """ + + names = { + key: ( + key + if key == GROUPED_AGGREGATE_ROW_ID + else prop.get("metadata", {}).get("display_name") + or prop.get("title") + or key + ) + for key, prop in properties.items() + } + counts = Counter(names.values()) + used_names = set(names.values()) | {GROUPED_AGGREGATE_ROW_ID} + for key in sorted(properties): + name = names[key] + if key == GROUPED_AGGREGATE_ROW_ID or ( + counts[name] == 1 and name != GROUPED_AGGREGATE_ROW_ID + ): + continue + + base_name = f"{name} [{key}]" + result_name = base_name + suffix = 2 + while result_name in used_names: + result_name = f"{base_name} ({suffix})" + suffix += 1 + used_names.add(result_name) + properties[key]["metadata"] = { + **properties[key].get("metadata", {}), + "result_name": result_name, + } + + def get_aggregation_result_metadata( + self, aggregation_series: LocalBaserowTableServiceAggregationSeries + ) -> dict: + field = aggregation_series.field + return { + "display_name": f"{field.name} {aggregation_series.aggregation_type}", + "source_field": { + "id": field.id, + "name": field.db_column, + "display_name": field.name, + }, + "aggregation": { + "type": aggregation_series.aggregation_type, + }, + } + + def prepare_value_path(self, service: Service, path: list[str]): + if len(path) < 1: + return path + + property_name, *rest = path + human_name = self._get_result_property_human_name(service, property_name) + if human_name == property_name: + return path + + return [human_name, *rest] + + def _get_result_property_human_name(self, service: Service, property_name: str): + if property_name == GROUPED_AGGREGATE_ROW_ID: + return property_name + + result_properties = self._get_result_properties(service) or {} + property_schema = result_properties.get(property_name) + if property_schema is None: + return property_name + + return ( + property_schema.get("metadata", {}).get("result_name") + or property_schema.get("metadata", {}).get("display_name") + or property_schema.get("title") + or property_name + ) + + def _convert_result_property_names_to_human_names( + self, service: Service, result: dict + ): + return { + self._get_result_property_human_name(service, key): value + for key, value in result.items() + } + + def _convert_allowed_field_names(self, service, allowed_fields): + return [ + self._get_result_property_human_name(service, field) + for field in allowed_fields + ] + def enhance_queryset(self, queryset): return ( super() @@ -278,6 +570,11 @@ def validate_agg_group_by(group_by): ) if group_by["field_id"] is None: + if not self._can_group_by_row_id(service.table.get_model()): + raise DRFValidationError( + detail="Row ID grouping is not supported for a multi-valued primary field.", + code="invalid_field", + ) return True field = next( @@ -316,6 +613,23 @@ def validate_agg_group_by(group_by): ] ) + def _get_allowed_sort_references(self, service, model=None) -> list[str]: + """Return sort references supported by the current series and grouping.""" + + references = [ + f"field_{series.field_id}_{series.aggregation_type}" + for series in service.service_aggregation_series.all() + if series.aggregation_type is not None and series.field_id is not None + ] + group_bys = list(service.service_aggregation_group_bys.all()) + if group_bys: + field_id = group_bys[0].field_id + if field_id is None: + model = model if model is not None else service.table.get_model() + field_id = model.get_primary_field().id + references.append(f"field_{field_id}") + return references + def _update_service_sorts( self, service: LocalBaserowGroupedAggregateRows, @@ -324,22 +638,7 @@ def _update_service_sorts( with atomic_if_not_already(): service.service_aggregation_sorts.all().delete() if service_sorts is not None: - model = service.table.get_model() - - allowed_sort_references = [ - f"field_{series.field_id}_{series.aggregation_type}" - for series in service.service_aggregation_series.all() - if series.aggregation_type is not None - and series.field_id is not None - ] - - if service.service_aggregation_group_bys.count() > 0: - group_by = service.service_aggregation_group_bys.all()[0] - allowed_sort_references += ( - [f"field_{group_by.field_id}"] - if group_by.field_id is not None - else [f"field_{model.get_primary_field().id}"] - ) + allowed_sort_references = self._get_allowed_sort_references(service) def validate_sort(service_sort): if service_sort["reference"] not in allowed_sort_references: @@ -369,6 +668,8 @@ def after_create(self, instance: LocalBaserowGroupedAggregateRows, values: dict) metadata. """ + super().after_create(instance, values) + if "service_aggregation_series" in values: self._update_service_aggregation_series( instance, values.pop("service_aggregation_series") @@ -390,7 +691,7 @@ def after_update( ) -> None: """ Responsible for updating service aggregation series and group bys. - At the moment all objects are recreated on update. + Existing sorts whose targets disappear are removed. :param instance: The service that was updated. :param values: A dictionary which may contain aggregation series and @@ -399,10 +700,17 @@ def after_update( service prior to `after_update` being called. """ + super().after_update(instance, values, changes) + # Following a Table change, from one Table to another, we drop all # the things that are no longer applicable for the other table. from_table, to_table = changes.get("table", (None, None)) + aggregation_changed = ( + "service_aggregation_series" in values + or "service_aggregation_group_bys" in values + ) + if "service_aggregation_series" in values: self._update_service_aggregation_series( instance, values.pop("service_aggregation_series") @@ -417,12 +725,31 @@ def after_update( elif from_table and to_table: instance.service_aggregation_group_bys.all().delete() + # Related objects may have been prefetched before the update. Sort + # validation and the response must use the newly persisted configuration. + if ( + aggregation_changed + or "service_aggregation_sorts" in values + or (from_table and to_table) + ): + prefetch_cache = getattr(instance, "_prefetched_objects_cache", {}) + for relation in ( + "service_aggregation_series", + "service_aggregation_group_bys", + "service_aggregation_sorts", + ): + prefetch_cache.pop(relation, None) + if "service_aggregation_sorts" in values: self._update_service_sorts( instance, values.pop("service_aggregation_sorts") ) elif from_table and to_table: instance.service_aggregation_sorts.all().delete() + elif aggregation_changed: + instance.service_aggregation_sorts.exclude( + reference__in=self._get_allowed_sort_references(instance) + ).delete() def export_prepared_values(self, instance: Service) -> dict[str, any]: values = super().export_prepared_values(instance) @@ -597,6 +924,10 @@ def dispatch_data( group_by_values = [] for group_by in service.service_aggregation_group_bys.all(): if group_by.field is None: + if not self._can_group_by_row_id(model): + raise ServiceImproperlyConfiguredDispatchException( + "Row ID grouping is not supported for a multi-valued primary field." + ) group_by_values.append("id") group_by_values.append(model.get_primary_field().db_column) break @@ -669,19 +1000,7 @@ def dispatch_data( other_buckets_qs = other_buckets_qs.annotate(**value.annotations) combined_agg_dict[key] = value.aggregation - allowed_sort_references = [ - f"field_{series.field_id}_{series.aggregation_type}" - for series in service.service_aggregation_series.all() - if series.aggregation_type is not None and series.field_id is not None - ] - - if service.service_aggregation_group_bys.count() > 0: - group_by = service.service_aggregation_group_bys.all()[0] - allowed_sort_references += ( - [f"field_{group_by.field_id}"] - if group_by.field_id is not None - else [f"field_{model.get_primary_field().id}"] - ) + allowed_sort_references = self._get_allowed_sort_references(service, model) sorts = [] sort_annotations = {} @@ -719,8 +1038,19 @@ def dispatch_data( sorts.append(field_order_by) queryset = queryset.annotate(**sort_annotations) + group_by_fields = [ + model.get_primary_field() if group_by.field is None else group_by.field + for group_by in service.service_aggregation_group_bys.all() + ] + group_by_value_serializers_by_db_column = { + field.db_column: field.get_type().get_query_value_serializer(field.specific) + for field in group_by_fields + } - def process_individual_result(result: dict): + def process_individual_result(result: dict, overflow=False): + """Finalize aggregates and assign readable group labels before deduplication.""" + + result = {**result} for agg_series in defined_agg_series: key = f"{agg_series.field.db_column}_{agg_series.aggregation_type}" raw_value = result.pop(f"{key}_raw") @@ -730,6 +1060,25 @@ def process_individual_result(result: dict): ) if "total" in result: del result["total"] + name_property = self.get_name_property(service) + if overflow: + name = "OTHER_VALUES" + elif name_property: + name = self._get_human_readable_result_value( + result.get(name_property), + self._get_name_field_object(service), + model, + ) + else: + name = "Result" + result[GROUPED_AGGREGATE_ROW_ID] = name or "-" + for ( + db_column, + serialize_value, + ) in group_by_value_serializers_by_db_column.items(): + if overflow or db_column not in result: + continue + result[db_column] = serialize_value(result[db_column]) return result if len(group_by_values) > 0: @@ -738,9 +1087,10 @@ def process_individual_result(result: dict): queryset = queryset[ : settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS + 1 ] + raw_results = list(queryset) - results = [process_individual_result(result) for result in queryset] - buckets_count = len(queryset) + results = [process_individual_result(result) for result in raw_results] + buckets_count = len(raw_results) if ( buckets_count > settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS @@ -748,29 +1098,34 @@ def process_individual_result(result: dict): # The number of buckets don't fit in the limit # so we will aggregate all the other buckets into one bucket_db_column = group_by_values[0] + raw_results = raw_results[ + : settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS + - 1 + ] results = results[ : settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS - 1 ] - buckets_taken = [result[bucket_db_column] for result in results] + buckets_taken = [result[bucket_db_column] for result in raw_results] other_buckets_qs = other_buckets_qs.exclude( **{f"{bucket_db_column}__in": buckets_taken} ) other_bucket_results = other_buckets_qs.aggregate(**combined_agg_dict) - other_bucket_results = process_individual_result(other_bucket_results) other_bucket_primary_field = ( {f"{model.get_primary_field().db_column}": "OTHER_VALUES"} if "id" in group_by_values else {} ) - results.append( + other_bucket_results = process_individual_result( { - f"{bucket_db_column}": "OTHER_VALUES", + bucket_db_column: "OTHER_VALUES", **other_bucket_primary_field, **other_bucket_results, - } + }, + overflow=True, ) + results.append(other_bucket_results) first_sort_by = service.service_aggregation_sorts.first() if first_sort_by and first_sort_by.sort_on == "SERIES": results = sorted( @@ -781,9 +1136,24 @@ def process_individual_result(result: dict): else: results = queryset.aggregate(**combined_agg_dict) results = process_individual_result(results) + results = [results] + + self._set_unique_record_ids(results) + if dispatch_context.only_record_id is not None: + current_record_id = str(dispatch_context.only_record_id) + results = [ + result + for result in results + if result[GROUPED_AGGREGATE_ROW_ID] == current_record_id + ] + + results = [ + self._convert_result_property_names_to_human_names(service, result) + for result in results + ] return { - "data": {"result": results}, + "data": {"results": results, "has_next_page": False}, "baserow_table_model": model, } @@ -792,3 +1162,29 @@ def dispatch_transform( data: any, ) -> DispatchResult: return DispatchResult(data=data["data"]) + + def sanitize_result(self, service, result, allowed_field_names): + """ + Filter against the human names returned by grouped aggregation dispatch. + """ + + allowed_field_names = self._convert_allowed_field_names( + service, allowed_field_names + ) + + return ServiceType.sanitize_result(self, service, result, allowed_field_names) + + def extract_properties( + self, service: Service, path: list[str], **kwargs + ) -> list[str]: + result_properties = self._get_result_properties(service) or {} + + if not path: + return list(result_properties.keys()) + + # DataSourceDataProviderType and CurrentRecordDataProviderType strip the row + # selector from list paths before calling the service extractor. + if len(path) >= 1 and path[0] in result_properties: + return [path[0]] + + return [] diff --git a/premium/backend/tests/baserow_premium_tests/api/dashboard/test_grouped_aggregate_rows_data_source_type.py b/premium/backend/tests/baserow_premium_tests/api/dashboard/test_grouped_aggregate_rows_data_source_type.py index e4c9d6f5c8..9beb6d0f9b 100644 --- a/premium/backend/tests/baserow_premium_tests/api/dashboard/test_grouped_aggregate_rows_data_source_type.py +++ b/premium/backend/tests/baserow_premium_tests/api/dashboard/test_grouped_aggregate_rows_data_source_type.py @@ -16,6 +16,16 @@ ) +def without_grouped_row_ids(result): + return { + **result, + "results": [ + {key: value for key, value in row.items() if key != "id"} + for row in result["results"] + ], + } + + @pytest.mark.django_db def test_grouped_aggregate_rows_get_dashboard_data_sources( api_client, premium_data_fixture @@ -240,9 +250,9 @@ def test_grouped_aggregate_rows_dispatch_dashboard_data_source( dashboard = premium_data_fixture.create_dashboard_application( user=user, workspace=workspace ) - field = premium_data_fixture.create_number_field(table=table) - field_2 = premium_data_fixture.create_number_field(table=table) - field_3 = premium_data_fixture.create_number_field(table=table) + field = premium_data_fixture.create_number_field(table=table, name="Amount") + field_2 = premium_data_fixture.create_number_field(table=table, name="Quantity") + field_3 = premium_data_fixture.create_number_field(table=table, name="Score") integration = premium_data_fixture.create_local_baserow_integration( application=dashboard, user=user ) @@ -354,31 +364,178 @@ def test_grouped_aggregate_rows_dispatch_dashboard_data_source( response_json = response.json() assert response.status_code == HTTP_200_OK, response_json - assert response_json == { - "result": [ + assert without_grouped_row_ids(response_json) == { + "has_next_page": False, + "results": [ { - f"field_{field.id}": 30.0, - f"field_{field.id}_sum": 90.0, - f"field_{field_2.id}_sum": 9.0, - f"field_{field_3.id}_sum": 3.0, + "Amount": 30.0, + "Amount sum": 90.0, + "Quantity sum": 9.0, + "Score sum": 3.0, }, { - f"field_{field.id}": 20.0, - f"field_{field.id}_sum": 60.0, - f"field_{field_2.id}_sum": 6.0, - f"field_{field_3.id}_sum": 6.0, + "Amount": 20.0, + "Amount sum": 60.0, + "Quantity sum": 6.0, + "Score sum": 6.0, }, { - f"field_{field.id}": 10.0, - f"field_{field.id}_sum": 30.0, - f"field_{field_2.id}_sum": 3.0, - f"field_{field_3.id}_sum": 6.0, + "Amount": 10.0, + "Amount sum": 30.0, + "Quantity sum": 3.0, + "Score sum": 6.0, }, { - f"field_{field.id}": None, - f"field_{field.id}_sum": None, - f"field_{field_2.id}_sum": 100.0, - f"field_{field_3.id}_sum": 100.0, + "Amount": None, + "Amount sum": None, + "Quantity sum": 100.0, + "Score sum": 100.0, + }, + ], + } + + +@pytest.mark.django_db +@pytest.mark.parametrize("filter_mode", [None, "raw", "simple", "advanced"]) +def test_grouped_aggregate_rows_dispatch_dashboard_data_source_without_group_by_returns_list( + api_client, premium_data_fixture, filter_mode +): + """Chart data sources can save and execute filters in every formula mode.""" + user, token = premium_data_fixture.create_user_and_token() + workspace = premium_data_fixture.create_workspace(user=user) + database = premium_data_fixture.create_database_application(workspace=workspace) + table = premium_data_fixture.create_database_table(user=user, database=database) + dashboard = premium_data_fixture.create_dashboard_application( + user=user, workspace=workspace + ) + field = premium_data_fixture.create_number_field(table=table) + integration = premium_data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service = premium_data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + data_source = premium_data_fixture.create_dashboard_local_baserow_grouped_aggregate_rows_data_source( + dashboard=dashboard, service=service, integration_args={"user": user} + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {f"field_{field.id}": 10}, + {f"field_{field.id}": 20}, + ], + ) + if filter_mode is not None: + response = api_client.patch( + reverse( + "api:dashboard:data_sources:item", + kwargs={"data_source_id": data_source.id}, + ), + { + "filters": [ + { + "field": field.id, + "type": "equal", + "value": {"formula": "20", "mode": filter_mode}, + } + ] + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK, response.json() + assert response.json()["filters"][0]["value"]["mode"] == filter_mode + + url = reverse( + "api:dashboard:data_sources:dispatch", + kwargs={"data_source_id": data_source.id}, + ) + + response = api_client.post( + url, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + response_json = response.json() + assert response.status_code == HTTP_200_OK, response_json + assert response_json == { + "has_next_page": False, + "results": [ + { + "id": "Result", + f"{field.name} sum": 20.0 if filter_mode else 30.0, + }, + ], + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_dispatch_dashboard_data_source_uses_human_names( + api_client, premium_data_fixture +): + user, token = premium_data_fixture.create_user_and_token() + workspace = premium_data_fixture.create_workspace(user=user) + database = premium_data_fixture.create_database_application(workspace=workspace) + table = premium_data_fixture.create_database_table(user=user, database=database) + dashboard = premium_data_fixture.create_dashboard_application( + user=user, workspace=workspace + ) + field = premium_data_fixture.create_number_field(table=table, name="Amount") + group_by_field = premium_data_fixture.create_text_field( + table=table, name="Category" + ) + integration = premium_data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service = premium_data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + data_source = premium_data_fixture.create_dashboard_local_baserow_grouped_aggregate_rows_data_source( + dashboard=dashboard, service=service, integration_args={"user": user} + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + + RowHandler().create_rows( + user, + table, + rows_values=[ + {field.db_column: 10, group_by_field.db_column: "Hardware"}, + {field.db_column: 20, group_by_field.db_column: "Hardware"}, + ], + ) + url = reverse( + "api:dashboard:data_sources:dispatch", + kwargs={"data_source_id": data_source.id}, + ) + + response = api_client.post( + url, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + response_json = response.json() + assert response.status_code == HTTP_200_OK, response_json + assert without_grouped_row_ids(response_json) == { + "has_next_page": False, + "results": [ + { + "Category": "Hardware", + "Amount sum": 30.0, }, ], } diff --git a/premium/backend/tests/baserow_premium_tests/integrations/local_baserow/service_types/test_grouped_aggregate_rows_service_type.py b/premium/backend/tests/baserow_premium_tests/integrations/local_baserow/service_types/test_grouped_aggregate_rows_service_type.py index 0ef4a0119a..c9f37d127c 100644 --- a/premium/backend/tests/baserow_premium_tests/integrations/local_baserow/service_types/test_grouped_aggregate_rows_service_type.py +++ b/premium/backend/tests/baserow_premium_tests/integrations/local_baserow/service_types/test_grouped_aggregate_rows_service_type.py @@ -6,7 +6,17 @@ import pytest from pytest_unordered import unordered from rest_framework.exceptions import ValidationError +from rest_framework.reverse import reverse +from rest_framework.status import HTTP_200_OK +from baserow.contrib.builder.data_providers.data_provider_types import ( + CurrentRecordDataProviderType, + DataSourceDataProviderType, +) +from baserow.contrib.builder.elements.element_types import RecordSelectorElementType +from baserow.contrib.builder.formula_property_extractor import ( + get_element_property_names, +) from baserow.contrib.database.rows.handler import RowHandler from baserow.core.services.exceptions import ( ServiceImproperlyConfiguredDispatchException, @@ -25,9 +35,847 @@ ) -def test_grouped_aggregate_rows_service_get_schema_name(): +@pytest.fixture(autouse=True) +def enable_grouped_aggregate_rows_feature(mocker): + mocker.patch( + "baserow_premium.integrations.local_baserow.service_types." + "LicenseHandler.workspace_has_feature", + return_value=True, + ) + mocker.patch( + "baserow_premium.integrations.local_baserow.service_types." + "LicenseHandler.raise_if_workspace_doesnt_have_feature", + return_value=None, + ) + + +def without_grouped_row_ids(result): + return { + **result, + "results": [ + {key: value for key, value in row.items() if key != "id"} + for row in result["results"] + ], + } + + +def expected_grouped_dispatch_data(service, expected): + expects_unordered_results = type(expected["results"]).__name__ == "UnorderedList" + results = [ + { + key: value + for key, value in service.get_type() + ._convert_result_property_names_to_human_names(service, row) + .items() + if key != "id" + } + for row in expected["results"] + ] + + return { + **expected, + "results": unordered(results) if expects_unordered_results else results, + } + + +def test_grouped_aggregate_rows_service_get_schema_name(): + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + assert service_type.get_schema_name(Mock(id=123)) == "GroupedAggregation123Schema" + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_generate_schema_without_group_by(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + schema = service_type.generate_schema(service) + result_property = schema["items"]["properties"][f"field_{field.id}_sum"] + + assert schema["type"] == "array" + assert f"field_{field.id}_sum" in schema["items"]["properties"] + assert result_property["metadata"] == { + "display_name": "Amount sum", + "source_field": { + "id": field.id, + "name": field.db_column, + "display_name": "Amount", + }, + "aggregation": { + "type": "sum", + }, + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_generate_schema_with_group_by(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + schema = service_type.generate_schema(service) + + assert schema["type"] == "array" + assert f"field_{field.id}_sum" in schema["items"]["properties"] + assert f"field_{group_by_field.id}" in schema["items"]["properties"] + assert ( + schema["items"]["properties"][f"field_{group_by_field.id}"]["filterable"] + is False + ) + assert ( + schema["items"]["properties"][f"field_{group_by_field.id}"]["sortable"] is False + ) + assert ( + schema["items"]["properties"][f"field_{group_by_field.id}"]["searchable"] + is False + ) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_generate_schema_with_row_id_group_by( + data_fixture, +): + table = data_fixture.create_database_table() + primary_field = data_fixture.create_text_field(table=table, primary=True) + field = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=None, order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + schema = service_type.generate_schema(service) + primary_field_schema = schema["items"]["properties"][primary_field.db_column] + + assert schema["items"]["properties"]["id"]["filterable"] is False + assert primary_field_schema["filterable"] is False + assert primary_field_schema["sortable"] is False + assert primary_field_schema["searchable"] is False + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_generate_schema_allowed_fields(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + assert service_type.generate_schema(service, allowed_fields=["other"]) == { + "type": "array", + "title": service_type.get_schema_name(service), + "items": {"type": "object", "properties": {}}, + } + assert ( + f"field_{field.id}_sum" + in service_type.generate_schema( + service, allowed_fields=[f"field_{field.id}_sum"] + )["items"]["properties"] + ) + assert service_type.generate_schema(service, allowed_fields=["id"])["items"][ + "properties" + ] == { + "id": { + "title": "Id", + "type": "string", + "sortable": False, + "filterable": False, + "searchable": False, + } + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_extract_properties(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + sum_property = f"field_{field.id}_sum" + group_by_property = f"field_{group_by_field.id}" + all_properties = ["id", group_by_property, sum_property] + + assert service_type.extract_properties(service, []) == all_properties + assert service_type.extract_properties(service, [sum_property]) == [sum_property] + assert service_type.extract_properties(service, ["0", sum_property]) == [] + assert service_type.extract_properties(service, ["*", group_by_property]) == [] + assert service_type.extract_properties(service, ["0"]) == [] + assert service_type.extract_properties(service, ["*", sum_property, "value"]) == [] + assert service_type.extract_properties(service, [group_by_property, "value"]) == [ + group_by_property + ] + assert service_type.extract_properties(service, ["unknown"]) == [] + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_prepare_value_path(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + sum_property = f"field_{field.id}_sum" + group_by_property = f"field_{group_by_field.id}" + + assert service_type.prepare_value_path(service, []) == [] + assert service_type.prepare_value_path(service, ["unknown"]) == ["unknown"] + assert service_type.prepare_value_path(service, [group_by_property]) == ["Category"] + assert service_type.prepare_value_path(service, [sum_property]) == ["Amount sum"] + assert service_type.prepare_value_path(service, [sum_property, "value"]) == [ + "Amount sum", + "value", + ] + + +@pytest.mark.django_db +@pytest.mark.parametrize("group_name", ["Amount sum", "id"]) +@pytest.mark.parametrize("group_by_row_id", [False, True]) +def test_grouped_aggregate_rows_result_name_collisions( + data_fixture, group_name, group_by_row_id +): + """Colliding labels remain distinct in results, formulas and restricted schemas.""" + + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + amount = data_fixture.create_number_field(table=table, name="Amount") + group_field = data_fixture.create_text_field( + table=table, name=group_name, primary=group_by_row_id + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + integration=data_fixture.create_local_baserow_integration(user=user), + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=amount, aggregation_type="sum", order=0 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=None if group_by_row_id else group_field, order=0 + ) + RowHandler().create_rows( + user, + table, + rows_values=[{group_field.db_column: "Fruit", amount.db_column: 10}], + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + group_key = group_field.db_column + sum_key = f"{amount.db_column}_sum" + group_result_name = f"{group_name} [{group_key}]" + sum_result_name = ( + f"Amount sum [{sum_key}]" if group_name == "Amount sum" else "Amount sum" + ) + + result = ServiceHandler().dispatch_service(service, FakeDispatchContext()).data + assert result == { + "has_next_page": False, + "results": [ + {"id": "Fruit", group_result_name: "Fruit", sum_result_name: Decimal("10")} + ], + } + expected_names = { + "id": "id", + group_key: group_result_name, + sum_key: sum_result_name, + } + expected_values = {"id": "Fruit", group_key: "Fruit", sum_key: Decimal("10")} + full_schema = service_type.generate_schema(service)["items"]["properties"] + assert set(full_schema) == set(expected_names) + assert full_schema[group_key]["title"] == group_name + for key, result_name in expected_names.items(): + path = service_type.prepare_value_path(service, [key]) + assert path == [result_name] + assert result["results"][0][path[0]] == expected_values[key] + restricted_schema = service_type.generate_schema(service, allowed_fields=[key]) + assert restricted_schema["items"]["properties"] == {key: full_schema[key]} + sanitized = service_type.sanitize_result(service, result, [key]) + assert sanitized == { + "has_next_page": False, + "results": [{result_name: result["results"][0][result_name]}], + } + + +def test_grouped_aggregate_result_names_avoid_generated_suffix_collisions(): + """Generated suffixes never overwrite real labels or depend on series order.""" + + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + titles = { + "id": "Id", + "field_1": "Amount sum", + "field_2_sum": "Amount sum", + "field_3": "Amount sum [field_1]", + "field_4": "Amount sum [field_1] (2)", + } + mappings = [] + for keys in (list(titles), list(reversed(titles))): + properties = {key: {"title": titles[key]} for key in keys} + service_type._disambiguate_result_property_names(properties) + mappings.append( + { + key: prop.get("metadata", {}).get("result_name", titles[key]) + for key, prop in properties.items() + } + ) + assert ( + mappings[0] + == mappings[1] + == { + "id": "Id", + "field_1": "Amount sum [field_1] (3)", + "field_2_sum": "Amount sum [field_2_sum]", + "field_3": "Amount sum [field_1]", + "field_4": "Amount sum [field_1] (2)", + } + ) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_extract_properties(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + page = data_fixture.create_builder_page(user=user) + data_source = data_fixture.create_builder_data_source(page=page, service=service) + sum_property = f"field_{field.id}_sum" + + result = DataSourceDataProviderType().extract_properties( + [data_source.id, "0", sum_property] + ) + + assert result == {service.id: [sum_property]} + + result = DataSourceDataProviderType().extract_properties( + [data_source.id, "0", "id"] + ) + + assert result == {service.id: ["id"]} + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_can_be_used_by_record_selector( + api_client, data_fixture +): + user, token = data_fixture.create_user_and_token() + builder = data_fixture.create_builder_application(user=user) + page = data_fixture.create_builder_page(builder=builder, user=user) + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + integration = data_fixture.create_local_baserow_integration( + user=user, application=builder + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + data_source = data_fixture.create_builder_data_source(page=page, service=service) + element = data_fixture.create_builder_element( + RecordSelectorElementType, user=user, page=page + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + + url = reverse( + "api:builder:element:item", + kwargs={"element_id": element.id}, + ) + response = api_client.patch( + url, + { + "data_source_id": data_source.id, + "items_per_page": 20, + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_200_OK + assert response.json()["data_source_id"] == data_source.id + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_current_record_extract_properties(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + page = data_fixture.create_builder_page(user=user) + data_source = data_fixture.create_builder_data_source(page=page, service=service) + sum_property = f"field_{field.id}_sum" + + result = CurrentRecordDataProviderType().extract_properties( + [sum_property], data_source.id + ) + + assert result == {service.id: [sum_property]} + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_table_field_extract_properties(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + page = data_fixture.create_builder_page(user=user) + data_source = data_fixture.create_builder_data_source(page=page, service=service) + sum_property = f"field_{field.id}_sum" + table_element = data_fixture.create_builder_table_element( + page=page, + data_source=data_source, + fields=[ + { + "name": "Amount sum", + "type": "text", + "config": {"value": f"get('current_record.{sum_property}')"}, + }, + ], + ) + + result = get_element_property_names([table_element], {}) + + assert result == {"external": {service.id: unordered([sum_property, "id"])}} + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_dispatch_adds_synthetic_row_id(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + integration = data_fixture.create_local_baserow_integration(user=user) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {field.db_column: 1, group_by_field.db_column: "A"}, + {field.db_column: 2, group_by_field.db_column: "B"}, + ], + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + dispatch_context = FakeDispatchContext() + + result = service_type.dispatch_data(service, {}, dispatch_context)["data"][ + "results" + ] + + assert [row["id"] for row in result] == ["A", "B"] + dispatch_context.only_record_id = result[1]["id"] + + result = service_type.dispatch_data(service, {}, dispatch_context)["data"][ + "results" + ] + + assert len(result) == 1 + assert result[0]["id"] == "B" + + +def test_grouped_readable_ids_disambiguate_duplicate_and_suffix_labels(): + """Duplicate labels get readable suffixes which cannot overwrite real labels.""" + + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + rows = [ + {"id": name} for name in ["Sales", "Sales", "Sales (1)", "-", "-", "- (1)", "0"] + ] + service_type._set_unique_record_ids(rows) + assert [row["id"] for row in rows] == [ + "Sales (2)", + "Sales (3)", + "Sales (1)", + "- (2)", + "- (3)", + "- (1)", + "0", + ] + + +def test_grouped_record_names_do_not_dispatch(mocker): + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + dispatch = mocker.patch.object(service_type, "dispatch_data") + names = ["Sales (2)", "-", "OTHER_VALUES (2)"] + assert service_type.get_record_names(Mock(), names, FakeDispatchContext()) == { + name: name for name in names + } + dispatch.assert_not_called() + + +@pytest.mark.django_db +@pytest.mark.parametrize("row_grouping", [False, True]) +def test_grouped_readable_ids_select_one_group(data_fixture, row_grouping): + """Empty and duplicate labels each select exactly one returned group.""" + + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + category = data_fixture.create_text_field( + table=table, name="Category", primary=True + ) + amount = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + integration=data_fixture.create_local_baserow_integration(user=user), + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=amount, aggregation_type="sum", order=0 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=None if row_grouping else category, order=0 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {category.db_column: name, amount.db_column: index} + for index, name in enumerate( + ["Sales", "Sales", "Sales (1)", "", "-", "0"], start=1 + ) + ], + ) + service_type = service.get_type() + rows = service_type.dispatch_data(service, {}, FakeDispatchContext())["data"][ + "results" + ] + assert any(row["Category"] == "" for row in rows) + assert all(row["id"] for row in rows) + assert len(rows) == (6 if row_grouping else 5) + assert len({row["id"] for row in rows}) == len(rows) + assert "0" in {row["id"] for row in rows} + for row in rows: + selected = service_type.dispatch_data( + service, {}, FakeDispatchContext(only_record_id=row["id"]) + )["data"]["results"] + assert selected == [row] + + +@pytest.mark.django_db +def test_grouped_readable_ids_distinguish_overflow(data_fixture, settings): + """The overflow label cannot collide with a real OTHER_VALUES group.""" + + settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS = 3 + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + category = data_fixture.create_text_field(table=table, name="Category") + amount = data_fixture.create_number_field(table=table, name="Amount") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + integration=data_fixture.create_local_baserow_integration(user=user), + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=amount, aggregation_type="sum", order=0 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=category, order=0 + ) + LocalBaserowTableServiceAggregationSortBy.objects.create( + service=service, + sort_on="GROUP_BY", + reference=category.db_column, + direction="ASC", + order=0, + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {category.db_column: name, amount.db_column: index} + for index, name in enumerate( + ["OTHER_VALUES", "OTHER_VALUES (1)", "Z", "ZZ"], start=1 + ) + ], + ) + rows = service.get_type().dispatch_data(service, {}, FakeDispatchContext())["data"][ + "results" + ] + assert {row["id"]: row["Amount sum"] for row in rows} == { + "OTHER_VALUES (2)": 1, + "OTHER_VALUES (1)": 2, + "OTHER_VALUES (3)": 7, + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_get_record_names(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + integration = data_fixture.create_local_baserow_integration(user=user) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {field.db_column: 1, group_by_field.db_column: "A"}, + {field.db_column: 2, group_by_field.db_column: "B"}, + ], + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + assert service_type.get_name_property(service) == group_by_field.db_column + + record_names = service_type.get_record_names( + service, {"A", "B"}, FakeDispatchContext() + ) + + assert set(record_names.keys()) == {"A", "B"} + assert set(record_names.values()) == {"A", "B"} + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_record_id_uses_field_type_human_value( + data_fixture, +): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_single_select_field(table=table) + option_a = data_fixture.create_select_option( + field=group_by_field, value="Category A", color="red" + ) + option_b = data_fixture.create_select_option( + field=group_by_field, value="Category B", color="blue" + ) + integration = data_fixture.create_local_baserow_integration(user=user) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {field.db_column: 1, group_by_field.db_column: option_a.id}, + {field.db_column: 2, group_by_field.db_column: option_b.id}, + ], + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + + result = service_type.dispatch_data(service, {}, FakeDispatchContext())["data"][ + "results" + ] + record_names = service_type.get_record_names( + service, {"Category A", "Category B"}, FakeDispatchContext() + ) + + assert {row["id"] for row in result} == {"Category A", "Category B"} + assert record_names == { + "Category A": "Category A", + "Category B": "Category B", + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_data_source_get_record_names_endpoint( + api_client, data_fixture +): + user, token = data_fixture.create_user_and_token() + builder = data_fixture.create_builder_application(user=user) + page = data_fixture.create_builder_page(builder=builder, user=user) + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + integration = data_fixture.create_local_baserow_integration( + user=user, application=builder + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + data_source = data_fixture.create_builder_data_source(page=page, service=service) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[ + {field.db_column: 1, group_by_field.db_column: "A"}, + {field.db_column: 2, group_by_field.db_column: "B"}, + ], + ) + + url = reverse( + "api:builder:data_source:record-names", + kwargs={"data_source_id": data_source.id}, + ) + response = api_client.get( + f"{url}?record_ids=A,B,Not%20currently%20loaded", + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_200_OK + assert response.json() == { + "A": "A", + "B": "B", + "Not currently loaded": "Not currently loaded", + } + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_ungrouped_record_id_is_readable(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + integration = data_fixture.create_local_baserow_integration(user=user) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + RowHandler().create_rows( + user, + table, + rows_values=[{field.db_column: 1}, {field.db_column: 2}], + ) service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") - assert service_type.get_schema_name(Mock(id=123)) == "GroupedAggregation123Schema" + + result = service_type.dispatch_data(service, {}, FakeDispatchContext())["data"][ + "results" + ] + + assert result[0]["id"] == "Result" + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_sanitize_result_uses_grouped_result_keys(data_fixture): + table = data_fixture.create_database_table() + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_text_field(table=table, name="Category") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + table=table, + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + sum_property = f"field_{field.id}_sum" + group_by_property = f"field_{group_by_field.id}" + + result = service_type.sanitize_result( + service, + { + "has_next_page": False, + "results": [ + { + group_by_property: "Fruit", + sum_property: 10, + "field_unused": "removed", + } + ], + }, + [group_by_property, sum_property], + ) + + assert result == { + "has_next_page": False, + "results": [ + { + group_by_property: "Fruit", + sum_property: 10, + } + ], + } @pytest.mark.django_db @@ -68,6 +916,13 @@ def test_create_grouped_aggregate_rows_service(data_fixture): {"field_id": field_2.id, "aggregation_type": "sum"}, ], "service_aggregation_group_bys": [{"field_id": field.id}], + "service_filters": [ + { + "field": field_2, + "type": "lower_than", + "value": "5", + } + ], }, user, ) @@ -86,6 +941,11 @@ def test_create_grouped_aggregate_rows_service(data_fixture): group_bys = service.service_aggregation_group_bys.all() assert group_bys.count() == 1 assert group_bys[0].field_id == field.id + service_filters = service.service_filters.all() + assert service_filters.count() == 1 + assert service_filters[0].field_id == field_2.id + assert service_filters[0].type == "lower_than" + assert service_filters[0].value["formula"] == "5" @pytest.mark.django_db @@ -519,6 +1379,9 @@ def test_update_grouped_aggregate_rows_service(data_fixture): LocalBaserowTableServiceAggregationGroupBy.objects.create( service=service, field=field, order=1 ) + data_fixture.create_local_baserow_table_service_filter( + service=service, field=field, type="lower_than", value="5", order=1 + ) values = service_type.prepare_values( { @@ -551,6 +1414,164 @@ def test_update_grouped_aggregate_rows_service(data_fixture): group_bys = service.service_aggregation_group_bys.all() assert group_bys.count() == 1 assert group_bys[0].field_id == table_2_field.id + assert service.service_filters.count() == 0 + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "change,remaining_sort_indexes", + [ + ("remove_series", [1, 2]), + ("change_series_field", [1, 2]), + ("change_aggregation", [1, 2]), + ("remove_group", [0, 1]), + ("change_group", [0, 1]), + ("to_row_id_group", [0, 1]), + ("from_row_id_group", [0, 1]), + ("unchanged", [0, 1, 2]), + ], +) +def test_update_grouped_aggregate_rows_removes_obsolete_sorts( + data_fixture, change, remaining_sort_indexes +): + """Configuration edits drop obsolete sorts while preserving valid sorts in order.""" + + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + primary = data_fixture.create_text_field(table=table, primary=True) + amount = data_fixture.create_number_field(table=table) + other_amount = data_fixture.create_number_field(table=table) + replacement_amount = data_fixture.create_number_field(table=table) + category = data_fixture.create_text_field(table=table) + integration = data_fixture.create_local_baserow_integration(user=user) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, integration=integration, table=table + ) + series = [ + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=index + ) + for index, field in enumerate([amount, other_amount]) + ] + row_id_group = change == "from_row_id_group" + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=None if row_id_group else category, order=0 + ) + sorts = [ + LocalBaserowTableServiceAggregationSortBy.objects.create( + service=service, + sort_on=sort_on, + reference=reference, + direction=direction, + order=index, + ) + for index, (sort_on, reference, direction) in enumerate( + [ + ("SERIES", f"field_{amount.id}_sum", "ASC"), + ("SERIES", f"field_{other_amount.id}_sum", "DESC"), + ( + "GROUP_BY", + f"field_{primary.id if row_id_group else category.id}", + "ASC", + ), + ] + ) + ] + series_values = [ + {"id": item.id, "field_id": item.field_id, "aggregation_type": "sum"} + for item in series + ] + if change == "remove_series": + series_values.pop(0) + elif change == "change_series_field": + series_values[0]["field_id"] = replacement_amount.id + elif change == "change_aggregation": + series_values[0]["aggregation_type"] = "min" + + if change in { + "remove_series", + "change_series_field", + "change_aggregation", + "unchanged", + }: + values = {"service_aggregation_series": series_values} + else: + group_values = { + "remove_group": [], + "change_group": [{"field_id": primary.id}], + "to_row_id_group": [{"field_id": None}], + "from_row_id_group": [{"field_id": category.id}], + } + values = {"service_aggregation_group_bys": group_values[change]} + + # Exercise the prefetched instance used by real service update endpoints. + service = service_type.enhance_queryset( + LocalBaserowGroupedAggregateRows.objects.all() + ).get(pk=service.pk) + service = ServiceHandler().update_service(service_type, service, **values).service + + assert [ + (sort.id, sort.reference, sort.direction, sort.order) + for sort in service.service_aggregation_sorts.all() + ] == [ + ( + sorts[index].id, + sorts[index].reference, + sorts[index].direction, + sorts[index].order, + ) + for index in remaining_sort_indexes + ] + result = ServiceHandler().dispatch_service(service, FakeDispatchContext()) + assert "results" in result.data + + +@pytest.mark.django_db +def test_update_grouped_aggregate_rows_service_filters(data_fixture): + user = data_fixture.create_user() + dashboard = data_fixture.create_dashboard_application(user=user) + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table) + field_2 = data_fixture.create_number_field(table=table) + view = data_fixture.create_grid_view(user=user, table=table) + integration = data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service_type = service_type_registry.get("local_baserow_grouped_aggregate_rows") + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + view=view, + ) + data_fixture.create_local_baserow_table_service_filter( + service=service, field=field, type="lower_than", value="5", order=1 + ) + + values = service_type.prepare_values( + { + "service_filters": [ + { + "field": field_2, + "type": "higher_than", + "value": "10", + } + ], + }, + user, + service, + ) + + service = ( + ServiceHandler().update_service(service_type, service=service, **values).service + ) + + service_filters = service.service_filters.all() + assert service_filters.count() == 1 + assert service_filters[0].field_id == field_2.id + assert service_filters[0].type == "higher_than" + assert service_filters[0].value["formula"] == "10" @pytest.mark.django_db @@ -1095,12 +2116,18 @@ def test_grouped_aggregate_rows_service_dispatch(data_fixture): result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}_sum": Decimal("8"), + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}_sum": Decimal("8"), + }, + ], }, - } + ) @pytest.mark.django_db @@ -1149,20 +2176,24 @@ def test_grouped_aggregate_rows_service_dispatch_same_agg_fields(data_fixture): result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_max": Decimal("8"), - f"field_{field.id}_min": Decimal("2"), - f"field_{field_2.id}": Decimal("1"), - }, - { - f"field_{field.id}_max": Decimal("10"), - f"field_{field.id}_min": Decimal("1"), - f"field_{field_2.id}": Decimal("2"), - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_max": Decimal("8"), + f"field_{field.id}_min": Decimal("2"), + f"field_{field_2.id}": Decimal("1"), + }, + { + f"field_{field.id}_max": Decimal("10"), + f"field_{field.id}_min": Decimal("1"), + f"field_{field_2.id}": Decimal("2"), + }, + ], + }, + ) @pytest.mark.django_db @@ -1207,12 +2238,18 @@ def test_grouped_aggregate_rows_service_dispatch_with_view(data_fixture): result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": { - f"field_{field.id}_sum": Decimal("6"), - f"field_{field_2.id}_sum": Decimal("4"), + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("6"), + f"field_{field_2.id}_sum": Decimal("4"), + }, + ], }, - } + ) @pytest.mark.django_db @@ -1257,12 +2294,18 @@ def test_grouped_aggregate_rows_service_dispatch_with_service_filters(data_fixtu result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": { - f"field_{field.id}_sum": Decimal("6"), - f"field_{field_2.id}_sum": Decimal("4"), + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("6"), + f"field_{field_2.id}_sum": Decimal("4"), + }, + ], }, - } + ) @pytest.mark.django_db @@ -1549,12 +2592,18 @@ def test_grouped_aggregate_rows_service_dispatch_with_total_aggregation(data_fix result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": { - f"field_{field.id}_checked_percentage": 75.0, - f"field_{field_2.id}_not_checked_percentage": 25.0, + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_checked_percentage": 75.0, + f"field_{field_2.id}_not_checked_percentage": 25.0, + }, + ], }, - } + ) @pytest.mark.django_db @@ -1629,30 +2678,188 @@ def test_grouped_aggregate_rows_service_dispatch_group_by(data_fixture): result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("1"), - f"field_{field_2.id}_sum": Decimal("1"), - f"field_{field_3.id}": None, - }, - { - f"field_{field.id}_sum": Decimal("1"), - f"field_{field_2.id}_sum": Decimal("1"), - f"field_{field_3.id}": "Third group", - }, - { - f"field_{field.id}_sum": Decimal("8"), - f"field_{field_2.id}_sum": Decimal("6"), - f"field_{field_3.id}": "First group", - }, - { - f"field_{field.id}_sum": Decimal("22"), - f"field_{field_2.id}_sum": Decimal("7"), - f"field_{field_3.id}": "Second group", - }, - ] - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("1"), + f"field_{field_2.id}_sum": Decimal("1"), + f"field_{field_3.id}": None, + }, + { + f"field_{field.id}_sum": Decimal("1"), + f"field_{field_2.id}_sum": Decimal("1"), + f"field_{field_3.id}": "Third group", + }, + { + f"field_{field.id}_sum": Decimal("8"), + f"field_{field_2.id}_sum": Decimal("6"), + f"field_{field_3.id}": "First group", + }, + { + f"field_{field.id}_sum": Decimal("22"), + f"field_{field_2.id}_sum": Decimal("7"), + f"field_{field_3.id}": "Second group", + }, + ], + }, + ) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_dispatch_group_by_single_select( + data_fixture, +): + user = data_fixture.create_user() + dashboard = data_fixture.create_dashboard_application(user=user) + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table) + field_2 = data_fixture.create_single_select_field(table=table) + option_a = data_fixture.create_select_option( + field=field_2, value="Category A", color="red" + ) + option_b = data_fixture.create_select_option( + field=field_2, value="Category B", color="blue" + ) + integration = data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=field_2, order=1 + ) + + RowHandler().create_rows( + user, + table, + rows_values=[ + {f"field_{field.id}": 2, f"field_{field_2.id}": option_a.id}, + {f"field_{field.id}": 4, f"field_{field_2.id}": option_a.id}, + {f"field_{field.id}": 8, f"field_{field_2.id}": option_b.id}, + {f"field_{field.id}": 1, f"field_{field_2.id}": None}, + ], + ) + + dispatch_context = FakeDispatchContext() + + result = ServiceHandler().dispatch_service(service, dispatch_context) + + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": unordered( + [ + { + f"{field.name} sum": Decimal("1"), + field_2.name: None, + }, + { + f"{field.name} sum": Decimal("6"), + field_2.name: { + "id": option_a.id, + "value": "Category A", + "color": "red", + }, + }, + { + f"{field.name} sum": Decimal("8"), + field_2.name: { + "id": option_b.id, + "value": "Category B", + "color": "blue", + }, + }, + ] + ), + }, + ) + + +@pytest.mark.django_db +def test_grouped_aggregate_rows_service_dispatch_other_bucket_uses_raw_group_values( + data_fixture, settings +): + settings.BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS = 2 + user = data_fixture.create_user() + dashboard = data_fixture.create_dashboard_application(user=user) + table = data_fixture.create_database_table(user=user) + field = data_fixture.create_number_field(table=table, name="Amount") + group_by_field = data_fixture.create_single_select_field( + table=table, name="Category" + ) + option_a = data_fixture.create_select_option( + field=group_by_field, value="Category A", color="red" + ) + option_b = data_fixture.create_select_option( + field=group_by_field, value="Category B", color="blue" + ) + option_c = data_fixture.create_select_option( + field=group_by_field, value="Category C", color="green" + ) + integration = data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, + integration=integration, + table=table, + ) + LocalBaserowTableServiceAggregationSeries.objects.create( + service=service, field=field, aggregation_type="sum", order=1 + ) + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=group_by_field, order=1 + ) + LocalBaserowTableServiceAggregationSortBy.objects.create( + service=service, + sort_on="GROUP_BY", + reference=f"field_{group_by_field.id}", + order=1, + direction="ASC", + ) + + RowHandler().create_rows( + user, + table, + rows_values=[ + {f"field_{field.id}": 10, f"field_{group_by_field.id}": option_a.id}, + {f"field_{field.id}": 20, f"field_{group_by_field.id}": option_b.id}, + {f"field_{field.id}": 30, f"field_{group_by_field.id}": option_c.id}, + ], + ) + + result = ServiceHandler().dispatch_service(service, FakeDispatchContext()) + + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + "Amount sum": Decimal("10"), + "Category": { + "id": option_a.id, + "value": "Category A", + "color": "red", + }, + }, + { + "Amount sum": Decimal("50"), + "Category": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -1695,36 +2902,40 @@ def test_grouped_aggregate_rows_service_dispatch_group_by_id(data_fixture): result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": unordered( - [ - { - f"field_{field.id}": Decimal("2"), - f"field_{field.id}_sum": Decimal("2"), - f"field_{field_2.id}_sum": Decimal("2"), - "id": 1, - }, - { - f"field_{field.id}": Decimal("4"), - f"field_{field.id}_sum": Decimal("4"), - f"field_{field_2.id}_sum": Decimal("2"), - "id": 2, - }, - { - f"field_{field.id}": Decimal("6"), - f"field_{field.id}_sum": Decimal("6"), - f"field_{field_2.id}_sum": Decimal("2"), - "id": 3, - }, - { - f"field_{field.id}": Decimal("8"), - f"field_{field.id}_sum": Decimal("8"), - f"field_{field_2.id}_sum": Decimal("2"), - "id": 4, - }, - ] - ) - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": unordered( + [ + { + f"field_{field.id}": Decimal("2"), + f"field_{field.id}_sum": Decimal("2"), + f"field_{field_2.id}_sum": Decimal("2"), + "id": 1, + }, + { + f"field_{field.id}": Decimal("4"), + f"field_{field.id}_sum": Decimal("4"), + f"field_{field_2.id}_sum": Decimal("2"), + "id": 2, + }, + { + f"field_{field.id}": Decimal("6"), + f"field_{field.id}_sum": Decimal("6"), + f"field_{field_2.id}_sum": Decimal("2"), + "id": 3, + }, + { + f"field_{field.id}": Decimal("8"), + f"field_{field.id}_sum": Decimal("8"), + f"field_{field_2.id}_sum": Decimal("2"), + "id": 4, + }, + ] + ), + }, + ) @pytest.mark.django_db @@ -1839,34 +3050,38 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_series_with_group_by( result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}": Decimal("30"), - f"field_{field.id}_sum": Decimal("90"), - f"field_{field_2.id}_sum": Decimal("9"), - f"field_{field_3.id}_sum": Decimal("3"), - }, - { - f"field_{field.id}": Decimal("20"), - f"field_{field.id}_sum": Decimal("60"), - f"field_{field_2.id}_sum": Decimal("6"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - { - f"field_{field.id}": Decimal("10"), - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - { - f"field_{field.id}": None, - f"field_{field.id}_sum": None, - f"field_{field_2.id}_sum": Decimal("100"), - f"field_{field_3.id}_sum": Decimal("100"), - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}": Decimal("30"), + f"field_{field.id}_sum": Decimal("90"), + f"field_{field_2.id}_sum": Decimal("9"), + f"field_{field_3.id}_sum": Decimal("3"), + }, + { + f"field_{field.id}": Decimal("20"), + f"field_{field.id}_sum": Decimal("60"), + f"field_{field_2.id}_sum": Decimal("6"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + { + f"field_{field.id}": Decimal("10"), + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + { + f"field_{field.id}": None, + f"field_{field.id}_sum": None, + f"field_{field_2.id}_sum": Decimal("100"), + f"field_{field_3.id}_sum": Decimal("100"), + }, + ], + }, + ) @pytest.mark.django_db @@ -1952,45 +3167,49 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_series_with_group_by_ro result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}": None, - f"field_{field.id}_sum": None, - f"field_{field_2.id}_sum": Decimal("5"), - f"field_{field_3.id}_sum": Decimal("1"), - "id": 5, - }, - { - f"field_{field.id}": Decimal("3"), - f"field_{field.id}_sum": Decimal("3"), - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("2"), - "id": 4, - }, - { - f"field_{field.id}": Decimal("3"), - f"field_{field.id}_sum": Decimal("3"), - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("3"), - "id": 3, - }, - { - f"field_{field.id}": Decimal("2"), - f"field_{field.id}_sum": Decimal("2"), - f"field_{field_2.id}_sum": Decimal("2"), - f"field_{field_3.id}_sum": Decimal("3"), - "id": 2, - }, - { - f"field_{field.id}": Decimal("1"), - f"field_{field.id}_sum": Decimal("1"), - f"field_{field_2.id}_sum": Decimal("1"), - f"field_{field_3.id}_sum": Decimal("4"), - "id": 1, - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}": None, + f"field_{field.id}_sum": None, + f"field_{field_2.id}_sum": Decimal("5"), + f"field_{field_3.id}_sum": Decimal("1"), + "id": 5, + }, + { + f"field_{field.id}": Decimal("3"), + f"field_{field.id}_sum": Decimal("3"), + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("2"), + "id": 4, + }, + { + f"field_{field.id}": Decimal("3"), + f"field_{field.id}_sum": Decimal("3"), + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("3"), + "id": 3, + }, + { + f"field_{field.id}": Decimal("2"), + f"field_{field.id}_sum": Decimal("2"), + f"field_{field_2.id}_sum": Decimal("2"), + f"field_{field_3.id}_sum": Decimal("3"), + "id": 2, + }, + { + f"field_{field.id}": Decimal("1"), + f"field_{field.id}_sum": Decimal("1"), + f"field_{field_2.id}_sum": Decimal("1"), + f"field_{field_3.id}_sum": Decimal("4"), + "id": 1, + }, + ], + }, + ) @pytest.mark.django_db @@ -2073,14 +3292,19 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_series_without_group_by result = ServiceHandler().dispatch_service(service, dispatch_context) - # the results are still a dictionary, not sorted on the backend - assert result.data == { - "result": { - f"field_{field.id}_sum": Decimal("9"), - f"field_{field_2.id}_sum": Decimal("14"), - f"field_{field_3.id}_sum": Decimal("13"), - } - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("9"), + f"field_{field_2.id}_sum": Decimal("14"), + f"field_{field_3.id}_sum": Decimal("13"), + } + ], + }, + ) @pytest.mark.django_db @@ -2183,30 +3407,34 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_group_by_field(data_fix result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}": None, - f"field_{field_2.id}_sum": Decimal("100"), - f"field_{field_3.id}_sum": Decimal("100"), - }, - { - f"field_{field.id}": Decimal("10"), - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - { - f"field_{field.id}": Decimal("20"), - f"field_{field_2.id}_sum": Decimal("6"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - { - f"field_{field.id}": Decimal("30"), - f"field_{field_2.id}_sum": Decimal("9"), - f"field_{field_3.id}_sum": Decimal("3"), - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}": None, + f"field_{field_2.id}_sum": Decimal("100"), + f"field_{field_3.id}_sum": Decimal("100"), + }, + { + f"field_{field.id}": Decimal("10"), + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + { + f"field_{field.id}": Decimal("20"), + f"field_{field_2.id}_sum": Decimal("6"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + { + f"field_{field.id}": Decimal("30"), + f"field_{field_2.id}_sum": Decimal("9"), + f"field_{field_3.id}_sum": Decimal("3"), + }, + ], + }, + ) @pytest.mark.django_db @@ -2280,40 +3508,44 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_group_by_row_id(data_fi result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}": "", - f"field_{field_2.id}_sum": Decimal("5"), - f"field_{field_3.id}_sum": Decimal("1"), - "id": 5, - }, - { - f"field_{field.id}": "A", - f"field_{field_2.id}_sum": Decimal("1"), - f"field_{field_3.id}_sum": Decimal("4"), - "id": 1, - }, - { - f"field_{field.id}": "B", - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("2"), - "id": 4, - }, - { - f"field_{field.id}": "H", - f"field_{field_2.id}_sum": Decimal("2"), - f"field_{field_3.id}_sum": Decimal("3"), - "id": 2, - }, - { - f"field_{field.id}": "I", - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("3"), - "id": 3, - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}": "", + f"field_{field_2.id}_sum": Decimal("5"), + f"field_{field_3.id}_sum": Decimal("1"), + "id": 5, + }, + { + f"field_{field.id}": "A", + f"field_{field_2.id}_sum": Decimal("1"), + f"field_{field_3.id}_sum": Decimal("4"), + "id": 1, + }, + { + f"field_{field.id}": "B", + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("2"), + "id": 4, + }, + { + f"field_{field.id}": "H", + f"field_{field_2.id}_sum": Decimal("2"), + f"field_{field_3.id}_sum": Decimal("3"), + "id": 2, + }, + { + f"field_{field.id}": "I", + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("3"), + "id": 3, + }, + ], + }, + ) @pytest.mark.django_db @@ -2515,58 +3747,62 @@ def test_grouped_aggregate_rows_service_dispatch_sort_by_series_with_group_by_ig f"field_{field_3.id}": 1, }, { - f"field_{field.id}": 30, - f"field_{field_2.id}": 3, - f"field_{field_3.id}": 1, - }, - { - f"field_{field.id}": 30, - f"field_{field_2.id}": 3, - f"field_{field_3.id}": 1, - }, - # group 4 - { - f"field_{field.id}": None, - f"field_{field_2.id}": 100, - f"field_{field_3.id}": 100, - }, - ], - ) - - dispatch_context = FakeDispatchContext() - - result = ServiceHandler().dispatch_service(service, dispatch_context) - - assert result.data == { - "result": unordered( - [ - { - f"field_{field.id}": None, - f"field_{field.id}_sum": None, - f"field_{field_2.id}_sum": Decimal("100"), - f"field_{field_3.id}_sum": Decimal("100"), - }, - { - f"field_{field.id}": Decimal("10"), - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}_sum": Decimal("3"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - { - f"field_{field.id}": Decimal("30"), - f"field_{field.id}_sum": Decimal("90"), - f"field_{field_2.id}_sum": Decimal("9"), - f"field_{field_3.id}_sum": Decimal("3"), - }, - { - f"field_{field.id}": Decimal("20"), - f"field_{field.id}_sum": Decimal("60"), - f"field_{field_2.id}_sum": Decimal("6"), - f"field_{field_3.id}_sum": Decimal("6"), - }, - ] - ), - } + f"field_{field.id}": 30, + f"field_{field_2.id}": 3, + f"field_{field_3.id}": 1, + }, + { + f"field_{field.id}": 30, + f"field_{field_2.id}": 3, + f"field_{field_3.id}": 1, + }, + # group 4 + { + f"field_{field.id}": None, + f"field_{field_2.id}": 100, + f"field_{field_3.id}": 100, + }, + ], + ) + + dispatch_context = FakeDispatchContext() + + result = ServiceHandler().dispatch_service(service, dispatch_context) + + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": unordered( + [ + { + f"field_{field.id}": None, + f"field_{field.id}_sum": None, + f"field_{field_2.id}_sum": Decimal("100"), + f"field_{field_3.id}_sum": Decimal("100"), + }, + { + f"field_{field.id}": Decimal("10"), + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}_sum": Decimal("3"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + { + f"field_{field.id}": Decimal("30"), + f"field_{field.id}_sum": Decimal("90"), + f"field_{field_2.id}_sum": Decimal("9"), + f"field_{field_3.id}_sum": Decimal("3"), + }, + { + f"field_{field.id}": Decimal("20"), + f"field_{field.id}_sum": Decimal("60"), + f"field_{field_2.id}_sum": Decimal("6"), + f"field_{field_3.id}_sum": Decimal("6"), + }, + ] + ), + }, + ) @pytest.mark.django_db @@ -2638,26 +3874,30 @@ def test_grouped_aggregate_rows_service_dispatch_max_buckets_sort_on_group_by_fi result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("10"), - f"field_{field_2.id}": "A", - }, - { - f"field_{field.id}_sum": Decimal("60"), - f"field_{field_2.id}": "H", - }, - { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}": "K", - }, - { - f"field_{field.id}_sum": Decimal("120"), - f"field_{field_2.id}": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("10"), + f"field_{field_2.id}": "A", + }, + { + f"field_{field.id}_sum": Decimal("60"), + f"field_{field_2.id}": "H", + }, + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}": "K", + }, + { + f"field_{field.id}_sum": Decimal("120"), + f"field_{field_2.id}": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -2729,26 +3969,30 @@ def test_grouped_aggregate_rows_service_dispatch_max_buckets_sort_on_series( result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("10"), - f"field_{field_2.id}": "A", - }, - { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}": "K", - }, - { - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}": "L", - }, - { - f"field_{field.id}_sum": Decimal("150"), - f"field_{field_2.id}": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("10"), + f"field_{field_2.id}": "A", + }, + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}": "K", + }, + { + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}": "L", + }, + { + f"field_{field.id}_sum": Decimal("150"), + f"field_{field_2.id}": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -2824,30 +4068,34 @@ def test_grouped_aggregate_rows_service_dispatch_max_buckets_sort_on_primary_fie result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("10"), - f"field_{field_2.id}": "A", - "id": rows[3].id, - }, - { - f"field_{field.id}_sum": Decimal("60"), - f"field_{field_2.id}": "H", - "id": rows[4].id, - }, - { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}": "K", - "id": rows[1].id, - }, - { - f"field_{field.id}_sum": Decimal("120"), - f"field_{field_2.id}": "OTHER_VALUES", - "id": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("10"), + f"field_{field_2.id}": "A", + "id": rows[3].id, + }, + { + f"field_{field.id}_sum": Decimal("60"), + f"field_{field_2.id}": "H", + "id": rows[4].id, + }, + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}": "K", + "id": rows[1].id, + }, + { + f"field_{field.id}_sum": Decimal("120"), + f"field_{field_2.id}": "OTHER_VALUES", + "id": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -2913,26 +4161,30 @@ def test_grouped_aggregate_rows_service_dispatch_within_max_buckets( result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("10"), - f"field_{field_2.id}": "A", - }, - { - f"field_{field.id}_sum": Decimal("60"), - f"field_{field_2.id}": "H", - }, - { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}": "K", - }, - { - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}": "L", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("10"), + f"field_{field_2.id}": "A", + }, + { + f"field_{field.id}_sum": Decimal("60"), + f"field_{field_2.id}": "H", + }, + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}": "K", + }, + { + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}": "L", + }, + ], + }, + ) @pytest.mark.django_db @@ -2982,14 +4234,18 @@ def test_grouped_aggregate_rows_service_dispatch_1_bucket(data_fixture, settings result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("20"), - f"field_{field_2.id}": "K", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("20"), + f"field_{field_2.id}": "K", + }, + ], + }, + ) @pytest.mark.django_db @@ -3051,14 +4307,18 @@ def test_grouped_aggregate_rows_service_dispatch_1_other_bucket(data_fixture, se result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("120"), - f"field_{field_2.id}": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("120"), + f"field_{field_2.id}": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -3115,18 +4375,22 @@ def test_grouped_aggregate_rows_service_dispatch_not_within_max_buckets_no_sort( result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}": "L", - }, - { - f"field_{field.id}_sum": Decimal("90"), - f"field_{field_2.id}": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}": "L", + }, + { + f"field_{field.id}_sum": Decimal("90"), + f"field_{field_2.id}": "OTHER_VALUES", + }, + ], + }, + ) @pytest.mark.django_db @@ -3190,18 +4454,22 @@ def test_grouped_aggregate_rows_service_dispatch_not_within_max_buckets_sort_gro result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("30"), - f"field_{field_2.id}": "L", - }, - { - f"field_{field.id}_sum": Decimal("90"), - f"field_{field_2.id}": "OTHER_VALUES", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("30"), + f"field_{field_2.id}": "L", + }, + { + f"field_{field.id}_sum": Decimal("90"), + f"field_{field_2.id}": "OTHER_VALUES", + }, + ], + }, + ) # TODO: different group by field types @@ -3286,30 +4554,34 @@ def test_grouped_aggregate_rows_service_dispatch_not_within_max_buckets_sort_on_ result = ServiceHandler().dispatch_service(service, dispatch_context) - assert result.data == { - "result": [ - { - f"field_{field.id}_sum": Decimal("59"), - f"field_{field_2.id}_sum": Decimal("50"), - f"field_{field_3.id}": "OTHER_VALUES", - }, - { - f"field_{field.id}_sum": Decimal("32"), - f"field_{field_2.id}_sum": Decimal("30"), - f"field_{field_3.id}": "H", - }, - { - f"field_{field.id}_sum": Decimal("32"), - f"field_{field_2.id}_sum": Decimal("40"), - f"field_{field_3.id}": "A", - }, - { - f"field_{field.id}_sum": Decimal("31"), - f"field_{field_2.id}_sum": Decimal("30"), - f"field_{field_3.id}": "L", - }, - ], - } + assert without_grouped_row_ids(result.data) == expected_grouped_dispatch_data( + service, + { + "has_next_page": False, + "results": [ + { + f"field_{field.id}_sum": Decimal("59"), + f"field_{field_2.id}_sum": Decimal("50"), + f"field_{field_3.id}": "OTHER_VALUES", + }, + { + f"field_{field.id}_sum": Decimal("32"), + f"field_{field_2.id}_sum": Decimal("30"), + f"field_{field_3.id}": "H", + }, + { + f"field_{field.id}_sum": Decimal("32"), + f"field_{field_2.id}_sum": Decimal("40"), + f"field_{field_3.id}": "A", + }, + { + f"field_{field.id}_sum": Decimal("31"), + f"field_{field_2.id}_sum": Decimal("30"), + f"field_{field_3.id}": "L", + }, + ], + }, + ) @pytest.mark.django_db @@ -3479,3 +4751,36 @@ def test_grouped_aggregate_rows_service_import_serialized(data_fixture): assert sorts[1].direction == "DESC" assert sorts[1].sort_on == "SERIES" assert sorts[1].reference == f"field_{field_2.id}_min" + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "field_type", ["multiple_select", "multiple_collaborators", "link_row", "file"] +) +def test_grouped_aggregate_rows_rejects_multivalued_primary_row_grouping( + data_fixture, field_type +): + """Reject new and saved row grouping when the primary field contains a collection.""" + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + getattr(data_fixture, f"create_{field_type}_field")(table=table, primary=True) + dashboard = data_fixture.create_dashboard_application(user=user) + integration = data_fixture.create_local_baserow_integration( + application=dashboard, user=user + ) + service = data_fixture.create_service( + LocalBaserowGroupedAggregateRows, table=table, integration=integration + ) + service_type = service.get_type() + with pytest.raises(ValidationError, match="multi-valued primary field"): + service_type._update_service_aggregation_group_bys( + service, [{"field_id": None}] + ) + assert not service.service_aggregation_group_bys.exists() + LocalBaserowTableServiceAggregationGroupBy.objects.create( + service=service, field=None, order=1 + ) + with pytest.raises( + ServiceImproperlyConfiguredDispatchException, match="multi-valued primary field" + ): + ServiceHandler().dispatch_service(service, FakeDispatchContext()) diff --git a/premium/web-frontend/modules/baserow_premium/assets/scss/components/dashboard_chart_widget.scss b/premium/web-frontend/modules/baserow_premium/assets/scss/components/dashboard_chart_widget.scss index 6a3a4a9fbf..385e332c7e 100644 --- a/premium/web-frontend/modules/baserow_premium/assets/scss/components/dashboard_chart_widget.scss +++ b/premium/web-frontend/modules/baserow_premium/assets/scss/components/dashboard_chart_widget.scss @@ -6,6 +6,14 @@ height: 280px; } +.dashboard-chart-widget__chart { + height: 100%; +} + +.dashboard-chart-widget__chart--hidden { + visibility: hidden; +} + .dashboard-chart-widget__loading { height: 341px; diff --git a/premium/web-frontend/modules/baserow_premium/dashboard/chartFieldFormatting.js b/premium/web-frontend/modules/baserow_premium/dashboard/chartFieldFormatting.js index 0248802c31..c4bc4855cf 100644 --- a/premium/web-frontend/modules/baserow_premium/dashboard/chartFieldFormatting.js +++ b/premium/web-frontend/modules/baserow_premium/dashboard/chartFieldFormatting.js @@ -21,6 +21,10 @@ export class SingleSelectFormattingType extends ChartFieldFormattingType { } formatGroupByFieldValue(field, value) { + if (value && typeof value === 'object') { + return value.value ?? '' + } + const selectOption = field.select_options.find((item) => item.id === value) if (selectOption) { diff --git a/premium/web-frontend/modules/baserow_premium/dashboard/components/data_source/GroupedAggregateRowsDataSourceForm.vue b/premium/web-frontend/modules/baserow_premium/dashboard/components/data_source/GroupedAggregateRowsDataSourceForm.vue index c9572a0f3f..ce6b0f9fb1 100644 --- a/premium/web-frontend/modules/baserow_premium/dashboard/components/data_source/GroupedAggregateRowsDataSourceForm.vue +++ b/premium/web-frontend/modules/baserow_premium/dashboard/components/data_source/GroupedAggregateRowsDataSourceForm.vue @@ -103,20 +103,34 @@ > - - - - + + + @@ -124,10 +138,11 @@ import { useVuelidate } from '@vuelidate/core' import form from '@baserow/modules/core/mixins/form' import { required } from '@vuelidate/validators' -import AggregationSeriesForm from '@baserow_premium/dashboard/components/data_source/AggregationSeriesForm' -import AggregationGroupByForm from '@baserow_premium/dashboard/components/data_source/AggregationGroupByForm' -import AggregationSortByForm from '@baserow_premium/dashboard/components/data_source/AggregationSortByForm' +import AggregationSeriesForm from '@baserow_premium/integrations/localBaserow/components/services/AggregationSeriesForm' +import AggregationGroupByForm from '@baserow_premium/integrations/localBaserow/components/services/AggregationGroupByForm' +import AggregationSortByForm from '@baserow_premium/integrations/localBaserow/components/services/AggregationSortByForm' import tableFields from '@baserow/modules/database/mixins/tableFields' +import ServiceRefinementForms from '@baserow/modules/integrations/localBaserow/components/services/ServiceRefinementForms' const includesIfSet = (array) => (value) => { if (value === null || value === undefined) { @@ -143,6 +158,7 @@ export default { AggregationSeriesForm, AggregationGroupByForm, AggregationSortByForm, + ServiceRefinementForms, }, mixins: [form, tableFields], props: { @@ -177,6 +193,8 @@ export default { allowedValues: [ 'table_id', 'view_id', + 'filters', + 'filter_type', 'aggregation_series', 'aggregation_group_bys', 'aggregation_sorts', @@ -184,6 +202,8 @@ export default { values: { table_id: null, view_id: null, + filters: [], + filter_type: 'AND', aggregation_series: [], aggregation_group_bys: [], aggregation_sorts: [], @@ -206,6 +226,7 @@ export default { ] this.values.aggregation_group_bys = [] this.values.aggregation_sorts = [] + this.values.filters = [] // reset widget conf this.$emit('widget-values-changed', { diff --git a/premium/web-frontend/modules/baserow_premium/dashboard/components/widget/Chart.vue b/premium/web-frontend/modules/baserow_premium/dashboard/components/widget/Chart.vue index 0fdf79addb..d708baf60d 100644 --- a/premium/web-frontend/modules/baserow_premium/dashboard/components/widget/Chart.vue +++ b/premium/web-frontend/modules/baserow_premium/dashboard/components/widget/Chart.vue @@ -1,8 +1,9 @@