diff --git a/backend/src/baserow/api/ai_provider/serializers.py b/backend/src/baserow/api/ai_provider/serializers.py index 26a2f3d0f8..96a4172d47 100644 --- a/backend/src/baserow/api/ai_provider/serializers.py +++ b/backend/src/baserow/api/ai_provider/serializers.py @@ -100,6 +100,15 @@ class AIProviderModelUpdateSerializer(serializers.Serializer): ) +class AIProviderModelUsageEntrySerializer(serializers.Serializer): + feature_type = serializers.CharField() + count = serializers.IntegerField() + + +class AIProviderModelUsageSerializer(serializers.Serializer): + usage = AIProviderModelUsageEntrySerializer(many=True) + + class AIProviderFeatureModelSerializer(serializers.Serializer): id = serializers.IntegerField() model_identifier = serializers.CharField() diff --git a/backend/src/baserow/api/ai_provider/urls.py b/backend/src/baserow/api/ai_provider/urls.py index c9180c1795..692ffe14f2 100644 --- a/backend/src/baserow/api/ai_provider/urls.py +++ b/backend/src/baserow/api/ai_provider/urls.py @@ -6,6 +6,7 @@ AIProviderModelDiscoveryView, AIProviderModelsTestView, AIProviderModelsView, + AIProviderModelUsageView, AIProviderModelView, AIProvidersView, AIProviderTypesView, @@ -36,4 +37,9 @@ ), path("models/test/", AIProviderModelsTestView.as_view(), name="test_models"), path("models//", AIProviderModelView.as_view(), name="model_item"), + path( + "models//usage/", + AIProviderModelUsageView.as_view(), + name="model_usage", + ), ] diff --git a/backend/src/baserow/api/ai_provider/views.py b/backend/src/baserow/api/ai_provider/views.py index 92c6d568c1..0d968c7dd5 100644 --- a/backend/src/baserow/api/ai_provider/views.py +++ b/backend/src/baserow/api/ai_provider/views.py @@ -63,6 +63,7 @@ AIProviderModelsTestRequestSerializer, AIProviderModelsTestResponseSerializer, AIProviderModelUpdateSerializer, + AIProviderModelUsageSerializer, AIProviderModelWriteSerializer, AIProviderScopeRequestSerializer, AIProviderTypeSerializer, @@ -379,6 +380,33 @@ def delete(self, request, model_id): return Response(status=HTTP_204_NO_CONTENT) +class AIProviderModelUsageView(APIView): + permission_classes = (IsAuthenticated,) + + @extend_schema( + tags=["AI providers"], + operation_id="get_ai_provider_model_usage", + parameters=[AIProviderScopeRequestSerializer], + responses={200: AIProviderModelUsageSerializer}, + ) + @map_exceptions(EXCEPTION_MAP) + def get(self, request, model_id): + _ensure_feature_enabled() + usage = AIProviderService.get_model_usage( + request.user, model_id, workspace_id=_get_workspace_id(request) + ) + return Response( + AIProviderModelUsageSerializer( + { + "usage": [ + {"feature_type": feature_type, "count": count} + for feature_type, count in usage.items() + ] + } + ).data + ) + + class AIProviderModelsTestView(APIView): permission_classes = (IsAuthenticated,) diff --git a/backend/src/baserow/config/settings/base.py b/backend/src/baserow/config/settings/base.py index 5e63780a9e..dab26cfcdb 100644 --- a/backend/src/baserow/config/settings/base.py +++ b/backend/src/baserow/config/settings/base.py @@ -1863,3 +1863,11 @@ def install_cachalot(): BASEROW_REALTIME_REPLAY_MAX_EVENTS = int( os.getenv("BASEROW_REALTIME_REPLAY_MAX_EVENTS", 200) ) + +REALTIME_REPLAY_RETENTION_HOURS = int( + os.getenv("BASEROW_REALTIME_REPLAY_RETENTION_HOURS", 24) +) +if REALTIME_REPLAY_RETENTION_HOURS <= 0: + raise ImproperlyConfigured( + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS must be a positive integer." + ) diff --git a/backend/src/baserow/contrib/database/application_types.py b/backend/src/baserow/contrib/database/application_types.py index 52df49d192..5d8a5bb953 100755 --- a/backend/src/baserow/contrib/database/application_types.py +++ b/backend/src/baserow/contrib/database/application_types.py @@ -234,7 +234,8 @@ def export_tables_serialized( rule_type, ) in field_rules_handler.applicable_rules_with_types: exported_field_rule = field_rules_handler.export_rule(rule) - serialized_field_rules.append(exported_field_rule) + if exported_field_rule is not None: + serialized_field_rules.append(exported_field_rule) structure = DatabaseExportSerializedStructure.table( id=table.id, @@ -770,6 +771,10 @@ def _import_field_rules(self, serialized_tables, id_mapping, import_export_confi field_rules_handler = FieldRuleHandler(table) serialized_rules = serialized_table["field_rules"] for serialized_rule in serialized_rules: + # exports made before #6095 may contain null entries for rules + # that were active but invalid at export time. + if not serialized_rule: + continue field_rules_handler.import_rule( serialized_rule, id_mapping["database_fields"] ) diff --git a/backend/src/baserow/contrib/database/field_rules/handlers.py b/backend/src/baserow/contrib/database/field_rules/handlers.py index cb0e24bba9..36b21c61ef 100644 --- a/backend/src/baserow/contrib/database/field_rules/handlers.py +++ b/backend/src/baserow/contrib/database/field_rules/handlers.py @@ -596,9 +596,13 @@ def validate_rows_for_rule( return rule_type.validate_rows(self.table, rule, queryset=queryset) - def export_rule(self, rule: FieldRule): + def export_rule(self, rule: FieldRule) -> dict | None: """ Exports a rule. + + :param rule: the rule to export. + :return: the serialized rule, or None if the rule is not exportable because + it is disabled or invalid. Callers must not add None to the export payload. """ exportable = rule.is_active and rule.is_valid diff --git a/backend/src/baserow/contrib/database/migrations/0223_gridview_group_by_layout.py b/backend/src/baserow/contrib/database/migrations/0223_gridview_group_by_layout.py new file mode 100644 index 0000000000..833daa9103 --- /dev/null +++ b/backend/src/baserow/contrib/database/migrations/0223_gridview_group_by_layout.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0222_button_field_start_workflow_action"), + ] + + operations = [ + migrations.AddField( + model_name="gridview", + name="group_by_layout", + field=models.CharField( + choices=[("section", "Section"), ("column", "Column")], + db_default="section", + default="section", + max_length=10, + help_text="How grouped rows are presented: sections with a header above " + "each group, or one column per group-by level beside the rows.", + ), + ), + ] diff --git a/backend/src/baserow/contrib/database/views/models.py b/backend/src/baserow/contrib/database/views/models.py index 62405ec76f..70922253ba 100644 --- a/backend/src/baserow/contrib/database/views/models.py +++ b/backend/src/baserow/contrib/database/views/models.py @@ -610,6 +610,10 @@ class RowHeightSizes(models.TextChoices): medium = "medium" large = "large" + class GroupByLayouts(models.TextChoices): + section = "section" + column = "column" + # `field_options` is a very misleading name # it should probably be more like `fields_with_field_options` # since this field will return instances of `Field` not of @@ -628,6 +632,14 @@ class RowHeightSizes(models.TextChoices): # Number of frozen (pinned) columns including the primary field. Max defined in # the serializer. frozen_column_count = models.PositiveSmallIntegerField(default=1, db_default=1) + group_by_layout = models.CharField( + choices=GroupByLayouts.choices, + default=GroupByLayouts.section, + db_default=GroupByLayouts.section, + max_length=10, + help_text="How grouped rows are presented: sections with a header above each " + "group, or one column per group-by level beside the rows.", + ) class GridViewFieldOptionsManager(models.Manager): diff --git a/backend/src/baserow/contrib/database/views/view_types.py b/backend/src/baserow/contrib/database/views/view_types.py index ba82fdb802..2f97c26b5f 100644 --- a/backend/src/baserow/contrib/database/views/view_types.py +++ b/backend/src/baserow/contrib/database/views/view_types.py @@ -85,11 +85,17 @@ class GridViewType(ViewType): has_public_info = True can_group_by = True when_shared_publicly_requires_realtime_events = True - allowed_fields = ["row_identifier_type", "row_height_size", "frozen_column_count"] + allowed_fields = [ + "row_identifier_type", + "row_height_size", + "frozen_column_count", + "group_by_layout", + ] copyable_view_attributes = [ "row_height_size", "frozen_column_count", "row_identifier_type", + "group_by_layout", ] field_options_allowed_fields = [ "width", @@ -102,6 +108,7 @@ class GridViewType(ViewType): "row_identifier_type", "row_height_size", "frozen_column_count", + "group_by_layout", ] serializer_field_overrides = { "frozen_column_count": serializers.IntegerField( @@ -142,6 +149,7 @@ def export_serialized( serialized["row_identifier_type"] = grid.row_identifier_type serialized["row_height_size"] = grid.row_height_size serialized["frozen_column_count"] = grid.frozen_column_count + serialized["group_by_layout"] = grid.group_by_layout serialized_field_options = [] for field_option in grid.get_field_options(): diff --git a/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py index e704a22cf1..af2e6da528 100644 --- a/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py +++ b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py @@ -1,3 +1,8 @@ +from django.db.models import Exists, OuterRef + +from baserow.contrib.automation.nodes.models import AutomationNode +from baserow.contrib.builder.workflow_actions.models import AIAgentWorkflowAction +from baserow.contrib.integrations.ai.models import AIAgentService from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT from baserow.core.ai_provider.registries import AIProviderModelFeatureType from baserow.core.ai_provider.resolution import ScopedAIProviderState @@ -8,6 +13,48 @@ class AIAgentAIProviderModelFeatureType(AIProviderModelFeatureType): type = AI_PROVIDER_FEATURE_AI_AGENT + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the AI Agent services selecting one provider model. + + One service is owned by an automation node or by builder workflow + actions, and trashing an owner leaves the service row untouched, so a + service counts only while at least one live owner still reaches it. The + owner managers already encode which ancestors count as trashed. A + service whose integration or application is gone belongs to no + workspace, so the joins drop it from both scopes. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier the services persist. + :param workspace: The workspace to narrow to, or None for the instance + scope, which counts every workspace. + :return: The number of services referencing the model. + """ + + live_automation_owner = Exists( + AutomationNode.objects.filter(service_id=OuterRef("pk")) + ) + live_builder_owner = Exists( + AIAgentWorkflowAction.objects.filter( + service_id=OuterRef("pk"), page__trashed=False + ) + ) + queryset = AIAgentService.objects.filter( + ai_generative_ai_type=provider_type, + ai_generative_ai_model=model_identifier, + integration__trashed=False, + integration__application__trashed=False, + integration__application__workspace__trashed=False, + ).filter(live_automation_owner | live_builder_owner) + if workspace is not None: + queryset = queryset.filter(integration__application__workspace=workspace) + return queryset.count() + def get_workspace_availability( self, workspace: Workspace | None, diff --git a/backend/src/baserow/core/ai_provider/handler.py b/backend/src/baserow/core/ai_provider/handler.py index 4c12a0e1dc..44c8123656 100644 --- a/backend/src/baserow/core/ai_provider/handler.py +++ b/backend/src/baserow/core/ai_provider/handler.py @@ -550,6 +550,27 @@ def delete_model(model: AIProviderModel) -> None: model.delete() clear_ai_provider_state_cache() + @staticmethod + def get_model_usage(model: AIProviderModel) -> dict[str, int]: + """ + Count the consumers referencing a model, per feature. + + Default-model features are skipped: their RESTRICT foreign key already + blocks the change instead of warning about it. + + :param model: The model about to be disabled, deleted or narrowed. + :return: The reference count of every per-consumer feature. + """ + + config = model.provider_config + return { + feature_type.type: feature_type.count_model_references( + config.provider_type, model.model_identifier, config.workspace + ) + for feature_type in ai_provider_model_feature_type_registry.get_all() + if not feature_type.supports_default_model + } + @staticmethod def _registered_default_model_feature_types() -> set[str]: """ diff --git a/backend/src/baserow/core/ai_provider/registries.py b/backend/src/baserow/core/ai_provider/registries.py index 20237fbcd3..2017c7e17b 100644 --- a/backend/src/baserow/core/ai_provider/registries.py +++ b/backend/src/baserow/core/ai_provider/registries.py @@ -1,3 +1,4 @@ +from baserow.core.models import Workspace from baserow.core.registry import Instance, Registry from .constants import ( @@ -24,6 +25,29 @@ class AIProviderModelFeatureType(Instance): supports_default_model = False required_model_capabilities = (AI_PROVIDER_MODEL_CAPABILITY_TEXT,) + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the rows this feature stores referencing one provider model. + + Consumers persist a provider type and model identifier instead of the + provider row id, so an instance model and a workspace override of the + same type share a count. The result is an upper bound: it may warn an + administrator, but it must never block a change. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier consumers persist. + :param workspace: The workspace owning the provider, or None for the + instance scope, which counts every workspace. + :return: The number of references this feature holds. + """ + + return 0 + def get_workspace_availability(self, workspace, state=None) -> dict: """Return the client-facing effective availability for this feature. diff --git a/backend/src/baserow/core/ai_provider/service.py b/backend/src/baserow/core/ai_provider/service.py index 4324e7f1ea..f5dfbe81fb 100644 --- a/backend/src/baserow/core/ai_provider/service.py +++ b/backend/src/baserow/core/ai_provider/service.py @@ -239,6 +239,29 @@ def delete_model( AIProviderHandler.delete_model(model) cls._send_updated(user, workspace, True, provider_type, {model_identifier}) + @classmethod + def get_model_usage( + cls, + user: AbstractUser, + model_id: int, + workspace_id: int | None = None, + ) -> dict[str, int]: + """ + Report what still depends on a model before an admin changes it. + + Default-model features are not reported: their selection already refuses + the change with ERROR_AI_PROVIDER_MODEL_IN_USE, which names them. + + :param user: The user asking for the counts. + :param model_id: The model about to be disabled, deleted or narrowed. + :param workspace_id: The workspace scope, or None for the instance scope. + :return: The per-consumer-feature counts to warn about. + """ + + workspace = cls._check_permissions(user, workspace_id) + model = AIProviderHandler.get_model(model_id, workspace=workspace) + return AIProviderHandler.get_model_usage(model) + @classmethod def test_models( cls, diff --git a/backend/src/baserow/ws/history.py b/backend/src/baserow/ws/history.py new file mode 100644 index 0000000000..95107cb6ba --- /dev/null +++ b/backend/src/baserow/ws/history.py @@ -0,0 +1,224 @@ +"""Keep compact delivery evidence when full realtime events expire.""" + +import json +from time import monotonic + +from django.db import connection, transaction + +# Preserve delivery and recovery rules, including individual recipients' event +# types, but discard business data. Socket exclusions are summarized separately. +_ROUTING_SQL = """ +jsonb_build_object( + 'type', payload->'type', + 'send_to_all_users', COALESCE(payload->'send_to_all_users', 'false'), + 'user_ids', ( + SELECT COALESCE(jsonb_agg(DISTINCT value ORDER BY value), '[]') + FROM jsonb_array_elements( + CASE WHEN jsonb_typeof(payload->'user_ids') = 'array' + THEN payload->'user_ids' ELSE '[]' END + ) + ), + 'exclude_user_ids', ( + SELECT COALESCE(jsonb_agg(DISTINCT value ORDER BY value), '[]') + FROM jsonb_array_elements( + CASE WHEN jsonb_typeof(payload->'exclude_user_ids') = 'array' + THEN payload->'exclude_user_ids' ELSE '[]' END + ) + ), + 'payload', jsonb_build_object('type', payload #> '{payload,type}'), + 'payload_map', ( + SELECT COALESCE(jsonb_object_agg( + recipient, jsonb_build_object('type', message->'type') + ), '{}') + FROM jsonb_each( + CASE WHEN jsonb_typeof(payload->'payload_map') = 'object' + THEN payload->'payload_map' ELSE '{}' END + ) AS messages(recipient, message) + ) +) +""" + +_SELECT_EXPIRED_SQL = f""" +WITH candidates AS MATERIALIZED ( + SELECT id, channel_group, payload, target_user_ids, all_users + FROM ws_realtime_events WHERE created_at < %s + ORDER BY created_at, id LIMIT %s FOR UPDATE SKIP LOCKED +), routes AS MATERIALIZED ( + SELECT id, channel_group, {_ROUTING_SQL} AS route, + target_user_ids, all_users, + CASE WHEN jsonb_typeof(payload->'ignore_web_socket_id') = 'string' + THEN payload->>'ignore_web_socket_id' END AS ignored_socket + FROM candidates +) +SELECT id, sha256(convert_to(jsonb_build_array( + channel_group, route, target_user_ids, all_users)::text, 'UTF8')), + channel_group, route::text, target_user_ids, all_users, ignored_socket +FROM routes +""".strip() # noqa: S608 + +_UPSERT_HISTORY_SQL = """ +INSERT INTO ws_realtime_event_history_summary AS history ( + route_key, channel_group, payload, target_user_ids, all_users, + latest_event_id, latest_socket_id, previous_event_id, previous_socket_id +) +SELECT decode(route_key, 'hex'), channel_group, payload_text::jsonb, + target_user_ids, all_users, + latest_event_id, latest_socket_id, previous_event_id, previous_socket_id +FROM jsonb_to_recordset(%s::jsonb) AS batch( + route_key text, channel_group text, payload_text text, + target_user_ids integer[], all_users boolean, + latest_event_id bigint, latest_socket_id text, + previous_event_id bigint, previous_socket_id text +) +ORDER BY route_key +ON CONFLICT (route_key) DO UPDATE SET + (latest_event_id, latest_socket_id, previous_event_id, previous_socket_id) = ( + SELECT (array_agg(event_id ORDER BY event_id DESC))[1], + (array_agg(socket_id ORDER BY event_id DESC))[1], + (array_agg(event_id ORDER BY event_id DESC))[2], + (array_agg(socket_id ORDER BY event_id DESC))[2] + FROM ( + SELECT DISTINCT ON (socket_id) event_id, socket_id + FROM (VALUES + (history.latest_event_id, history.latest_socket_id), + (history.previous_event_id, history.previous_socket_id), + (excluded.latest_event_id, excluded.latest_socket_id), + (excluded.previous_event_id, excluded.previous_socket_id) + ) AS evidence(event_id, socket_id) + WHERE event_id IS NOT NULL + ORDER BY socket_id, event_id DESC + ) AS latest_per_socket + ) +RETURNING route_key, channel_group, payload::text, target_user_ids, all_users +""".strip() + +_DELETE_EXPIRED_SQL = "DELETE FROM ws_realtime_events WHERE id = ANY(%s)" + +_ADVANCE_HIGH_WATER_SQL = """ +UPDATE ws_realtime_event_history_state +SET compacted_event_id = GREATEST(compacted_event_id, %s) +WHERE id = 1 +""".strip() + +_CONFIGURE_TRANSACTION_SQL = """ +SELECT + set_config('statement_timeout', CASE + WHEN current_setting('statement_timeout')::interval = interval '0' + OR current_setting('statement_timeout')::interval > %s::interval + THEN %s ELSE current_setting('statement_timeout') END, true), + set_config('lock_timeout', CASE + WHEN current_setting('lock_timeout')::interval = interval '0' + OR current_setting('lock_timeout')::interval > %s::interval + THEN %s ELSE current_setting('lock_timeout') END, true) +""".strip() + + +class _CleanupDeadlineExceeded(Exception): + pass + + +def _make_executor(cursor, deadline, statement_timeout_ms, lock_timeout_ms): + """Apply transaction-local limits, tightening them as the budget runs out.""" + + configured_timeout_ms = None + + def execute(sql, params=None): + nonlocal configured_timeout_ms + remaining_ms = int((deadline - monotonic()) * 1000) + if remaining_ms <= 0: + raise _CleanupDeadlineExceeded + timeout_ms = min(statement_timeout_ms, remaining_ms) + if timeout_ms != configured_timeout_ms: + statement_limit = f"{timeout_ms}ms" + lock_limit = f"{lock_timeout_ms}ms" + cursor.execute( + _CONFIGURE_TRANSACTION_SQL, + [statement_limit, statement_limit, lock_limit, lock_limit], + ) + configured_timeout_ms = timeout_ms + if monotonic() >= deadline: + raise _CleanupDeadlineExceeded + cursor.execute(sql, params) + if monotonic() >= deadline: + raise _CleanupDeadlineExceeded + + return execute + + +def _summarize_candidates(candidates): + """Keep the newest two different socket exclusions for each exact audience. + + Any client can exclude at most one socket. Its latest relevant expired event + is therefore either the newest event or the newest with a different socket. + """ + + audiences = {} + for event_id, key, channel, payload, targets, all_users, socket in candidates: + key = bytes(key).hex() + audience = (channel, payload, targets, all_users) + if key not in audiences: + audiences[key] = (audience, {}) + original, sockets = audiences[key] + if original != audience: + raise RuntimeError("Realtime history route hash collision") + sockets[socket] = max(event_id, sockets.get(socket, 0)) + + summaries = [] + for key, ((channel, payload, targets, all_users), sockets) in audiences.items(): + latest, *rest = sorted(sockets.items(), key=lambda pair: pair[1], reverse=True) + previous_socket, previous_id = rest[0] if rest else (None, None) + summaries.append( + { + "route_key": key, + "channel_group": channel, + "payload_text": payload, + "target_user_ids": targets, + "all_users": all_users, + "latest_event_id": latest[1], + "latest_socket_id": latest[0], + "previous_event_id": previous_id, + "previous_socket_id": previous_socket, + } + ) + return summaries + + +def compact_events_batch( + cutoff, deadline, *, batch_size, statement_timeout_ms, lock_timeout_ms +): + """Publish expired-event evidence and delete full payloads in one commit.""" + + try: + with transaction.atomic(durable=True), connection.cursor() as cursor: + execute = _make_executor( + cursor, deadline, statement_timeout_ms, lock_timeout_ms + ) + execute("SELECT ws_initialize_realtime_history()") + execute(_SELECT_EXPIRED_SQL, [cutoff, batch_size]) + candidates = cursor.fetchall() + if not candidates: + return 0 + summaries = _summarize_candidates(candidates) + execute(_UPSERT_HISTORY_SQL, [json.dumps(summaries)]) + expected = {entry["route_key"]: entry for entry in summaries} + for key, channel, payload, targets, all_users in cursor.fetchall(): + entry = expected[bytes(key).hex()] + if (channel, payload, targets, all_users) != ( + entry["channel_group"], + entry["payload_text"], + entry["target_user_ids"], + entry["all_users"], + ): + # ON CONFLICT serializes concurrent summaries. A hash collision + # must roll back that merge as well as leave originals intact. + raise RuntimeError("Realtime history route hash collision") + ids = [row[0] for row in candidates] + execute(_DELETE_EXPIRED_SQL, [ids]) + deleted = cursor.rowcount + execute(_ADVANCE_HIGH_WATER_SQL, [max(ids)]) + if cursor.rowcount != 1: + raise RuntimeError("Realtime history state is missing") + return deleted + except _CleanupDeadlineExceeded: + # Roll back the summary and deletion together before reporting no progress. + return 0 diff --git a/backend/src/baserow/ws/migrations/0003_realtime_event_history.py b/backend/src/baserow/ws/migrations/0003_realtime_event_history.py new file mode 100644 index 0000000000..381d7c0cad --- /dev/null +++ b/backend/src/baserow/ws/migrations/0003_realtime_event_history.py @@ -0,0 +1,168 @@ +# Store expired delivery evidence separately so cleanup can delete full payloads. +# The PostgreSQL helper initializes a safe replay boundary on deployment and after +# an UNLOGGED reset. Existing event data and indexes remain unchanged; drain old +# cleanup and update readers before enabling the new cleanup. + +from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex +from django.db import migrations, models, transaction + +INITIALIZE_HISTORY = """ +CREATE OR REPLACE FUNCTION ws_initialize_realtime_history() RETURNS void +LANGUAGE plpgsql AS $function$ +DECLARE + event_sequence regclass; + sequence_value bigint; + sequence_called boolean; + history_floor bigint; +BEGIN + -- Called by migration, replay and cleanup when history state is missing. + -- UNLOGGED history can disappear after a crash; cursors below its new floor + -- must refresh because neither full events nor summaries prove their gap. + IF EXISTS (SELECT 1 FROM ws_realtime_event_history_state WHERE id = 1) THEN + RETURN; + END IF; + -- Wait for pending INSERTs before trusting their allocated IDs as a boundary. + LOCK TABLE ws_realtime_events IN SHARE MODE; + IF EXISTS (SELECT 1 FROM ws_realtime_event_history_state WHERE id = 1) THEN + RETURN; + END IF; + event_sequence := pg_get_serial_sequence('ws_realtime_events', 'id')::regclass; + -- ws.0002 keeps this sequence LOGGED. Cached IDs could otherwise be inserted + -- below the new floor after this transaction releases its table lock. + IF NOT EXISTS ( + SELECT 1 FROM pg_sequence WHERE seqrelid = event_sequence AND seqcache = 1 + ) THEN + RAISE EXCEPTION 'Realtime history requires CACHE 1 for the event sequence'; + END IF; + EXECUTE format('SELECT last_value, is_called FROM %s', event_sequence) + INTO sequence_value, sequence_called; + history_floor := CASE WHEN sequence_called THEN sequence_value ELSE 0 END; + INSERT INTO ws_realtime_event_history_state (id, floor, compacted_event_id) + VALUES (1, history_floor, history_floor) + ON CONFLICT (id) DO NOTHING; +END; +$function$; +""" + + +def _set_timeouts(cursor): + for setting, timeout in [("lock_timeout", "1s"), ("statement_timeout", "3s")]: + cursor.execute( + "SELECT set_config(%s, %s, true) WHERE " + "current_setting(%s)::interval = interval '0' OR " + "current_setting(%s)::interval > %s::interval", + [setting, timeout, setting, setting, timeout], + ) + + +def forwards(apps, schema_editor): + """Install history initialization; the scheduled task performs compaction.""" + + db = schema_editor.connection + with transaction.atomic(using=db.alias), db.cursor() as cursor: + _set_timeouts(cursor) + cursor.execute("ALTER TABLE ws_realtime_event_history_summary SET UNLOGGED") + cursor.execute("ALTER TABLE ws_realtime_event_history_state SET UNLOGGED") + cursor.execute(INITIALIZE_HISTORY) + cursor.execute("SELECT ws_initialize_realtime_history()") + + +def backwards(apps, schema_editor): + # Django drops only the two new tables afterward. Remaining full events and + # the durable sequence are preserved; rollback cannot restore deleted events. + # Disable replay and re-establish client baselines before restoring old readers. + db = schema_editor.connection + with transaction.atomic(using=db.alias), db.cursor() as cursor: + _set_timeouts(cursor) + cursor.execute("DROP FUNCTION IF EXISTS ws_initialize_realtime_history()") + + +class Migration(migrations.Migration): + dependencies = [("ws", "0002_realtime_event_indexes")] + + operations = [ + migrations.CreateModel( + name="RealtimeEventHistorySummary", + fields=[ + ("route_key", models.BinaryField(primary_key=True, serialize=False)), + ("channel_group", models.TextField()), + ("payload", models.JSONField()), + ( + "target_user_ids", + ArrayField( + models.IntegerField(), + default=list, + db_default=[], + editable=False, + ), + ), + ( + "all_users", + models.BooleanField( + default=False, db_default=False, editable=False + ), + ), + ("latest_event_id", models.BigIntegerField()), + ("latest_socket_id", models.TextField(null=True)), + ("previous_event_id", models.BigIntegerField(null=True)), + ("previous_socket_id", models.TextField(null=True)), + ], + options={ + "db_table": "ws_realtime_event_history_summary", + "indexes": [ + models.Index( + fields=["channel_group"], + name="ws_history_summary_channel_idx", + ), + GinIndex( + fields=["target_user_ids"], + condition=models.Q(channel_group="users"), + name="ws_history_summary_targets_idx", + ), + models.Index( + fields=["all_users"], + condition=models.Q(channel_group="users", all_users=True), + name="ws_history_summary_all_idx", + ), + ], + "constraints": [ + models.CheckConstraint( + condition=models.Q( + previous_event_id__isnull=True, + previous_socket_id__isnull=True, + ) + | ( + models.Q(previous_event_id__isnull=False) + & models.Q( + previous_event_id__lt=models.F("latest_event_id") + ) + ), + name="ws_history_previous_pair", + ) + ], + }, + ), + migrations.CreateModel( + name="RealtimeEventHistoryState", + fields=[ + ( + "id", + models.PositiveSmallIntegerField( + primary_key=True, serialize=False, db_default=1 + ), + ), + ("floor", models.BigIntegerField(db_default=0)), + ("compacted_event_id", models.BigIntegerField(db_default=0)), + ], + options={ + "db_table": "ws_realtime_event_history_state", + "constraints": [ + models.CheckConstraint( + condition=models.Q(id=1), name="ws_history_state_singleton" + ) + ], + }, + ), + migrations.RunPython(forwards, backwards), + ] diff --git a/backend/src/baserow/ws/models.py b/backend/src/baserow/ws/models.py index 71968a8ca8..155dc641b2 100644 --- a/backend/src/baserow/ws/models.py +++ b/backend/src/baserow/ws/models.py @@ -10,11 +10,8 @@ class RealtimeEvent(models.Model): channel_group = models.TextField() payload = models.JSONField() created_at = models.DateTimeField(auto_now_add=True) - # PostgreSQL trigger ws_realtime_event_targets_before_write populates these - # from payload on INSERT or updates to payload/channel_group. It calls - # ws_set_realtime_event_targets() from migrations/0002_realtime_event_indexes.py, - # including for older workers during a rolling deployment, so replay can use - # recipient indexes instead of scanning JSON. + # The ws_set_realtime_event_targets() pg-trigger populates these from payload on + # insert/update target_user_ids = ArrayField( models.IntegerField(), default=list, db_default=[], editable=False ) @@ -44,3 +41,74 @@ class Meta: name="ws_realtime_created_id_idx", ), ] + + +class RealtimeEventHistorySummary(models.Model): + """Expired event maxima for an exact audience, without business payloads.""" + + UNLOGGED = True + + route_key = models.BinaryField(primary_key=True) + channel_group = models.TextField() + # Routing fields only; the two socket values are stored separately so new + # browser sessions do not create new summary rows for the same audience. + payload = models.JSONField() + target_user_ids = ArrayField( + models.IntegerField(), default=list, db_default=[], editable=False + ) + all_users = models.BooleanField(default=False, db_default=False, editable=False) + latest_event_id = models.BigIntegerField() + latest_socket_id = models.TextField(null=True) + # The latest event with a different ignored socket. A null socket means the + # event ignores nobody; only a null event ID means this second pair is absent. + previous_event_id = models.BigIntegerField(null=True) + previous_socket_id = models.TextField(null=True) + + class Meta: + db_table = "ws_realtime_event_history_summary" + # Keep changing maxima out of indexes so summary updates can use HOT. + indexes = [ + models.Index( + fields=["channel_group"], name="ws_history_summary_channel_idx" + ), + GinIndex( + fields=["target_user_ids"], + condition=models.Q(channel_group="users"), + name="ws_history_summary_targets_idx", + ), + models.Index( + fields=["all_users"], + condition=models.Q(channel_group="users", all_users=True), + name="ws_history_summary_all_idx", + ), + ] + constraints = [ + models.CheckConstraint( + condition=models.Q( + previous_event_id__isnull=True, previous_socket_id__isnull=True + ) + | ( + models.Q(previous_event_id__isnull=False) + & models.Q(previous_event_id__lt=models.F("latest_event_id")) + ), + name="ws_history_previous_pair", + ) + ] + + +class RealtimeEventHistoryState(models.Model): + """Known history boundary and the high-water mark after full rows are removed.""" + + UNLOGGED = True + + id = models.PositiveSmallIntegerField(primary_key=True, db_default=1) + floor = models.BigIntegerField(db_default=0) + compacted_event_id = models.BigIntegerField(db_default=0) + + class Meta: + db_table = "ws_realtime_event_history_state" + constraints = [ + models.CheckConstraint( + condition=models.Q(id=1), name="ws_history_state_singleton" + ) + ] diff --git a/backend/src/baserow/ws/realtime_events.py b/backend/src/baserow/ws/realtime_events.py index 5f3110737f..d6134e4916 100644 --- a/backend/src/baserow/ws/realtime_events.py +++ b/backend/src/baserow/ws/realtime_events.py @@ -1,14 +1,14 @@ from __future__ import annotations +import json from dataclasses import dataclass from datetime import timedelta from time import monotonic from typing import TYPE_CHECKING, Any, Optional from django.conf import settings -from django.db import connection, transaction -from django.db.models import Max, Q -from django.db.models.functions import Coalesce +from django.db import connection +from django.db.models import Case, F, Q, When from django.db.models.query import QuerySet from django.utils import timezone @@ -25,13 +25,13 @@ from baserow.ws.consumers import SubscribedPages from baserow.ws.models import RealtimeEvent -REALTIME_EVENTS_RETENTION = timedelta(hours=24) -REALTIME_EVENTS_CLEANUP_INTERVAL_MINUTES = 1 +REALTIME_EVENTS_CLEANUP_INTERVAL_MINUTES = 10 REALTIME_EVENTS_CLEANUP_BATCH_SIZE = 5000 -REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS = 30 +# Leave headroom below Celery's default five-minute soft task limit. +REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS = 4 * 60 REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS = 3000 REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS = 250 -REALTIME_EVENTS_CLEANUP_LOCK_SECONDS = 120 +REALTIME_EVENTS_CLEANUP_LOCK_SECONDS = REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS + 90 # ``replay_events`` cursor sentinels. Must match the constants in # web-frontend/modules/core/plugins/realtimeProtocol.js. @@ -54,6 +54,7 @@ class ReplayEventsResult: # Transient infrastructure failure, rather than an unrecoverable replay gap. # Older clients still receive the force-refresh fallback. retry_after_ms: int | None = None + refresh_reason: str | None = None class RealtimeEventHandler: @@ -65,6 +66,12 @@ def is_recording_enabled() -> bool: return settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS > 0 + @staticmethod + def get_replay_retention() -> timedelta: + """Maximum age of full events accepted by replay and retained by cleanup.""" + + return timedelta(hours=settings.REALTIME_REPLAY_RETENTION_HOURS) + @staticmethod def record_events( events_data: list[tuple[str, dict[str, Any]]], @@ -188,46 +195,17 @@ def cleanup_old_realtime_events( @staticmethod def _delete_realtime_events_batch(cutoff, deadline) -> int: - """Delete one bounded oldest-first batch and commit before returning.""" - - # A caller must not accidentally turn many batches into one transaction. - # RealtimeEvent is UNLOGGED, so its storage is always on the primary DB. - with transaction.atomic(durable=True), connection.cursor() as cursor: - remaining_ms = int((deadline - monotonic()) * 1000) - if remaining_ms <= 0: - return 0 - statement_timeout = ( - f"{min(REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS, remaining_ms)}ms" - ) - lock_timeout = f"{REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS}ms" - # Both limits are transaction-local and must preserve stricter - # database/operator settings. These expressions only read settings. - cursor.execute( - "SELECT " - "set_config('statement_timeout', CASE WHEN " - "current_setting('statement_timeout')::interval = interval '0' OR " - "current_setting('statement_timeout')::interval > %s::interval " - "THEN %s ELSE current_setting('statement_timeout') END, true), " - "set_config('lock_timeout', CASE WHEN " - "current_setting('lock_timeout')::interval = interval '0' OR " - "current_setting('lock_timeout')::interval > %s::interval " - "THEN %s ELSE current_setting('lock_timeout') END, true)", - [statement_timeout, statement_timeout, lock_timeout, lock_timeout], - ) - if monotonic() >= deadline: - return 0 - # The (created_at, id) index finds the oldest bounded candidate set. - # This ephemeral log has no model deletion hooks or relationships; - # delete directly without loading payloads or collecting model rows. - cursor.execute( - "WITH expired AS MATERIALIZED (" - "SELECT id FROM ws_realtime_events WHERE created_at < %s " - "ORDER BY created_at, id LIMIT %s FOR UPDATE SKIP LOCKED" - ") DELETE FROM ws_realtime_events AS event " - "USING expired WHERE event.id = expired.id", - [cutoff, REALTIME_EVENTS_CLEANUP_BATCH_SIZE], - ) - return cursor.rowcount + """Save expired delivery evidence and delete one independently committed batch.""" + + from baserow.ws.history import compact_events_batch + + return compact_events_batch( + cutoff, + deadline, + batch_size=REALTIME_EVENTS_CLEANUP_BATCH_SIZE, + statement_timeout_ms=REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS, + lock_timeout_ms=REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS, + ) @staticmethod def get_page_group_names(pages: "SubscribedPages") -> list[str]: @@ -258,91 +236,154 @@ def get_replay_events_result( last_seen_id: int, web_socket_id: Optional[str], ) -> ReplayEventsResult: - """ - Decide how a realtime client should catch up after connecting. Only - called when replay recording is enabled — well-behaved clients skip - ``replay_events`` when the authentication handshake says it is off, - and ``_handle_replay_events`` drops the message otherwise. + """Distinguish unchanged, replayable and expired history in one snapshot.""" - :param user_id: The id of the reconnecting user. - :param page_group_names: Page channel group names the user is - subscribed to. Must not include ``"users"`` — that channel is - added unconditionally by - ``get_users_channel_live_delivery_filter``. - :param last_seen_id: ``FIRST_CONNECT_CURSOR`` for a fresh connection, - ``NO_REPLAY_AVAILABLE`` for a reconnect with no usable high-water - mark, or a positive event id the client last saw. - :param web_socket_id: The client's persistent web socket id, used to - exclude events the client itself originated. - :returns: A result containing replay events or a force-refresh instruction. - """ + from baserow.ws.models import RealtimeEvent - if last_seen_id == FIRST_CONNECT_CURSOR: - # Connecting for the first time - clients only need the latest event id to - # know where to start for future reconnects. + def refresh(reason): return ReplayEventsResult( - force_refresh=False, - latest_event_id=RealtimeEventHandler.get_latest_event_id(), - replay_events=[], + True, NO_REPLAY_AVAILABLE, [], refresh_reason=reason ) if last_seen_id == NO_REPLAY_AVAILABLE: - # Reconnect without a high-water mark — we can't prove what was - # missed, so the client has to refresh. + return refresh("missing_cursor") + if last_seen_id == FIRST_CONNECT_CURSOR: return ReplayEventsResult( - force_refresh=True, - latest_event_id=NO_REPLAY_AVAILABLE, - replay_events=[], + False, RealtimeEventHandler.get_latest_event_id(), [] ) - replay_window_events = list( - RealtimeEventHandler.get_replay_window( - user_id, page_group_names, last_seen_id, web_socket_id - ) + rows = RealtimeEventHandler._get_replay_snapshot( + user_id, + page_group_names, + last_seen_id, + web_socket_id, ) - - if replay_window_events: - # The first event must be the last seen event, and we must not exceed the - # replay limit with the remaining events, for a successful replay. - baseline = replay_window_events[0] - latest_event_id = replay_window_events[-1].id - replay_events = replay_window_events[1:] - max_events = settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS - can_replay = ( - baseline.id == last_seen_id - # Cleanup can skip a locked expired baseline while deleting - # later expired events. That surviving row cannot prove that - # the replay window is complete. - and baseline.created_at >= timezone.now() - REALTIME_EVENTS_RETENTION - and len(replay_events) <= max_events - ) - if can_replay: - return ReplayEventsResult( - force_refresh=False, - latest_event_id=latest_event_id, - replay_events=replay_events, + if not rows or last_seen_id < rows[0][0]: + return refresh("unknown_history") + latest_event_id = rows[0][1] + if last_seen_id > latest_event_id: + return refresh("cursor_ahead") + + if rows[0][2]: + return refresh("expired_payload") + + cutoff = timezone.now() - RealtimeEventHandler.get_replay_retention() + events = [] + for _, _, _, event_id, channel_group, payload, created_at in rows: + if event_id is None: + continue + if created_at < cutoff: + return refresh("expired_payload") + events.append( + RealtimeEvent( + id=event_id, + channel_group=channel_group, + payload=json.loads(payload) + if isinstance(payload, str) + else payload, + created_at=created_at, ) - - # Empty window or unable to anchor against ``last_seen_id`` — the - # cursor has expired, the client missed too many events, or the - # filter excluded the baseline. Force a refresh. + ) + if len(events) > settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS: + return refresh("event_limit") + # Do not acknowledge unrelated higher IDs: a lower relevant INSERT may + # still be uncommitted. Retain the original cursor when nothing replays. return ReplayEventsResult( - force_refresh=True, - latest_event_id=NO_REPLAY_AVAILABLE, - replay_events=[], + False, events[-1].id if events else last_seen_id, events ) + @staticmethod + def _initialize_realtime_history(): + # Usually a cheap existence check. After an UNLOGGED reset the function + # waits for in-flight inserts before recording a conservative loss floor. + with connection.cursor() as cursor: + cursor.execute("SELECT ws_initialize_realtime_history()") + @staticmethod def get_latest_event_id() -> int: - """ - Return the latest persisted realtime event id. + """Return the high-water mark, including evicted history.""" + + with connection.cursor() as cursor: + sql = ( + "SELECT GREATEST(floor, compacted_event_id, " + "COALESCE((SELECT id FROM ws_realtime_events " + "ORDER BY id DESC LIMIT 1), 0)) " + "FROM ws_realtime_event_history_state WHERE id = 1" + ) + cursor.execute(sql) + row = cursor.fetchone() + if row is None: + RealtimeEventHandler._initialize_realtime_history() + cursor.execute(sql) + row = cursor.fetchone() + return row[0] - :return: The highest event id, or ``0`` when no events exist. - """ + @staticmethod + def _get_replay_snapshot(user_id, page_group_names, last_seen_id, web_socket_id): + """Read expired evidence, payloads and loss state together across cleanup.""" - from baserow.ws.models import RealtimeEvent + events = RealtimeEventHandler.get_replay_window( + user_id, page_group_names, last_seen_id, web_socket_id + ) + history = RealtimeEventHandler.get_expired_history( + user_id, page_group_names, last_seen_id, web_socket_id + ) + events_sql, events_params = events.values_list( + "id", "channel_group", "payload", "created_at" + ).query.sql_with_params() + history_sql, history_params = history.values( + "route_key" + ).query.sql_with_params() + sql = ( + "WITH history AS MATERIALIZED (" # noqa: S608 + "SELECT state.floor, GREATEST(state.floor, state.compacted_event_id, " + "COALESCE((SELECT id FROM ws_realtime_events " + "ORDER BY id DESC LIMIT 1), 0)) AS latest_event_id, " + f"CASE WHEN state.floor <= %s THEN EXISTS({history_sql}) " + "ELSE false END AS expired " + "FROM ws_realtime_event_history_state AS state WHERE state.id = 1) " + "SELECT history.*, replay.* FROM history " + "LEFT JOIN LATERAL (" + "SELECT events.id, events.channel_group, " + "CASE WHEN events.created_at < %s THEN NULL ELSE events.payload END, " + f"events.created_at FROM ({events_sql}) AS events " + "WHERE history.floor <= %s AND NOT history.expired" + ") AS replay ON true ORDER BY replay.id" + ) + params = [ + last_seen_id, + *history_params, + timezone.now() - RealtimeEventHandler.get_replay_retention(), + *events_params, + last_seen_id, + ] + with connection.cursor() as cursor: + cursor.execute(sql, params) + rows = cursor.fetchall() + if not rows: + RealtimeEventHandler._initialize_realtime_history() + cursor.execute(sql, params) + rows = cursor.fetchall() + return rows - return RealtimeEvent.objects.aggregate(latest=Coalesce(Max("id"), 0))["latest"] + @staticmethod + def get_expired_history(user_id, page_group_names, last_seen_id, web_socket_id): + """Match compacted audiences, accounting for the client's socket exclusion.""" + + from baserow.ws.models import RealtimeEventHistorySummary + + latest = F("latest_event_id") + if web_socket_id is not None: + latest = Case( + When(latest_socket_id=web_socket_id, then=F("previous_event_id")), + default=latest, + ) + return RealtimeEventHistorySummary.objects.alias( + relevant_event_id=latest + ).filter( + RealtimeEventHandler.get_relevant_events_filter(user_id, page_group_names), + relevant_event_id__gt=last_seen_id, + ) @staticmethod def get_replay_window( @@ -351,37 +392,21 @@ def get_replay_window( last_seen_id: int, web_socket_id: Optional[str], ) -> QuerySet[RealtimeEvent]: - """ - Return the baseline event followed by replayable events. + """Return relevant payloads after the cursor, capped at limit plus one. - :param user_id: The id of the reconnecting user. - :param page_group_names: Page channel group names the user is - subscribed to. Must not include ``"users"``. - :param last_seen_id: Highest event id the client has already processed. - :param web_socket_id: The client's persistent web socket id, used to - exclude events the client itself originated. - :return: An ordered queryset containing ``last_seen_id`` when it still - exists, plus relevant events after it. The queryset is capped at - baseline plus one more than the configured replay limit so the caller - can detect that the client must refresh. + Keep the ID bound outside audience OR predicates so PostgreSQL's ordered + index scan can start at the cursor instead of visiting retained prehistory. """ from baserow.ws.models import RealtimeEvent - replay_filter = ( - Q(id__gt=last_seen_id) - & RealtimeEventHandler.get_not_own_event_filter(web_socket_id) - & RealtimeEventHandler.get_relevant_events_filter(user_id, page_group_names) - ) - - replay_filter |= Q(id=last_seen_id) - - # Keep the cursor bound outside the OR so PostgreSQL can start an ordered - # primary-key scan at the cursor. Otherwise LIMIT can select a plan that - # scans the entire retained history before reaching the baseline. return RealtimeEvent.objects.filter( - replay_filter, id__gte=last_seen_id - ).order_by("id")[: settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS + 2] + RealtimeEventHandler.get_not_own_event_filter(web_socket_id) + & RealtimeEventHandler.get_relevant_events_filter( + user_id, page_group_names + ), + id__gt=last_seen_id, + ).order_by("id")[: settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS + 1] @staticmethod def get_relevant_events_filter( @@ -425,7 +450,10 @@ def get_not_own_event_filter(web_socket_id: Optional[str]) -> Q: """ return ( - ~Q(payload__ignore_web_socket_id=web_socket_id) + ( + ~Q(payload__ignore_web_socket_id=web_socket_id) + | ~Q(payload__has_key="ignore_web_socket_id") + ) if web_socket_id is not None else Q() ) diff --git a/backend/src/baserow/ws/replay.py b/backend/src/baserow/ws/replay.py index a1daca152b..9c760ef2c1 100644 --- a/backend/src/baserow/ws/replay.py +++ b/backend/src/baserow/ws/replay.py @@ -234,7 +234,9 @@ def _get_executor(): def _force_refresh(): - return ReplayEventsResult(True, NO_REPLAY_AVAILABLE, []) + return ReplayEventsResult( + True, NO_REPLAY_AVAILABLE, [], refresh_reason="missing_cursor" + ) def _retry_later(): @@ -315,6 +317,7 @@ async def get_replay_events_result( started_at = monotonic() outcome = "error" + reason = "none" try: if last_seen_id == NO_REPLAY_AVAILABLE: result = _force_refresh() @@ -341,6 +344,7 @@ async def get_replay_events_result( result = await asyncio.shield(task) if result.force_refresh: outcome = "refresh" + reason = result.refresh_reason or "unknown" elif last_seen_id == FIRST_CONNECT_CURSOR: outcome = "baseline" else: @@ -362,6 +366,6 @@ async def get_replay_events_result( outcome = "cancelled" raise finally: - attributes = {"process.pid": os.getpid(), "outcome": outcome} + attributes = {"process.pid": os.getpid(), "outcome": outcome, "reason": reason} websocket_replay_requests.add(1, attributes) websocket_replay_duration.record((monotonic() - started_at) * 1000, attributes) diff --git a/backend/src/baserow/ws/tasks.py b/backend/src/baserow/ws/tasks.py index c13b3a57b1..d8af063a28 100644 --- a/backend/src/baserow/ws/tasks.py +++ b/backend/src/baserow/ws/tasks.py @@ -861,7 +861,6 @@ def cleanup_old_realtime_events(self): from baserow.ws.realtime_events import ( REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS, REALTIME_EVENTS_CLEANUP_LOCK_SECONDS, - REALTIME_EVENTS_RETENTION, RealtimeEventHandler, ) from baserow.ws.telemetry import record_realtime_cleanup_skipped @@ -882,7 +881,7 @@ def cleanup_old_realtime_events(self): return try: return RealtimeEventHandler.cleanup_old_realtime_events( - REALTIME_EVENTS_RETENTION, deadline=deadline + RealtimeEventHandler.get_replay_retention(), deadline=deadline ) finally: try: diff --git a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py index 74288c7c59..0a0b6781aa 100644 --- a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py +++ b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py @@ -15,6 +15,7 @@ ) from baserow.core.ai_provider.constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_TEST_MAX_TOKENS, AI_PROVIDER_TEST_TIMEOUT_SECONDS, @@ -39,6 +40,7 @@ ("post", "test_models"), ("patch", "model_item"), ("delete", "model_item"), + ("get", "model_usage"), ) @@ -83,7 +85,7 @@ def _request_ai_provider_api(api_client, case, headers): data = {"provider_type": "openai"} elif url_name == "test_models": data = {"model_ids": [model.id]} - elif url_name == "model_item": + elif url_name in ("model_item", "model_usage"): kwargs = {"model_id": model.id} if method == "patch": data = {"is_enabled": False} @@ -881,6 +883,152 @@ def test_models_can_be_created_updated_disabled_and_deleted( assert not AIProviderModel.objects.exists() +@pytest.mark.django_db +def test_model_usage_reports_every_per_consumer_feature( + api_client, staff_headers, enabled_ai_providers +): + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}), + **staff_headers, + ) + + assert response.status_code == HTTP_200_OK + assert response.json() == { + "usage": [ + {"feature_type": AI_PROVIDER_FEATURE_AI_AGENT, "count": 0}, + {"feature_type": AI_PROVIDER_FEATURE_AI_FIELDS, "count": 0}, + ] + } + + +@pytest.mark.django_db +def test_model_usage_omits_default_model_features_which_the_error_reports( + api_client, staff_headers, enabled_ai_providers +): + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5", feature_types=["kuma"] + ) + response = api_client.put( + reverse("api:ai_provider:feature_item", kwargs={"feature_type": "kuma"}), + {"mode": "model", "model_id": model.id}, + format="json", + **staff_headers, + ) + assert response.status_code == HTTP_200_OK + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}), + **staff_headers, + ) + + assert response.status_code == HTTP_200_OK + usage = response.json()["usage"] + assert "kuma" not in [entry["feature_type"] for entry in usage] + assert all(entry["count"] == 0 for entry in usage) + + response = api_client.delete( + reverse("api:ai_provider:model_item", kwargs={"model_id": model.id}), + **staff_headers, + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_IN_USE" + assert "kuma" in response.json()["detail"] + + +@pytest.mark.django_db +def test_model_usage_is_scoped_like_the_other_model_endpoints( + api_client, data_fixture, staff_headers, enabled_ai_providers +): + user, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=user) + headers = {"HTTP_AUTHORIZATION": f"JWT {token}"} + workspace_query = f"?workspace_id={workspace.id}" + instance_provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="instance-secret" + ) + instance_model = AIProviderModel.objects.create( + provider_config=instance_provider, model_identifier="gpt-5" + ) + workspace_provider = AIProviderConfig.objects.create( + provider_type="mistral", api_key="workspace-secret", workspace=workspace + ) + workspace_model = AIProviderModel.objects.create( + provider_config=workspace_provider, model_identifier="mistral-large" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": workspace_model.id}) + + workspace_query, + **headers, + ) + assert response.status_code == HTTP_200_OK + assert [entry["feature_type"] for entry in response.json()["usage"]] == [ + AI_PROVIDER_FEATURE_AI_AGENT, + AI_PROVIDER_FEATURE_AI_FIELDS, + ] + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": instance_model.id}) + + workspace_query, + **headers, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": workspace_model.id}), + **staff_headers, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + +@pytest.mark.django_db +def test_model_usage_requires_workspace_admin_permission( + api_client, data_fixture, enabled_ai_providers +): + owner = data_fixture.create_user() + member, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=owner) + data_fixture.create_user_workspace( + user=member, workspace=workspace, permissions="MEMBER" + ) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="secret", workspace=workspace + ) + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}) + + f"?workspace_id={workspace.id}", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_USER_INVALID_GROUP_PERMISSIONS" + + +@pytest.mark.django_db +def test_model_usage_of_an_unknown_model_returns_not_found( + api_client, staff_headers, enabled_ai_providers +): + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": 999999}), + **staff_headers, + ) + + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + @pytest.mark.django_db def test_saved_models_are_tested_in_one_request_and_results_are_persisted( api_client, staff_headers, enabled_ai_providers diff --git a/backend/tests/baserow/config/test_settings_validation.py b/backend/tests/baserow/config/test_settings_validation.py new file mode 100644 index 0000000000..c9ac70389c --- /dev/null +++ b/backend/tests/baserow/config/test_settings_validation.py @@ -0,0 +1,39 @@ +import os +import subprocess +import sys + +import pytest + + +def _import_base_settings(retention_hours: str) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join(sys.path), + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS": retention_hours, + } + return subprocess.run( # noqa: S603 - only the current interpreter runs fixed code. + [sys.executable, "-c", "import baserow.config.settings.base"], + check=False, + capture_output=True, + env=env, + text=True, + ) + + +@pytest.mark.parametrize("retention_hours", ["0", "-1"]) +def test_non_positive_realtime_replay_retention_is_rejected(retention_hours: str): + """ + A non-positive retention disables compaction instead of failing, so the guard + in ``settings.base`` is the only thing stopping it. + + The settings module validates at import time, so the check can only be + exercised from a fresh interpreter; the ``settings`` fixture never re-runs it. + """ + + result = _import_base_settings(retention_hours) + + assert result.returncode != 0 + assert ( + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS must be a positive integer" + in result.stderr + ) diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py index 4625b74a31..3eaec9e4e3 100644 --- a/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py +++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py @@ -641,6 +641,7 @@ def test_to_baserow_database_export(): "row_identifier_type": "count", "row_height_size": "small", "frozen_column_count": 1, + "group_by_layout": "section", "filter_type": "AND", "filters_disabled": False, "filters": [], diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py index f5a5535d5b..008b0302e1 100644 --- a/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py +++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py @@ -263,6 +263,7 @@ def test_import_grid_view(): "public": False, "row_height_size": "medium", "frozen_column_count": 1, + "group_by_layout": "section", "row_identifier_type": "count", "sortings": [], "type": "grid", diff --git a/backend/tests/baserow/contrib/database/api/views/grid/test_grid_view_views.py b/backend/tests/baserow/contrib/database/api/views/grid/test_grid_view_views.py index ea52f8e03f..1f36662f87 100644 --- a/backend/tests/baserow/contrib/database/api/views/grid/test_grid_view_views.py +++ b/backend/tests/baserow/contrib/database/api/views/grid/test_grid_view_views.py @@ -3482,6 +3482,7 @@ def test_get_public_grid_view(api_client, data_fixture): "row_identifier_type": grid_view.row_identifier_type, "row_height_size": grid_view.row_height_size, "frozen_column_count": 1, + "group_by_layout": grid_view.group_by_layout, "show_logo": True, "allow_public_export": False, "ownership_type": "collaborative", @@ -5820,3 +5821,36 @@ def test_list_rows_group_by_alone_applies_ordering(api_client, data_fixture): results = response.json()["results"] result_ids = [r["id"] for r in results] assert result_ids == [row_a.id, row_b.id, row_c.id] + + +@pytest.mark.django_db +def test_update_grid_view_group_by_layout(api_client, data_fixture): + user, token = data_fixture.create_user_and_token() + table = data_fixture.create_database_table(user=user) + grid_view = data_fixture.create_grid_view(table=table) + url = reverse("api:database:views:item", kwargs={"view_id": grid_view.id}) + + response = api_client.get(url, format="json", HTTP_AUTHORIZATION=f"JWT {token}") + assert response.status_code == HTTP_200_OK + assert response.json()["group_by_layout"] == "section" + + for group_by_layout in ("column", "section"): + response = api_client.patch( + url, + {"group_by_layout": group_by_layout}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json()["group_by_layout"] == group_by_layout + grid_view.refresh_from_db() + assert grid_view.group_by_layout == group_by_layout + + response = api_client.patch( + url, + {"group_by_layout": "vertical"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_REQUEST_BODY_VALIDATION" diff --git a/backend/tests/baserow/contrib/database/api/views/test_view_copy_configuration.py b/backend/tests/baserow/contrib/database/api/views/test_view_copy_configuration.py index 5d8851d41e..38db0313a7 100644 --- a/backend/tests/baserow/contrib/database/api/views/test_view_copy_configuration.py +++ b/backend/tests/baserow/contrib/database/api/views/test_view_copy_configuration.py @@ -22,6 +22,7 @@ def test_copy_view_configuration(api_client, data_fixture): filter_type="OR", row_height_size="large", frozen_column_count=2, + group_by_layout="column", ) dest_view = data_fixture.create_grid_view(table=table) @@ -58,6 +59,7 @@ def test_copy_view_configuration(api_client, data_fixture): assert response_json["filter_type"] == "OR" assert response_json["row_height_size"] == "large" assert response_json["frozen_column_count"] == 2 + assert response_json["group_by_layout"] == "column" assert len(response_json["filters"]) == 1 assert response_json["filters"][0]["value"] == "a" assert len(response_json["filter_groups"]) == 1 diff --git a/backend/tests/baserow/contrib/database/api/views/test_view_views.py b/backend/tests/baserow/contrib/database/api/views/test_view_views.py index 20f7a0177d..08e3cb8b7c 100644 --- a/backend/tests/baserow/contrib/database/api/views/test_view_views.py +++ b/backend/tests/baserow/contrib/database/api/views/test_view_views.py @@ -1019,6 +1019,7 @@ def test_user_with_password_can_get_info_about_a_public_password_protected_view( "row_identifier_type": grid_view.row_identifier_type, "row_height_size": grid_view.row_height_size, "frozen_column_count": 1, + "group_by_layout": grid_view.group_by_layout, "show_logo": grid_view.show_logo, "allow_public_export": grid_view.allow_public_export, "ownership_type": "collaborative", @@ -1051,6 +1052,7 @@ def test_user_with_password_can_get_info_about_a_public_password_protected_view( "row_identifier_type": grid_view.row_identifier_type, "row_height_size": grid_view.row_height_size, "frozen_column_count": 1, + "group_by_layout": grid_view.group_by_layout, "show_logo": grid_view.show_logo, "allow_public_export": grid_view.allow_public_export, "ownership_type": "collaborative", diff --git a/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py b/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py new file mode 100644 index 0000000000..60c3ca4e3e --- /dev/null +++ b/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py @@ -0,0 +1,70 @@ +import pytest + +from baserow.contrib.database.field_rules.handlers import FieldRuleHandler +from baserow.contrib.database.field_rules.models import FieldRule +from baserow.core.registries import ImportExportConfig, application_type_registry + + +def _create_table_with_rule(data_fixture, is_valid: bool): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + database = data_fixture.create_database_application(workspace=workspace) + table = data_fixture.create_database_table(database=database) + data_fixture.create_text_field(table=table, name="text") + + rule = FieldRuleHandler(table, user).create_rule("dummy", {}) + FieldRule.objects.filter(id=rule.id).update(is_valid=is_valid) + return user, database, table + + +@pytest.mark.django_db +def test_export_serialized_skips_invalid_field_rules( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=False) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + assert serialized["tables"][0]["field_rules"] == [] + + +@pytest.mark.django_db +def test_import_serialized_with_invalid_field_rule( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=False) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + imported_workspace = data_fixture.create_workspace(user=user) + imported_database = database_type.import_serialized( + imported_workspace, serialized, config, {}, None, None + ) + + imported_table = imported_database.table_set.get() + assert not FieldRule.objects.filter(table=imported_table).exists() + + +@pytest.mark.django_db +def test_import_serialized_with_valid_field_rule( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=True) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + assert [r["type"] for r in serialized["tables"][0]["field_rules"]] == ["dummy"] + + imported_workspace = data_fixture.create_workspace(user=user) + imported_database = database_type.import_serialized( + imported_workspace, serialized, config, {}, None, None + ) + + imported_table = imported_database.table_set.get() + assert FieldRule.objects.filter(table=imported_table).count() == 1 diff --git a/backend/tests/baserow/contrib/database/view/test_view_configuration_copy.py b/backend/tests/baserow/contrib/database/view/test_view_configuration_copy.py index 7b7be51b60..40deaeacf4 100644 --- a/backend/tests/baserow/contrib/database/view/test_view_configuration_copy.py +++ b/backend/tests/baserow/contrib/database/view/test_view_configuration_copy.py @@ -163,6 +163,7 @@ def test_copy_view_configuration_field_options_and_row_height(data_fixture): row_height_size="large", frozen_column_count=3, row_identifier_type="count", + group_by_layout="column", ) dest_view = data_fixture.create_grid_view(table=table, row_height_size="small") @@ -184,6 +185,7 @@ def test_copy_view_configuration_field_options_and_row_height(data_fixture): assert dest_view.row_height_size == "large" assert dest_view.frozen_column_count == 3 assert dest_view.row_identifier_type == "count" + assert dest_view.group_by_layout == "column" dest_options = dest_view.gridviewfieldoptions_set.get(field=field) assert dest_options.hidden is True assert dest_options.width == 300 diff --git a/backend/tests/baserow/contrib/database/view/test_view_types.py b/backend/tests/baserow/contrib/database/view/test_view_types.py index 041afaac07..139e0a1f49 100644 --- a/backend/tests/baserow/contrib/database/view/test_view_types.py +++ b/backend/tests/baserow/contrib/database/view/test_view_types.py @@ -39,6 +39,7 @@ def test_import_export_grid_view(data_fixture): filter_type="AND", filters_disabled=False, row_identifier_type="count", + group_by_layout="column", ) field = data_fixture.create_text_field(table=grid_view.table) imported_field = data_fixture.create_text_field(table=grid_view.table) @@ -85,6 +86,7 @@ def test_import_export_grid_view(data_fixture): assert grid_view.filter_type == imported_grid_view.filter_type assert grid_view.filters_disabled == imported_grid_view.filters_disabled assert grid_view.row_identifier_type == imported_grid_view.row_identifier_type + assert imported_grid_view.group_by_layout == "column" assert imported_grid_view.viewfilter_set.all().count() == 1 assert imported_grid_view.viewsort_set.all().count() == 1 diff --git a/backend/tests/baserow/contrib/database/view/test_view_webhook_event_types.py b/backend/tests/baserow/contrib/database/view/test_view_webhook_event_types.py index 7c57a0adb9..a40ec3ee9f 100644 --- a/backend/tests/baserow/contrib/database/view/test_view_webhook_event_types.py +++ b/backend/tests/baserow/contrib/database/view/test_view_webhook_event_types.py @@ -51,6 +51,7 @@ def test_view_created_event_type(data_fixture): "slug": view.slug, "row_height_size": "small", "frozen_column_count": 1, + "group_by_layout": "section", }, } @@ -95,6 +96,7 @@ def test_view_created_event_type_test_payload(data_fixture): "public": False, "row_height_size": "small", "frozen_column_count": 1, + "group_by_layout": "section", }, } @@ -150,6 +152,7 @@ def test_view_updated_event_type(data_fixture): "slug": view.slug, "row_height_size": "small", "frozen_column_count": 1, + "group_by_layout": "section", }, } @@ -194,6 +197,7 @@ def test_view_updated_event_type_test_payload(data_fixture): "public": False, "row_height_size": "small", "frozen_column_count": 1, + "group_by_layout": "section", }, } diff --git a/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py index 5c7ea6e7da..3d8c79d9a8 100644 --- a/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py +++ b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py @@ -1,11 +1,14 @@ import pytest +from baserow.contrib.builder.workflow_actions.models import AIAgentWorkflowAction +from baserow.contrib.integrations.ai.models import AIIntegration from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT from baserow.core.ai_provider.handler import AIProviderHandler from baserow.core.ai_provider.models import AIProviderConfig, AIProviderModel from baserow.core.ai_provider.registries import ( ai_provider_model_feature_type_registry, ) +from baserow.core.trash.handler import TrashHandler def test_ai_agent_feature_type_is_registered(): @@ -56,3 +59,220 @@ def test_agent_selection_does_not_block_model_deletion(data_fixture, settings): AIProviderHandler.delete_model(model) assert not AIProviderModel.objects.filter(id=model.id).exists() + + +def _attach_builder_owner(data_fixture, service, page, trashed=False): + action = data_fixture.create_workflow_action( + AIAgentWorkflowAction, + page=page, + element=data_fixture.create_builder_button_element(page=page), + service=service, + ) + if trashed: + action.trashed = True + action.save() + return action + + +@pytest.mark.django_db +def test_ai_agent_count_model_references(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + workspace = page.builder.workspace + integration = data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ) + + def add_service(**kwargs): + service = data_fixture.create_ai_agent_service( + integration=integration, **kwargs + ) + _attach_builder_owner(data_fixture, service, page) + return service + + add_service(ai_generative_ai_type="openai", ai_generative_ai_model="agent-model") + add_service(ai_generative_ai_type="openai", ai_generative_ai_model="another-model") + add_service(ai_generative_ai_type="anthropic", ai_generative_ai_model="agent-model") + add_service( + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + trashed=True, + ) + other_page = data_fixture.create_builder_page(user=user) + other_workspace = other_page.builder.workspace + other_service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=other_page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, other_service, other_page) + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + assert ( + feature_type.count_model_references("openai", "agent-model", other_workspace) + == 1 + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 2 + assert feature_type.count_model_references("openai", "unknown-model", None) == 0 + assert feature_type.count_model_references("mistral", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_ancestors(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, service, page) + application = service.integration.application + workspace = application.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + + application.trashed = True + application.save() + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + application.trashed = False + application.save() + workspace.trashed = True + workspace.save() + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_skips_services_without_integration( + data_fixture, +): + data_fixture.create_ai_agent_service( + integration=None, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert ( + feature_type.count_model_references( + "openai", "agent-model", data_fixture.create_workspace() + ) + == 0 + ) + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_automation_owners( + data_fixture, +): + user = data_fixture.create_user() + workflow = data_fixture.create_automation_workflow(user=user) + integration = data_fixture.create_integration( + AIIntegration, application=workflow.automation, user=user + ) + node = data_fixture.create_automation_node( + user=user, + workflow=workflow, + type="ai_agent", + service=data_fixture.create_ai_agent_service( + integration=integration, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ), + ) + workspace = workflow.automation.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + + TrashHandler.trash(user, workspace, workflow.automation, node) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + TrashHandler.restore_item(user, "automation_node", node.id) + + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + + TrashHandler.trash(user, workspace, workflow.automation, workflow) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_builder_owners(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + element = data_fixture.create_builder_button_element(page=page) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + action = data_fixture.create_workflow_action( + AIAgentWorkflowAction, page=page, element=element, service=service + ) + workspace = page.builder.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + + action.trashed = True + action.save() + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + action.trashed = False + action.save() + TrashHandler.trash(user, workspace, page.builder, element) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + TrashHandler.restore_item(user, "builder_element", element.id) + TrashHandler.trash(user, workspace, page.builder, page) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_keeps_services_with_one_live_owner( + data_fixture, +): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, service, page) + _attach_builder_owner(data_fixture, service, page, trashed=True) + workspace = page.builder.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 diff --git a/backend/tests/baserow/core/ai_provider/test_handler.py b/backend/tests/baserow/core/ai_provider/test_handler.py index 5961ddc899..7b67d3534c 100644 --- a/backend/tests/baserow/core/ai_provider/test_handler.py +++ b/backend/tests/baserow/core/ai_provider/test_handler.py @@ -904,3 +904,103 @@ class SecondFeatureType(AIProviderModelFeatureType): assert exc_info.value.model_identifier == "first-model" assert exc_info.value.feature_types == ["first_feature"] assert second_model.id + + +@pytest.mark.django_db +def test_get_model_usage_counts_per_consumer_features_only(monkeypatch): + class CountingFeatureType(AIProviderModelFeatureType): + type = "counting_feature" + + def count_model_references( + self, provider_type, model_identifier, workspace=None + ): + return 3 + + class PlainFeatureType(AIProviderModelFeatureType): + type = "plain_feature" + + class DefaultModelFeatureType(AIProviderModelFeatureType): + type = "default_model_feature" + supports_default_model = True + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + { + "counting_feature": CountingFeatureType(), + "plain_feature": PlainFeatureType(), + "default_model_feature": DefaultModelFeatureType(), + }, + ) + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-4o" + ) + + assert AIProviderHandler.get_model_usage(model) == { + "counting_feature": 3, + "plain_feature": 0, + } + + +@pytest.mark.django_db +def test_get_model_usage_passes_the_provider_scope(monkeypatch, data_fixture): + calls = [] + + class RecordingFeatureType(AIProviderModelFeatureType): + type = "recording_feature" + + def count_model_references( + self, provider_type, model_identifier, workspace=None + ): + calls.append((provider_type, model_identifier, workspace)) + return 0 + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + {"recording_feature": RecordingFeatureType()}, + ) + workspace = data_fixture.create_workspace() + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="secret", workspace=workspace + ) + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-4o" + ) + + assert AIProviderHandler.get_model_usage(model) == {"recording_feature": 0} + assert calls == [("openai", "gpt-4o", workspace)] + + +@pytest.mark.django_db +def test_get_model_usage_excludes_selected_default_model_feature_types(monkeypatch): + class ConsumerFeatureType(AIProviderModelFeatureType): + type = "consumer_feature" + + class DefaultModelFeatureType(AIProviderModelFeatureType): + type = "default_model_feature" + supports_default_model = True + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + { + "consumer_feature": ConsumerFeatureType(), + "default_model_feature": DefaultModelFeatureType(), + }, + ) + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, + model_identifier="gpt-4o", + feature_types=["consumer_feature", "default_model_feature"], + ) + + assert AIProviderHandler.get_model_usage(model) == {"consumer_feature": 0} + + AIProviderHandler.update_feature_setting( + "default_model_feature", AI_PROVIDER_FEATURE_MODE_MODEL, model=model + ) + + assert AIProviderHandler.get_model_usage(model) == {"consumer_feature": 0} diff --git a/backend/tests/baserow/ws/conftest.py b/backend/tests/baserow/ws/conftest.py index 461c5b0767..7630efb6d2 100644 --- a/backend/tests/baserow/ws/conftest.py +++ b/backend/tests/baserow/ws/conftest.py @@ -26,10 +26,21 @@ def _install_realtime_targets(django_db_setup, django_db_blocker): migration.forwards(None, editor) +@pytest.fixture(scope="session") +def _install_realtime_history(_install_realtime_targets, django_db_blocker): + migration = import_module("baserow.ws.migrations.0003_realtime_event_history") + with django_db_blocker.unblock(), connection.schema_editor(atomic=False) as editor: + migration.forwards(None, editor) + + @pytest.fixture -def _django_db_helper(_install_realtime_targets, _django_db_helper): - # Wrapping pytest-django's helper does not request a DB for non-DB tests. - return _django_db_helper +def _django_db_helper(_install_realtime_history, _django_db_helper): + # Initialize known history inside pytest-django's per-test transaction. + from baserow.ws.models import RealtimeEventHistoryState + + RealtimeEventHistoryState.objects.update_or_create( + pk=1, defaults={"floor": 0, "compacted_event_id": 0} + ) @pytest.fixture(autouse=True) diff --git a/backend/tests/baserow/ws/test_ws_history_migration.py b/backend/tests/baserow/ws/test_ws_history_migration.py new file mode 100644 index 0000000000..cb5877ada5 --- /dev/null +++ b/backend/tests/baserow/ws/test_ws_history_migration.py @@ -0,0 +1,262 @@ +from contextlib import closing +from datetime import timedelta +from importlib import import_module +from time import monotonic + +from django.db import DatabaseError, OperationalError, connection, transaction +from django.db.migrations.loader import MigrationLoader +from django.utils import timezone + +import pytest + +from baserow.ws import history +from baserow.ws.models import ( + RealtimeEvent, + RealtimeEventHistoryState, + RealtimeEventHistorySummary, +) +from baserow.ws.realtime_events import RealtimeEventHandler + +pytestmark = [pytest.mark.django_db(transaction=True), pytest.mark.websockets] + + +def record(channel="table-1", socket=None): + return RealtimeEventHandler.record_events( + [ + ( + channel, + { + "type": "broadcast_to_group", + "ignore_web_socket_id": socket, + "payload": {"type": "rows_updated", "private_data": "discard"}, + }, + ) + ] + )[0] + + +def compact(): + return history.compact_events_batch( + timezone.now(), + monotonic() + 10, + batch_size=5000, + statement_timeout_ms=3000, + lock_timeout_ms=250, + ) + + +def test_history_catalog_keeps_new_tables_unlogged_and_sequence_logged(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT relname, relpersistence FROM pg_class WHERE oid IN (" + "'ws_realtime_event_history_summary'::regclass, " + "'ws_realtime_event_history_state'::regclass, " + "pg_get_serial_sequence('ws_realtime_events', 'id')::regclass)" + ) + persistence = dict(cursor.fetchall()) + assert persistence.pop("ws_realtime_event_history_summary") == "u" + assert persistence.pop("ws_realtime_event_history_state") == "u" + assert list(persistence.values()) == ["p"] + cursor.execute( + "SELECT seqcache FROM pg_sequence WHERE seqrelid = " + "pg_get_serial_sequence('ws_realtime_events', 'id')::regclass" + ) + assert cursor.fetchone() == (1,) + cursor.execute( + "SELECT indisvalid, pg_get_expr(indpred, indrelid) FROM pg_index " + "WHERE indexrelid = 'ws_realtime_created_id_idx'::regclass" + ) + assert cursor.fetchone() == (True, None) + cursor.execute( + "SELECT count(*) FROM pg_trigger " + "WHERE tgrelid = 'ws_realtime_events'::regclass " + "AND NOT tgisinternal AND (tgtype & 8) <> 0" + ) + assert cursor.fetchone() == (0,) + + +def test_history_migration_roundtrip_preserves_event_data_and_age_index(): + loader = MigrationLoader(None) + schema_migration = loader.get_migration("ws", "0003_realtime_event_history") + previous_state = loader.project_state([("ws", "0002_realtime_event_indexes")]) + + def filenodes(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT pg_relation_filenode('ws_realtime_events'), " + "pg_relation_filenode('ws_realtime_created_id_idx')" + ) + return cursor.fetchone() + + at_current_schema = True + try: + # Exercise all CreateModel/RunPython operations in both directions, + # without reapplying ws.0002's deliberate replay-buffer reset. + with connection.schema_editor() as editor: + schema_migration.unapply(previous_state.clone(), editor) + at_current_schema = False + old_id, recent_id = record(), record() + RealtimeEvent.objects.filter(pk=old_id).update( + created_at=timezone.now() - timedelta(days=2) + ) + originals = list(RealtimeEvent.objects.order_by("id").values()) + original_filenodes = filenodes() + + with connection.schema_editor() as editor: + schema_migration.apply(previous_state.clone(), editor) + at_current_schema = True + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert filenodes() == original_filenodes + assert not RealtimeEventHistorySummary.objects.exists() + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == state.compacted_event_id == recent_id + + with connection.schema_editor() as editor: + schema_migration.unapply(previous_state.clone(), editor) + at_current_schema = False + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert filenodes() == original_filenodes + with connection.cursor() as cursor: + cursor.execute( + "SELECT to_regclass('ws_realtime_event_history_summary'), " + "to_regclass('ws_realtime_event_history_state'), " + "to_regprocedure('ws_initialize_realtime_history()')" + ) + assert cursor.fetchone() == (None, None, None) + # Older writers still use the unchanged table and durable sequence. + assert record() > recent_id + finally: + if not at_current_schema: + with connection.schema_editor() as editor: + schema_migration.apply(previous_state.clone(), editor) + + +def test_history_initialization_is_idempotent_with_existing_summary_and_floor(): + expired_id = record() + assert compact() == 1 + retained_id = record() + original_summary = RealtimeEventHistorySummary.objects.values().get() + original_state = RealtimeEventHistoryState.objects.values().get() + migration = import_module("baserow.ws.migrations.0003_realtime_event_history") + + for _ in range(2): + with connection.schema_editor() as editor: + migration.forwards(None, editor) + assert RealtimeEventHistorySummary.objects.values().get() == original_summary + assert RealtimeEventHistoryState.objects.values().get() == original_state + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [retained_id] + assert original_state["floor"] == 0 + assert original_state["compacted_event_id"] == expired_id + + +def test_history_reset_uses_allocated_sequence_ids_as_conservative_floor(settings): + settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = 100 + committed_id = record() + assert compact() == 1 + with pytest.raises(RuntimeError, match="rollback"): + with transaction.atomic(): + allocated_id = record() + raise RuntimeError("rollback") + assert allocated_id > committed_id + + # Simulate loss of every UNLOGGED history relation while retaining the + # logged sequence, including IDs allocated by a rolled-back transaction. + with connection.cursor() as cursor: + cursor.execute( + "TRUNCATE ws_realtime_events, ws_realtime_event_history_summary, " + "ws_realtime_event_history_state" + ) + assert RealtimeEventHandler.get_latest_event_id() >= allocated_id + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == state.compacted_event_id >= allocated_id + assert not RealtimeEventHistorySummary.objects.exists() + result = RealtimeEventHandler.get_replay_events_result( + 42, ["table-1"], committed_id, None + ) + assert result.force_refresh + assert result.refresh_reason == "unknown_history" + assert record() > state.floor + + +def test_history_initialization_waits_for_inflight_inserts(): + RealtimeEventHistoryState.objects.all().delete() + with closing( + connection.Database.connect(**connection.get_connection_params()) + ) as writer: + with writer.cursor() as cursor: + cursor.execute( + "INSERT INTO ws_realtime_events (channel_group, payload, created_at) " + "VALUES ('other', '{}', now()) RETURNING id" + ) + inserted_id = cursor.fetchone()[0] + + # Initialization must wait for the invisible writer's RowExclusive lock + # before treating its allocated ID as covered by the loss floor. + with pytest.raises(OperationalError) as error: + with transaction.atomic(), connection.cursor() as cursor: + cursor.execute("SET LOCAL lock_timeout = '25ms'") + cursor.execute("SELECT ws_initialize_realtime_history()") + cause = error.value.__cause__ + assert ( + getattr(cause, "pgcode", None) or getattr(cause, "sqlstate", None) + ) == "55P03" + assert not RealtimeEventHistoryState.objects.exists() + writer.commit() + + with connection.cursor() as cursor: + cursor.execute("SELECT ws_initialize_realtime_history()") + assert RealtimeEventHistoryState.objects.get(pk=1).floor >= inserted_id + + +def test_missing_history_with_cached_sequence_ids_fails_closed(): + RealtimeEventHistoryState.objects.all().delete() + with connection.cursor() as cursor: + cursor.execute( + "DO $block$ BEGIN EXECUTE format('ALTER SEQUENCE %s CACHE 100', " + "pg_get_serial_sequence('ws_realtime_events', 'id')); END $block$" + ) + try: + with ( + connection.cursor() as cursor, + pytest.raises(DatabaseError, match="CACHE 1"), + ): + cursor.execute("SELECT ws_initialize_realtime_history()") + assert not RealtimeEventHistoryState.objects.exists() + finally: + with connection.cursor() as cursor: + cursor.execute( + "DO $block$ BEGIN EXECUTE format('ALTER SEQUENCE %s CACHE 1', " + "pg_get_serial_sequence('ws_realtime_events', 'id')); END $block$" + ) + cursor.execute("SELECT ws_initialize_realtime_history()") + + +@pytest.mark.parametrize("existing_summary", [False, True]) +def test_route_hash_collision_rolls_back_summary_and_keeps_originals( + monkeypatch, existing_summary +): + record("first-audience", socket="first") + if existing_summary: + assert compact() == 1 + collision_key = bytes(RealtimeEventHistorySummary.objects.get().route_key) + else: + collision_key = b"x" * 32 + record("second-audience", socket="second") + original_events = list(RealtimeEvent.objects.order_by("id").values()) + original_summaries = list(RealtimeEventHistorySummary.objects.values()) + original_state = RealtimeEventHistoryState.objects.values().get() + summarize = history._summarize_candidates + + def force_hash_collision(candidates): + # Inject a digest collision after real SQL routing extraction. The + # existing-summary case still executes the actual ON CONFLICT update. + return summarize( + [(event_id, collision_key, *route) for event_id, _, *route in candidates] + ) + + monkeypatch.setattr(history, "_summarize_candidates", force_hash_collision) + with pytest.raises(RuntimeError, match="route hash collision"): + compact() + assert list(RealtimeEvent.objects.order_by("id").values()) == original_events + assert list(RealtimeEventHistorySummary.objects.values()) == original_summaries + assert RealtimeEventHistoryState.objects.values().get() == original_state diff --git a/backend/tests/baserow/ws/test_ws_index_migration.py b/backend/tests/baserow/ws/test_ws_index_migration.py index 17c8ed1967..e8aebcfbbf 100644 --- a/backend/tests/baserow/ws/test_ws_index_migration.py +++ b/backend/tests/baserow/ws/test_ws_index_migration.py @@ -6,7 +6,7 @@ import pytest -from baserow.ws.models import RealtimeEvent +from baserow.ws.models import RealtimeEvent, RealtimeEventHistoryState from baserow.ws.realtime_events import RealtimeEventHandler @@ -85,20 +85,28 @@ def test_replay_reset_upgrade_and_rollback_keep_ids_and_old_writer_compatibility _apply("backwards") try: old_cursor = _legacy_insert() + lost_id = _legacy_insert() old_filenode = _table_storage()[0] _apply("forwards") assert RealtimeEvent.objects.count() == 0 assert _table_storage()[0] != old_filenode assert {"target_user_ids", "all_users"} <= _columns() + # Activate history after the reset, as ws.0003 does in deployment. The + # sequence records the missed event even though its payload is now gone. + RealtimeEventHistoryState.objects.all().delete() + RealtimeEventHandler._initialize_realtime_history() + assert RealtimeEventHistoryState.objects.get(pk=1).floor == lost_id new_id = _legacy_insert() - assert new_id > old_cursor + assert new_id > lost_id > old_cursor event = RealtimeEvent.objects.get(id=new_id) assert event.target_user_ids == [42] assert event.all_users is False - assert RealtimeEventHandler.get_replay_events_result( + result = RealtimeEventHandler.get_replay_events_result( 42, [], old_cursor, "socket" - ).force_refresh + ) + assert result.force_refresh is True + assert result.refresh_reason == "unknown_history" indexes = _indexes() assert migration.OLD_INDEX not in indexes assert migration.USERS_INDEX not in indexes diff --git a/backend/tests/baserow/ws/test_ws_realtime_cleanup.py b/backend/tests/baserow/ws/test_ws_realtime_cleanup.py index 0adca409c0..a5b00ee524 100644 --- a/backend/tests/baserow/ws/test_ws_realtime_cleanup.py +++ b/backend/tests/baserow/ws/test_ws_realtime_cleanup.py @@ -1,31 +1,42 @@ from contextlib import closing from datetime import timedelta -from unittest.mock import patch +from unittest.mock import Mock, patch from django.db import OperationalError, connection, transaction from django.utils import timezone import pytest -from baserow.ws import realtime_events, tasks +from baserow.ws import history, realtime_events, tasks from baserow.ws.models import RealtimeEvent from baserow.ws.realtime_events import RealtimeEventHandler from baserow.ws.tasks import cleanup_old_realtime_events @pytest.mark.parametrize("max_events", [0, 5]) +@pytest.mark.parametrize("retention_hours", [1, 24, 48, 240]) def test_cleanup_task_uses_independent_retention_even_when_recording_disabled( - settings, max_events + settings, max_events, retention_hours ): settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = max_events + settings.REALTIME_REPLAY_RETENTION_HOURS = retention_hours settings.SIMPLE_JWT = {"REFRESH_TOKEN_LIFETIME": timedelta(days=7)} with ( patch.object(RealtimeEventHandler, "cleanup_old_realtime_events") as cleanup, - patch("django.core.cache.cache.lock"), + patch("django.core.cache.cache.lock") as make_lock, patch.object(tasks, "monotonic", return_value=100), ): cleanup_old_realtime_events() - cleanup.assert_called_once_with(timedelta(hours=24), deadline=130) + cleanup.assert_called_once_with(timedelta(hours=retention_hours), deadline=340) + assert make_lock.call_args.kwargs["timeout"] == 330 + + +def test_cleanup_runs_every_ten_minutes(): + sender = Mock() + tasks.setup_periodic_ws_realtime_events_cleanup(sender) + sender.add_periodic_task.assert_called_once_with( + timedelta(minutes=10), cleanup_old_realtime_events.s() + ) def test_cleanup_task_skips_an_overlapping_run(): @@ -141,7 +152,9 @@ def fail_second_batch(*args): @pytest.mark.django_db(transaction=True) -def test_retention_boundary_preserves_fresh_replay_and_expires_old_cursor(settings): +def test_retention_boundary_preserves_replay_after_an_acknowledged_expired_event( + settings, +): settings.SIMPLE_JWT = {"REFRESH_TOKEN_LIFETIME": timedelta(days=7)} now = timezone.now() with patch.object(realtime_events.timezone, "now", return_value=now): @@ -155,12 +168,11 @@ def test_retention_boundary_preserves_fresh_replay_and_expires_old_cursor(settin assert list( RealtimeEvent.objects.order_by("id").values_list("id", flat=True) ) == [boundary, fresh, latest] - expired_result = RealtimeEventHandler.get_replay_events_result( - 1, ["table-1"], expired, None - ) - assert expired_result.force_refresh is True - assert expired_result.replay_events == [] - for cursor, expected in ((boundary, [fresh, latest]), (fresh, [latest])): + for cursor, expected in ( + (expired, [boundary, fresh, latest]), + (boundary, [fresh, latest]), + (fresh, [latest]), + ): result = RealtimeEventHandler.get_replay_events_result( 1, ["table-1"], cursor, None ) @@ -175,6 +187,7 @@ def test_cleanup_stops_at_its_work_budget(monkeypatch): create_events(timedelta(days=2), 5) now = 0 monkeypatch.setattr(realtime_events, "monotonic", lambda: now) + monkeypatch.setattr(history, "monotonic", lambda: now) original = RealtimeEventHandler._delete_realtime_events_batch def consume_budget(*args): @@ -211,7 +224,7 @@ def test_cleanup_timeouts_are_local_and_preserve_stricter_settings( observed = [] def check_timeouts(execute, sql, params, many, context): - if sql.startswith("WITH expired"): + if sql.startswith("WITH candidates"): with connection.cursor() as cursor: cursor.execute("SHOW statement_timeout") statement = cursor.fetchone()[0] @@ -242,6 +255,81 @@ def check_timeouts(execute, sql, params, many, context): connection.close() +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "statement_timeout,lock_timeout,expected_statements,expected_lock", + [ + ("0", "0", ["3s", "3s", "2s", "1s", "1s"], "250ms"), + ( + "1500ms", + "50ms", + ["1500ms", "1500ms", "1500ms", "1s", "1s"], + "50ms", + ), + ], +) +def test_cleanup_tightens_timeouts_as_the_batch_uses_its_remaining_budget( + monkeypatch, statement_timeout, lock_timeout, expected_statements, expected_lock +): + expired = create_events(timedelta(days=2), 2) + now = 0 + monkeypatch.setattr(history, "monotonic", lambda: now) + observed = [] + timeout_updates = 0 + statement_times = { + "SELECT ws_initialize_realtime_history()": 5, + "WITH candidates": 8, + "INSERT INTO ws_realtime_event_history_summary": 9, + "DELETE FROM ws_realtime_events": 9, + "UPDATE ws_realtime_event_history_state": 9, + } + + def observe_settings_and_advance_clock(execute, sql, params, many, context): + nonlocal now, timeout_updates + if "set_config('statement_timeout'" in sql: + timeout_updates += 1 + for prefix, finished_at in statement_times.items(): + if sql.startswith(prefix): + with connection.cursor() as cursor: + cursor.execute("SHOW statement_timeout") + statement = cursor.fetchone()[0] + cursor.execute("SHOW lock_timeout") + lock = cursor.fetchone()[0] + observed.append((prefix, statement, lock)) + result = execute(sql, params, many, context) + now = finished_at + return result + return execute(sql, params, many, context) + + with connection.cursor() as cursor: + cursor.execute( + "SELECT set_config('statement_timeout', %s, false)", [statement_timeout] + ) + cursor.execute("SELECT set_config('lock_timeout', %s, false)", [lock_timeout]) + try: + with connection.execute_wrapper(observe_settings_and_advance_clock): + assert ( + RealtimeEventHandler._delete_realtime_events_batch( + timezone.now() - timedelta(days=1), deadline=10 + ) + == 2 + ) + assert observed == [ + (prefix, statement, expected_lock) + for prefix, statement in zip(statement_times, expected_statements) + ] + # Reuse the cap until the remaining budget falls below three seconds. + assert timeout_updates == 3 + assert not RealtimeEvent.objects.filter(id__in=expired).exists() + with connection.cursor() as cursor: + cursor.execute("SHOW statement_timeout") + assert cursor.fetchone()[0] == statement_timeout + cursor.execute("SHOW lock_timeout") + assert cursor.fetchone()[0] == lock_timeout + finally: + connection.close() + + @pytest.mark.django_db(transaction=True) def test_cleanup_statement_timeout_stops_slow_database_work(monkeypatch): monkeypatch.setattr( @@ -250,7 +338,7 @@ def test_cleanup_statement_timeout_stops_slow_database_work(monkeypatch): expired = create_events(timedelta(days=2), 1) def slow_delete(execute, sql, params, many, context): - if sql.startswith("WITH expired"): + if sql.startswith("WITH candidates"): return execute("SELECT pg_sleep(1)", [], many, context) return execute(sql, params, many, context) diff --git a/backend/tests/baserow/ws/test_ws_realtime_events.py b/backend/tests/baserow/ws/test_ws_realtime_events.py index 2fbadd0d6a..b6bab75d43 100644 --- a/backend/tests/baserow/ws/test_ws_realtime_events.py +++ b/backend/tests/baserow/ws/test_ws_realtime_events.py @@ -5,6 +5,7 @@ from django.conf import settings from django.db import connection from django.test import override_settings +from django.utils import timezone import pytest from asgiref.sync import sync_to_async @@ -12,7 +13,7 @@ from loguru import logger from baserow.config.asgi import application -from baserow.ws.models import RealtimeEvent +from baserow.ws.models import RealtimeEvent, RealtimeEventHistoryState from baserow.ws.realtime_events import ( FIRST_CONNECT_CURSOR, NO_REPLAY_AVAILABLE, @@ -594,6 +595,7 @@ def test_replay_raises_when_over_threshold(): web_socket_id=None, ) assert result.force_refresh is True + assert result.refresh_reason == "event_limit" assert result.replay_events == [] @@ -1192,7 +1194,6 @@ def test_replay_window_ordered_scan_does_not_visit_events_before_cursor(): nodes.extend(node.get("Plans", [])) assert [event.id for event in window] == [ - baseline, events[900].id, events[905].id, events[910].id, @@ -1237,7 +1238,7 @@ def test_stale_users_replay_does_not_filter_unrelated_individual_payloads(): filtered_rows += node.get("Rows Removed by Index Recheck", 0) nodes.extend(node.get("Plans", [])) - assert [event.id for event in window] == [baseline, targeted, everyone] + assert [event.id for event in window] == [targeted, everyone] # The shared event type must not make replay inspect every other user's # payload. Check work performed by PostgreSQL, rather than machine timing. assert filtered_rows < 50 @@ -1297,30 +1298,34 @@ def test_replay_events_result_future_last_seen_uses_one_query( ) assert result.force_refresh is True + assert result.refresh_reason == "cursor_ahead" assert result.latest_event_id == NO_REPLAY_AVAILABLE assert result.replay_events == [] -@pytest.mark.django_db +@pytest.mark.django_db(transaction=True) @pytest.mark.websockets -def test_replay_events_result_missing_last_seen_uses_one_query( +def test_replay_events_result_compacted_last_seen_uses_one_query( django_assert_num_queries, ): - missing_id = _record_user_broadcast(1, {"type": "missing"}) - RealtimeEvent.objects.filter(id=missing_id).delete() - _record_user_broadcast(1, {"type": "latest"}) + compacted_id = _record_user_broadcast(1, {"type": "acknowledged"}) + RealtimeEvent.objects.filter(id=compacted_id).update( + created_at=timezone.now() - timedelta(days=2) + ) + latest_id = _record_user_broadcast(1, {"type": "latest"}) + assert RealtimeEventHandler.cleanup_old_realtime_events(timedelta(days=1)) == 1 with django_assert_num_queries(1): result = _replay_events_result( user_id=1, page_group_names=[], - last_seen_id=missing_id, + last_seen_id=compacted_id, web_socket_id=None, ) - assert result.force_refresh is True - assert result.latest_event_id == NO_REPLAY_AVAILABLE - assert result.replay_events == [] + assert result.force_refresh is False + assert result.latest_event_id == latest_id + assert [event.id for event in result.replay_events] == [latest_id] @pytest.mark.django_db @@ -1758,13 +1763,12 @@ async def test_replay_events_replays_individual_payloads_event(data_fixture): @pytest.mark.asyncio @pytest.mark.django_db(transaction=True) @pytest.mark.websockets -async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): +async def test_replay_events_cant_replay_below_the_known_history_floor(data_fixture): user, token = await sync_to_async(data_fixture.create_user_and_token)() await sync_to_async(data_fixture.create_workspace)(user=user) - # Last-seen event cleaned by retention while a newer one survives: replay can't - # anchor, so it must force a refresh. Capture the real id — the sequence isn't - # reset between transactional tests. + # A cursor below the known-history floor cannot prove recovery, even when + # newer full events survive. Use real IDs because sequences are not reset. stale_last_seen_id = await sync_to_async(_record_event)( "users", { @@ -1775,8 +1779,7 @@ async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): "ignore_web_socket_id": "ws-other", }, ) - await sync_to_async(RealtimeEvent.objects.filter(id=stale_last_seen_id).delete)() - await sync_to_async(_record_event)( + latest_id = await sync_to_async(_record_event)( "users", { "type": "broadcast_to_users", @@ -1786,6 +1789,9 @@ async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): "ignore_web_socket_id": "ws-other", }, ) + await sync_to_async(RealtimeEventHistoryState.objects.filter(pk=1).update)( + floor=latest_id + ) communicator = WebsocketCommunicator( application, diff --git a/backend/tests/baserow/ws/test_ws_realtime_history.py b/backend/tests/baserow/ws/test_ws_realtime_history.py new file mode 100644 index 0000000000..6e949c56b2 --- /dev/null +++ b/backend/tests/baserow/ws/test_ws_realtime_history.py @@ -0,0 +1,474 @@ +import json +import re +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from threading import Barrier +from time import monotonic + +from django.db import connection, connections +from django.utils import timezone + +import pytest + +from baserow.ws import realtime_events +from baserow.ws.models import ( + RealtimeEvent, + RealtimeEventHistoryState, + RealtimeEventHistorySummary, +) +from baserow.ws.realtime_events import FIRST_CONNECT_CURSOR, RealtimeEventHandler + +pytestmark = [pytest.mark.django_db(transaction=True), pytest.mark.websockets] + + +@pytest.fixture(autouse=True) +def replay_settings(settings): + settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = 100 + settings.REALTIME_REPLAY_RETENTION_HOURS = 24 + + +def group_event(*, event_type="rows_updated", socket=None, excluded=None, **data): + event = { + "type": "broadcast_to_group", + "payload": {"type": event_type, **data}, + "ignore_web_socket_id": socket, + } + if excluded is not None: + event["exclude_user_ids"] = excluded + return event + + +def users_event(*, recipients=None, everyone=False, socket=None): + return { + "type": "broadcast_to_users", + "payload": {"type": "workspace_updated", "private_data": "omit-me"}, + "user_ids": [42] if recipients is None else recipients, + "send_to_all_users": everyone, + "ignore_web_socket_id": socket, + } + + +def individual_event(*, recipients=None, socket=None): + return { + "type": "broadcast_to_users_individual_payloads", + "payload_map": { + str(user_id): {"type": event_type, "private_data": "omit-me"} + for user_id, event_type in ( + {42: "workspace_updated"} if recipients is None else recipients + ).items() + }, + "ignore_web_socket_id": socket, + } + + +def record(group="table-1", payload=None, *, age=timedelta()): + event_id = RealtimeEventHandler.record_events( + [(group, group_event() if payload is None else payload)] + )[0] + RealtimeEvent.objects.filter(pk=event_id).update(created_at=timezone.now() - age) + return event_id + + +def replay(cursor, *, user=42, groups=None, socket="own"): + return RealtimeEventHandler.get_replay_events_result( + user, ["table-1"] if groups is None else groups, cursor, socket + ) + + +def cleanup(): + return RealtimeEventHandler.cleanup_old_realtime_events(timedelta(days=1)) + + +@pytest.mark.parametrize("missed_old", [False, True]) +@pytest.mark.parametrize("recent", [False, True]) +def test_missing_baseline_preserves_no_change_replay_and_refresh(missed_old, recent): + baseline = record("other-page", age=timedelta(days=3)) + record("other-page", age=timedelta(days=2)) + if missed_old: + record(age=timedelta(days=2)) + fresh_id = record(age=timedelta(hours=1)) if recent else None + + # An expired relevant gap requires refresh even while its payload still + # exists, and a newer full event must not hide that gap after compaction. + assert replay(baseline).force_refresh is missed_old + assert cleanup() == 2 + missed_old + assert not RealtimeEvent.objects.filter(pk=baseline).exists() + + result = replay(baseline) + assert result.force_refresh is missed_old + assert result.refresh_reason == ("expired_payload" if missed_old else None) + assert [event.id for event in result.replay_events] == ( + [fresh_id] if recent and not missed_old else [] + ) + if not missed_old: + assert result.latest_event_id == (fresh_id if recent else baseline) + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == 0 + assert state.compacted_event_id > baseline + + +@pytest.mark.parametrize( + "group,payload,socket,relevant", + [ + ("table-1", group_event(), "own", True), + ("other-page", group_event(), "own", False), + ("table-1", group_event(socket="own"), "own", False), + ("table-1", group_event(socket="other"), "own", True), + ("table-1", group_event(socket="own"), None, True), + ("table-1", group_event(excluded=[42]), "own", False), + ("table-1", group_event(excluded=[7]), "own", True), + ("users", users_event(), "own", True), + ("users", users_event(recipients=[7]), "own", False), + ("users", users_event(recipients=[], everyone=True), "own", True), + ("users", users_event(socket="own"), "own", False), + ("users", users_event(socket="own"), None, True), + ("users", individual_event(), "own", True), + ( + "users", + individual_event(recipients={7: "workspace_updated"}), + "own", + False, + ), + ("users", individual_event(socket="own"), "own", False), + ("users", individual_event(socket="own"), None, True), + ], + ids=[ + "page", + "other-page", + "own-page", + "other-socket-page", + "page-without-client-socket", + "excluded-page", + "other-excluded-page", + "target-user", + "other-user", + "everyone", + "own-user", + "user-without-client-socket", + "individual", + "other-individual", + "own-individual", + "individual-without-client-socket", + ], +) +def test_summary_and_expired_payload_have_identical_relevance( + group, payload, socket, relevant +): + baseline = record("baseline", age=timedelta(days=3)) + expired = record(group, payload, age=timedelta(days=2)) + + assert replay(baseline, socket=socket).force_refresh is relevant + assert cleanup() == 2 + assert not RealtimeEvent.objects.filter(pk=expired).exists() + result = replay(baseline, socket=socket) + assert result.force_refresh is relevant + assert result.replay_events == [] + + +@pytest.mark.parametrize("kind", ["group", "users", "individual"]) +def test_missing_and_null_socket_share_one_summary_without_losing_changes(kind): + group = "table-1" if kind == "group" else "users" + make_payload = { + "group": group_event, + "users": users_event, + "individual": individual_event, + }[kind] + payload = make_payload() + del payload["ignore_web_socket_id"] + missing = record(group, payload, age=timedelta(days=2)) + null = record(group, make_payload(), age=timedelta(days=2)) + own = record(group, make_payload(socket="own"), age=timedelta(days=2)) + + assert cleanup() == 3 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (own, "own") + assert (summary.previous_event_id, summary.previous_socket_id) == (null, None) + assert replay(missing).force_refresh is True + assert replay(null).force_refresh is False + assert replay(null, socket=None).force_refresh is True + + +@pytest.mark.parametrize( + "ignored_socket,client_socket", + [(42, "42"), (True, "true"), (["own"], "own"), ({"id": "own"}, "own")], + ids=["number", "boolean", "array", "object"], +) +def test_non_string_socket_values_do_not_exclude_a_client( + ignored_socket, client_socket +): + baseline = record("baseline", age=timedelta(days=3)) + record(payload=group_event(socket=ignored_socket), age=timedelta(days=2)) + assert replay(baseline, socket=client_socket).force_refresh is True + + assert cleanup() == 2 + summary = RealtimeEventHistorySummary.objects.get(channel_group="table-1") + assert summary.latest_socket_id is None + assert replay(baseline, socket=client_socket).force_refresh is True + + +def test_many_originating_sockets_keep_only_two_distinct_socket_watermarks(): + ids = [ + record(payload=group_event(socket=f"socket-{index}"), age=timedelta(days=2)) + for index in range(20) + ] + newest = record(payload=group_event(socket="socket-19"), age=timedelta(days=2)) + + assert cleanup() == 21 + assert not RealtimeEvent.objects.exists() + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (newest, "socket-19") + assert (summary.previous_event_id, summary.previous_socket_id) == ( + ids[-2], + "socket-18", + ) + assert replay(ids[-3], socket="socket-19").force_refresh is True + assert replay(ids[-2], socket="socket-19").force_refresh is False + assert replay(ids[-2], socket="socket-18").force_refresh is True + assert replay(ids[-2], socket=None).force_refresh is True + + +def test_summary_merges_batches_in_id_order_despite_different_timestamp_order( + monkeypatch, +): + monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 2) + ids = [ + record(payload=group_event(socket=socket), age=timedelta(days=days)) + for socket, days in [("a", 2), ("b", 5), ("a", 4), ("c", 3)] + ] + + assert cleanup() == 4 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (ids[-1], "c") + assert (summary.previous_event_id, summary.previous_socket_id) == (ids[-2], "a") + assert RealtimeEventHistoryState.objects.get(pk=1).compacted_event_id == ids[-1] + assert replay(ids[-2], socket="c").force_refresh is False + assert replay(ids[1], socket="c").force_refresh is True + + +def test_later_cleanup_of_lower_ids_does_not_regress_summary_or_high_water(): + lower = record(payload=group_event(socket="a"), age=timedelta(hours=1)) + other = record(payload=group_event(socket="b"), age=timedelta(days=2)) + latest = record(payload=group_event(socket="a"), age=timedelta(days=2)) + assert cleanup() == 2 + original = RealtimeEventHistorySummary.objects.values().get() + + RealtimeEvent.objects.filter(pk=lower).update( + created_at=timezone.now() - timedelta(days=3) + ) + assert cleanup() == 1 + assert RealtimeEventHistorySummary.objects.values().get() == original + assert RealtimeEventHandler.get_latest_event_id() == latest + + repeated = record(payload=group_event(socket="a"), age=timedelta(days=2)) + assert cleanup() == 1 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (repeated, "a") + assert (summary.previous_event_id, summary.previous_socket_id) == (other, "b") + + newest = record(payload=group_event(socket="c"), age=timedelta(days=2)) + assert cleanup() == 1 + summary.refresh_from_db() + assert (summary.latest_event_id, summary.latest_socket_id) == (newest, "c") + assert (summary.previous_event_id, summary.previous_socket_id) == (repeated, "a") + + +@pytest.mark.parametrize("individual", [False, True], ids=["group", "individual"]) +def test_summary_retains_recovery_types_without_business_payload(individual): + for event_type in ("rows_updated", "rows_deleted"): + payload = ( + individual_event(recipients={42: event_type, 7: "workspace_updated"}) + if individual + else group_event(event_type=event_type, secret="omit-me", rows=[1, 2, 3]) + ) + payload["request_debug_data"] = "omit-me" + record("users" if individual else "table-1", payload, age=timedelta(days=2)) + + assert cleanup() == 2 + summaries = list(RealtimeEventHistorySummary.objects.order_by("latest_event_id")) + assert len(summaries) == 2 + for summary, event_type in zip(summaries, ("rows_updated", "rows_deleted")): + assert "ignore_web_socket_id" not in summary.payload + assert "request_debug_data" not in summary.payload + assert "omit-me" not in json.dumps(summary.payload) + if individual: + assert summary.payload["payload_map"] == { + "7": {"type": "workspace_updated"}, + "42": {"type": event_type}, + } + assert summary.target_user_ids == [7, 42] + assert summary.all_users is False + else: + assert summary.payload["payload"] == {"type": event_type} + + +@pytest.mark.parametrize("kind", ["recipients", "exclusions"]) +def test_reordered_and_duplicate_audience_ids_share_one_summary(kind): + for audience in ([42, 7, 42], [7, 42]): + payload = ( + users_event(recipients=audience) + if kind == "recipients" + else group_event(excluded=audience) + ) + latest = record( + "users" if kind == "recipients" else "table-1", + payload, + age=timedelta(days=2), + ) + + assert cleanup() == 2 + summary = RealtimeEventHistorySummary.objects.get() + key = "user_ids" if kind == "recipients" else "exclude_user_ids" + assert summary.payload[key] == [7, 42] + assert summary.latest_event_id == latest + + +def test_retention_increase_cannot_make_summarized_payload_replayable(settings): + baseline = record("baseline", age=timedelta(days=3)) + expired = record(age=timedelta(days=2)) + assert cleanup() == 2 + + settings.REALTIME_REPLAY_RETENTION_HOURS = 240 + result = replay(baseline) + assert result.force_refresh is True + assert result.replay_events == [] + recent = record(age=timedelta(hours=1)) + result = replay(expired) + assert result.force_refresh is False + assert [event.id for event in result.replay_events] == [recent] + + +def test_empty_full_buffer_keeps_summarized_high_water_without_raising_loss_floor(): + record(age=timedelta(days=3)) + latest = record(age=timedelta(days=2)) + + assert cleanup() == 2 + assert not RealtimeEvent.objects.exists() + assert RealtimeEventHandler.get_latest_event_id() == latest + state = RealtimeEventHistoryState.objects.get(pk=1) + assert (state.floor, state.compacted_event_id) == (0, latest) + baseline = replay(FIRST_CONNECT_CURSOR) + assert baseline.force_refresh is False + assert baseline.latest_event_id == latest + assert baseline.replay_events == [] + + fresh = record() + assert fresh > latest + result = replay(latest) + assert result.force_refresh is False + assert [event.id for event in result.replay_events] == [fresh] + + +def is_event_delete(sql): + return re.search(r'\bDELETE\s+FROM\s+"?ws_realtime_events\b', sql, re.IGNORECASE) + + +def test_failure_after_deletion_rolls_back_payloads_summary_and_high_water(): + record(payload=group_event(value="first"), age=timedelta(days=2)) + record(payload=group_event(value="second"), age=timedelta(days=2)) + record(age=timedelta(hours=1)) + originals = list(RealtimeEvent.objects.order_by("id").values()) + state = RealtimeEventHistoryState.objects.values().get() + + def fail_after_delete(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if is_event_delete(sql): + raise RuntimeError("failed after deleting expired payloads") + return result + + with ( + connection.execute_wrapper(fail_after_delete), + pytest.raises(RuntimeError, match="failed after deleting expired payloads"), + ): + cleanup() + + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert not RealtimeEventHistorySummary.objects.exists() + assert RealtimeEventHistoryState.objects.values().get() == state + + +def test_replay_on_another_connection_remains_correct_during_compaction_delete(): + baseline = record("baseline", age=timedelta(days=3)) + record(age=timedelta(days=2)) + fresh = record(age=timedelta(hours=1)) + reads = [] + + def read_history(): + try: + result = replay(baseline) + return RealtimeEventHandler.get_latest_event_id(), result + finally: + connections["default"].close() + + with ThreadPoolExecutor(max_workers=1) as executor: + + def read_while_delete_is_uncommitted(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if is_event_delete(sql): + assert connection.in_atomic_block + # The reader has its own connection and must finish before the + # deleting transaction commits, with the expired gap still visible. + reads.append(executor.submit(read_history).result(timeout=5)) + return result + + with connection.execute_wrapper(read_while_delete_is_uncommitted): + assert cleanup() == 2 + + assert len(reads) == 1 + latest, result = reads[0] + assert latest == fresh + assert result.force_refresh is True + assert result.replay_events == [] + assert replay(baseline).force_refresh is True + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [fresh] + + +@pytest.mark.parametrize( + "latest_socket", ["b", None], ids=["two-sockets", "null-socket"] +) +def test_concurrent_batches_merge_the_same_route_without_losing_either_socket( + monkeypatch, latest_socket +): + monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 1) + baseline = record("baseline") + first = record(payload=group_event(socket="a"), age=timedelta(days=2)) + latest = record(payload=group_event(socket=latest_socket), age=timedelta(days=2)) + candidates_locked = Barrier(2) + + def compact_one(): + db = connections["default"] + + def synchronize_candidates(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if "FOR UPDATE SKIP LOCKED" in sql: + # Both transactions lock separate full rows before either inserts + # the shared route. The upsert must merge their evidence safely. + candidates_locked.wait(timeout=5) + return result + + try: + with db.execute_wrapper(synchronize_candidates): + return RealtimeEventHandler._delete_realtime_events_batch( + timezone.now() - timedelta(days=1), monotonic() + 10 + ) + finally: + db.close() + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(compact_one) for _ in range(2)] + try: + assert [future.result(timeout=10) for future in futures] == [1, 1] + finally: + candidates_locked.abort() + + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == ( + latest, + latest_socket, + ) + assert (summary.previous_event_id, summary.previous_socket_id) == (first, "a") + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [baseline] + state = RealtimeEventHistoryState.objects.get(pk=1) + assert (state.floor, state.compacted_event_id) == (0, latest) + assert replay(baseline, socket="b").force_refresh is True + assert replay(first, socket="b").force_refresh is (latest_socket is None) + assert replay(first, socket="a").force_refresh is True diff --git a/backend/tests/baserow/ws/test_ws_replay_executor.py b/backend/tests/baserow/ws/test_ws_replay_executor.py index c5cfbc4915..51edd812ab 100644 --- a/backend/tests/baserow/ws/test_ws_replay_executor.py +++ b/backend/tests/baserow/ws/test_ws_replay_executor.py @@ -324,7 +324,9 @@ async def test_replay_executor_releases_capacity_if_cancelled_before_start( async def test_missing_replay_cursor_does_not_need_a_database_or_pool(): with patch.object(replay, "_get_executor") as executor: result = await replay.get_replay_events_result(1, [], NO_REPLAY_AVAILABLE, None) - assert result == ReplayEventsResult(True, NO_REPLAY_AVAILABLE, []) + assert result == ReplayEventsResult( + True, NO_REPLAY_AVAILABLE, [], refresh_reason="missing_cursor" + ) executor.assert_not_called() diff --git a/backend/tests/baserow/ws/test_ws_storage_telemetry.py b/backend/tests/baserow/ws/test_ws_storage_telemetry.py index 1ab8c8946b..2d12b6f566 100644 --- a/backend/tests/baserow/ws/test_ws_storage_telemetry.py +++ b/backend/tests/baserow/ws/test_ws_storage_telemetry.py @@ -8,7 +8,7 @@ import pytest from redis.exceptions import ConnectionError as RedisConnectionError -from baserow.ws import realtime_events, telemetry +from baserow.ws import history, realtime_events, telemetry from baserow.ws.models import RealtimeEvent from baserow.ws.realtime_events import RealtimeEventHandler from baserow.ws.tasks import cleanup_old_realtime_events @@ -248,6 +248,7 @@ def test_cleanup_budget_reports_progress_without_claiming_completion( monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 2) now = [0.0] monkeypatch.setattr(realtime_events, "monotonic", lambda: now[0]) + monkeypatch.setattr(history, "monotonic", lambda: now[0]) delete_batch = RealtimeEventHandler._delete_realtime_events_batch def slow_batch(cutoff, deadline): diff --git a/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json b/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json new file mode 100644 index 0000000000..2190af9314 --- /dev/null +++ b/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fixed a crash when creating a snapshot, duplicating or exporting a database containing an active but invalid field rule.", + "issue_origin": "github", + "issue_number": 6095, + "domain": "database", + "bullet_points": [], + "created_at": "2026-09-15" +} diff --git a/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json b/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json new file mode 100644 index 0000000000..488c0a8e6b --- /dev/null +++ b/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Avoids unnecessary refresh prompts when reconnecting after old real-time history expired", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-09-14" +} diff --git a/changelog/entries/unreleased/feature/adds_a_columns_layout_option_for_grouped_grid_views.json b/changelog/entries/unreleased/feature/adds_a_columns_layout_option_for_grouped_grid_views.json new file mode 100644 index 0000000000..74a000563b --- /dev/null +++ b/changelog/entries/unreleased/feature/adds_a_columns_layout_option_for_grouped_grid_views.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Adds a Columns layout option for grouped grid views", + "issue_origin": "github", + "issue_number": null, + "domain": "database", + "bullet_points": [], + "created_at": "2026-09-04" +} diff --git a/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json b/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json new file mode 100644 index 0000000000..ade253f91a --- /dev/null +++ b/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Administrators are now warned which AI fields and AI Agent actions use an AI provider model before it is disabled or deleted.", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-09-11" +} diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml index 253bad35cd..1b5f41091b 100644 --- a/docker-compose.no-caddy.yml +++ b/docker-compose.no-caddy.yml @@ -92,6 +92,7 @@ x-backend-variables: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_MAX_FIELD_TEXT_LENGTH: BASEROW_REALTIME_REPLAY_MAX_EVENTS: + BASEROW_REALTIME_REPLAY_RETENTION_HOURS: BASEROW_AUTOMATION_HISTORY_PAGE_SIZE_LIMIT: BASEROW_AUTOMATION_WORKFLOW_RATE_LIMIT_MAX_RUNS: diff --git a/docker-compose.yml b/docker-compose.yml index 54cf9bdc7d..aa28525d9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ x-backend-variables: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_MAX_FIELD_TEXT_LENGTH: BASEROW_REALTIME_REPLAY_MAX_EVENTS: + BASEROW_REALTIME_REPLAY_RETENTION_HOURS: BASEROW_AUTOMATION_HISTORY_PAGE_SIZE_LIMIT: BASEROW_AUTOMATION_WORKFLOW_RATE_LIMIT_MAX_RUNS: diff --git a/docs/installation/ai-assistant.md b/docs/installation/ai-assistant.md index f0ff4c83f2..f313be200a 100644 --- a/docs/installation/ai-assistant.md +++ b/docs/installation/ai-assistant.md @@ -29,6 +29,20 @@ a model; it does not force AI Fields or AI Agent actions to use Kuma's model. Us **Test model** to check every selected feature. AI Fields and AI Agent actions check for a text response, while Kuma also checks tool calling. +Changing a model that is already in use is confirmed, not applied silently. Disabling +or deleting a model, unchecking one of its features, or renaming its identifier first +reports how many AI Fields and AI Agent actions use it, and applies the change once +confirmed: those consumers store the provider type and the model identifier rather +than a reference to the model, so they keep the old selection and stop working until +they are repointed. Counts are per provider type and identifier, so an instance model +and a workspace model sharing an identifier report the same consumers. The Kuma +selection is a real reference instead, so disabling or deleting the model, or +unchecking Kuma, is refused while it is selected as the Kuma model: repoint that +selection first, in the **AI features** section of the scope that holds it. Renaming +is allowed and the Kuma selection follows the model to its new identifier. A consumer left on a model that is disabled or +gone keeps showing its saved provider and model, marked unavailable, so it can be +found and repointed. + For an existing installation, see the [AI provider upgrade and import instructions](../development/feature-flags.md#preparing-the-ai-providers-feature). Schema migrations run during the normal upgrade. Provider imports and republishing diff --git a/docs/installation/configuration.md b/docs/installation/configuration.md index 9bfdaa3be8..be97cc2381 100644 --- a/docs/installation/configuration.md +++ b/docs/installation/configuration.md @@ -235,7 +235,8 @@ Baserow can throttle the number of concurrent requests a single user (or, option | BASEROW\_ENTERPRISE\_MAX\_PERIODIC\_DATA\_SYNC\_CONSECUTIVE\_ERRORS | The maximum number of consecutive periodic data sync error before it's disabled. | 4 | | BASEROW\_DEADLOCK\_INITIAL\_BACKOFF | The initial backoff time for database deadlock retries. | 2 | | BASEROW\_DEADLOCK\_MAX\_RETRIES | The maximum number of database deadlock retries. | 1 | -| BASEROW\_REALTIME\_REPLAY\_MAX\_EVENTS | Maximum number of missed events replayed to a reconnecting WebSocket client. When the count exceeds this limit, the client is asked to refresh instead of receiving individual events. Set to `0` to disable event recording and replay. Events are retained for 24 hours independently of JWT lifetime, and retention cleanup continues when recording is disabled. | 200 | +| BASEROW\_REALTIME\_REPLAY\_MAX\_EVENTS | Maximum number of missed events replayed to a reconnecting WebSocket client. When the count exceeds this limit, the client is asked to refresh instead of receiving individual events. Set to `0` to disable event recording and replay. Full events are replayable for the configured retention window, independently of JWT lifetime. Expired events become routing summaries without business payloads; relevant missed expired history requires a refresh. Cleanup continues when recording is disabled. | 200 | +| BASEROW\_REALTIME\_REPLAY\_RETENTION\_HOURS | Maximum age in hours of full realtime events available to replay after a WebSocket reconnect. Must be a positive integer. Increasing this window retains more full payloads and requires more database storage, but cannot restore events already compacted. Independent of JWT lifetime. | 24 | | BASEROW\_DEPENDANT\_ROWS\_REALTIME\_UPDATE\_LIMIT | Per table, the maximum number of rows changed by a dependency cascade (formulas, lookups, link row display values in other tables) that are broadcast as exact realtime row updates. Above the limit, subscribers of the affected table receive a whole-table refresh event instead. Set to `0` to disable realtime events for dependant rows entirely. Higher values increase the background serialization work and websocket traffic per edit. | 200 | | BASEROW\_PERIODIC\_FIELD\_UPDATE\_BATCH\_COUNT | The number of parallel tasks the periodic field update (e.g. `now()` formulas) is split into each cycle. Raise it to spread the work across more workers when a single task can't refresh every workspace before the next cycle starts. If cycles are still being skipped, also give each cycle more time with `BASEROW_PERIODIC_FIELD_UPDATE_CRONTAB`. | 1 | diff --git a/docs/installation/monitoring.md b/docs/installation/monitoring.md index 286e63bb51..9ee652e7d7 100644 --- a/docs/installation/monitoring.md +++ b/docs/installation/monitoring.md @@ -160,7 +160,7 @@ for the execution and recovery model. | `baserow.websocket_sync_queue_duration` / `baserow.websocket_sync_execution_duration` | Milliseconds before the executor starts work versus milliseconds on its thread, including database connection cleanup. | | `baserow.websocket_sync_pending` / `baserow.websocket_sync_executing` | Awaiting callers, including queued and running calls, versus work actually executing. Executing work can outlive a cancelled caller. | | `baserow.websocket_event_loop_lag` | Scheduling delay in milliseconds, sampled once per second while WebSocket applications are active. | -| `baserow.websocket_replay_requests` | Decisions by `baseline`, `replayed`, `refresh`, `overloaded`, `deadline_exceeded`, `query_timeout`, `database_error`, `cancelled`, or `error`. | +| `baserow.websocket_replay_requests` | Decisions by `baseline`, `replayed`, `refresh`, `overloaded`, `deadline_exceeded`, `query_timeout`, `database_error`, `cancelled`, or `error`. Refresh `reason` distinguishes `missing_cursor`, `unknown_history`, `cursor_ahead`, `expired_payload`, and `event_limit`; other outcomes use `none`. | | `baserow.websocket_replay_duration` / `baserow.websocket_replay_events` | Caller wait in milliseconds and number of events returned by a completed decision. | | `baserow.websocket_replay_inflight` / `baserow.websocket_replay_capacity` | Occupied slots and initialized replay pool capacity, including work still running after cancellation or a deadline. | | `baserow.websocket_replay_queued` / `baserow.websocket_replay_queue_capacity` / `baserow.websocket_replay_queue_duration` | Waiting requests, admission queue capacity, and wait in milliseconds, including cancelled/expired waits. Admission precedes the separate synchronous executor queue measurement. | @@ -168,7 +168,7 @@ for the execution and recovery model. | `baserow.realtime_recording_events` | Attempted recording envelopes by `destination=users/page` and handler `outcome=success/error`. A successful handler does not guarantee an enclosing transaction committed. | | `baserow.realtime_recording_batch_size` / `baserow.realtime_recording_duration` | Envelopes per attempted batch and handler duration in milliseconds, including adaptation and database work. | | `baserow.realtime_cleanup_deleted` / `baserow.realtime_cleanup_batch_size` | Deleted events and rows per successfully committed cleanup batch. | -| `baserow.realtime_cleanup_batch_duration` | Batch duration in milliseconds, including commit. Failed batches have `outcome=error` and contribute no deleted rows. | +| `baserow.realtime_cleanup_batch_duration` | Batch duration in milliseconds, including expired-history summary updates and commit. Failed batches have `outcome=error` and contribute no deleted rows. | | `baserow.realtime_cleanup_run_deleted` / `baserow.realtime_cleanup_run_duration` | Committed progress and run duration in milliseconds, by outcome. Earlier commits still count if a later batch fails. | | `baserow.realtime_cleanup_skipped` | Scheduled attempts skipped for `reason=overlap` (another task owns the lease) or `reason=lock_error` (lease acquisition failed). | @@ -217,14 +217,25 @@ has exhausted memory. Compare recording rate with committed cleanup deletions over time. A cleanup run ending with `budget` retained its earlier commits but exhausted its time allowance; `success` can still leave locked rows for the next run. Repeated overlap or lock -errors explain runs that never reached the database. Track oldest event age, +errors explain runs that never reached the database. Track oldest full-event age, +event-table size, history-summary count and size, `pg_stat_user_tables` live/dead tuple estimates and vacuum/analyze timestamps, and database I/O alongside these metrics. Deletion and vacuum make space reusable; they do not normally reduce allocated table files. Replay refresh fallbacks also create HTTP reads, so include that traffic when assessing capacity. +Cleanup deletes expired business payloads while preserving routing summaries for +exact audiences. Summaries have no fixed size bound: distinct audiences can +accumulate, although new ignored sockets alone do not create additional rows. +Measure summary growth alongside recent event volume and cleanup progress. + For users-channel replay, compare rows and heap blocks visited with events actually returned. Recipient selection should use `target_user_ids` and `all_users`, with `ws_realtime_targets_idx` and `ws_realtime_all_users_idx` available to the planner. There is no full-payload GIN index. Include recipient-trigger work in recording measurements; smaller indexes do not by themselves guarantee faster inserts. + +Refresh reasons separate storage coverage from event volume: `expired_payload` +means relevant missed history was compacted or falls outside the replay window; +`unknown_history` means the cursor predates the known history boundary after +activation or reset; `event_limit` means too many replayable changes remain. diff --git a/docs/technical/collapsible-grid-group-by.md b/docs/technical/collapsible-grid-group-by.md index 28a63462fe..00a9be099d 100644 --- a/docs/technical/collapsible-grid-group-by.md +++ b/docs/technical/collapsible-grid-group-by.md @@ -63,7 +63,7 @@ keeps layout size proportional to loaded group metadata, not total row count. ### Compact Collapse State -Collapse state is modeled as a mode plus exceptions: +The Sections layout applies collapse state modeled as a mode plus exceptions: - expand mode with no exceptions means every group is expanded; - collapse mode with no exceptions means every group is collapsed; @@ -74,7 +74,9 @@ regardless of how many groups exist. Collapse-all only ever shows the top-level headers, so it loads its single visible depth with depth-based group metadata. Expand-all and mixed states load per parent, descending each visible parent's subtree to its leaves so one request returns everything the viewport needs -instead of one request per depth. +instead of one request per depth. The Columns layout retains this state but does +not apply or mutate it: Columns always renders and fetches the expanded tree, and +switching back to Sections restores the retained collapse state. ### Viewport-Driven Fetching @@ -230,8 +232,9 @@ Known costs to keep in mind: backend and frontend changes. - The rows endpoint and the group metadata layer must apply the same view constraints and ordering. -- Geometry used for placeholders, group headers, row sections, and add-row lines - must match the rendered UI dimensions, otherwise scroll math drifts. +- Geometry used for placeholders, group headers or spans, row sections, and + add-row lines must match the rendered UI dimensions, otherwise scroll math + drifts. - Optimistic updates must keep row counts and row placement indexes consistent until the next server reconciliation. - Cross-group row moves must update the complete destination group path before @@ -240,10 +243,12 @@ Known costs to keep in mind: ## Group Aggregations -When a column has a footer "Summarize" aggregation, each group header shows that -aggregation computed over the group's rows, at every depth. The values travel -inside the group metadata response, and the footer total can be bundled into the -same request, so grouped mode never calls the standalone aggregations endpoint. +When a column has a footer "Summarize" aggregation, the Sections layout shows that +aggregation in each group header, computed over the group's rows at every depth. +The Columns layout does not render per-group aggregation values, but it uses the +same group metadata and keeps the overall footer total. The values travel inside +the group metadata response, and the footer total can be bundled into the same +request, so grouped mode never calls the standalone aggregations endpoint. Updates are consolidated and backend-authoritative. A single group metadata request refreshes both the loaded group window and the footer totals, and every @@ -251,3 +256,33 @@ edit path and the realtime echo funnel through the same group-aware refresh. Row edits refresh silently; a spinner is reserved for changing a column's aggregation function. Sorting needs no refresh because these aggregations are order-independent. + +## Layouts + +A grid view stores a `group_by_layout` setting with the values `section` and +`column`; the corresponding user-facing labels are **Sections** and **Columns**. +Both layouts share the group metadata and row paging layers, while the setting +also controls the effective collapse, fetch, and render behavior. + +- **Sections** (`section`, the default): each group contributes a header and a gap. + Editable views expose an add-row line for each leaf group, groups can be + collapsed, and per-group aggregations are shown. +- **Columns** (`column`): one column per group-by level sits at the left of the + frozen section, and each group's value is drawn once as a cell spanning its + rows with a row count. Headers, gaps, and per-group add-row lines take no space. + The tree is always rendered and fetched fully expanded while the saved Sections + collapse state is retained but ignored, so collapse controls are unavailable. + Editable views expose one trailing add-row line for the whole grid. Per-group + aggregations are still computed but not rendered; the overall footer remains. + +The builder emits a `groupSpan` item per group at every depth instead of a header +in Columns. Its requests use expanded per-parent descendant loading even when the +retained Sections state is collapse-all. Consumers that only read row sections +(row fetching, drag targets, scroll-to-row, multi-select) are unaffected; any +loop over layout items that rejects unexpected types must skip spans. In Columns, +unloaded sibling ranges use adjacent loaded groups' absolute row offsets and +the parent's boundaries to determine their row space. Missing pages within a +range share that space until their offsets are known, keeping loaded groups +anchored while viewport fetches refine the gaps. Selection bounds also use the +full row count, including unloaded groups. In Sections, placeholders are sized by +header height and selection uses the visible row indexes as before. diff --git a/docs/technical/websockets.md b/docs/technical/websockets.md index ae1e70adc1..1d989d39aa 100644 --- a/docs/technical/websockets.md +++ b/docs/technical/websockets.md @@ -204,6 +204,16 @@ resets the buffer before restoring its old indexes; neither direction restores discarded history. These resets affect only realtime replay, not the underlying user data. +Migration `ws.0003` adds separate unlogged history-summary and history-state tables; +it preserves the existing event table and indexes. Before migration, pause and +drain the old cleanup task. Update all replay readers before enabling the new +cleanup, because older readers cannot use the summaries. Initial activation +establishes a conservative history floor from the logged event sequence. Cursors +below that floor refresh. After an unlogged-table reset, initialization takes the +same boundary while briefly blocking inserts, preserving the sequence's `CACHE 1` +requirement. Before rolling back, disable replay and establish fresh client +baselines; removing the summaries cannot restore compacted payloads. + ### Last Seen Event ID During normal delivery, the frontend advances its cursor to the highest `_event_id` @@ -218,11 +228,11 @@ sends a `replay_events` message carrying its last seen event ID as `last_seen_id The server uses that cursor and those subscriptions to decide whether recovery is possible, with these completed outcomes: -1. **Nothing missed** — The replay window contains only the client's `last_seen_id`. The client is already up to date for the channel groups being restored. +1. **Nothing missed** — No relevant event follows the client's `last_seen_id`, in either full events or expired-history summaries. The client is already up to date for the channel groups being restored, even if the acknowledged event itself has been removed. 2. **Events replayed** — The server fetches the missed events for the client's page channel groups and implicit `users` group, filters out the client's own broadcasts (via its web socket id) and any events not relevant to that user, and re-invokes them through the consumer's handlers in order — exactly as if they had arrived live. The client catches up without a page reload. -3. **Can't replay** — Either too many events were missed (more than `BASEROW_REALTIME_REPLAY_MAX_EVENTS`), the client's `last_seen_id` has already been cleaned up by retention, or the server finds a persisted event it cannot safely re-deliver through a websocket broadcast handler. The server responds with `force_refresh=true` and the client shows a "workspace data is outdated" toast with a refresh action. +3. **Can't replay** — A relevant missed event has expired, the cursor predates known history or is ahead of the recorded high-water mark, too many events were missed (more than `BASEROW_REALTIME_REPLAY_MAX_EVENTS`), or the server finds a persisted event it cannot safely re-deliver through a websocket broadcast handler. The server responds with `force_refresh=true` and the client shows a "workspace data is outdated" toast with a refresh action. -Every `replay_events_result` with `force_refresh=false` includes `latest_event_id`, the latest event ID the server can safely acknowledge for that replay decision. If a client connects without a `last_seen_id` (a fresh page load), the server returns the latest persisted event ID as the new baseline because there is nothing to replay. When replay succeeds, `latest_event_id` is the last event in the replay window and might be lower than the global latest persisted ID if newer events were irrelevant to that client. If the server responds with `force_refresh=true`, `latest_event_id` is not meaningful and the client should refresh instead. +Every `replay_events_result` with `force_refresh=false` includes `latest_event_id`, the latest event ID the server can safely acknowledge for that replay decision. A fresh page load receives the recorded high-water mark, including compacted history, as its baseline. When replay succeeds, `latest_event_id` is the last replayed event, or the original cursor when nothing relevant changed. It can be lower than the global high-water mark when newer events were irrelevant. Full events, expired summaries and the known-history floor are checked in one database snapshot, so cleanup cannot expose a gap between deleting an event and recording its expired history. If `force_refresh=true`, `latest_event_id` is not meaningful and the client should refresh instead. ### Replay resource limits and retries @@ -243,7 +253,7 @@ connection attempt. For overload, timeouts, and database failures, clients advertising `supports_retry=true` receive `replay_events_retry` and retry on the same socket with backoff and jitter. Older clients receive the existing refresh fallback. -An expired cursor, excessive event gap, or disabled recording still requires a +Expired relevant history, an excessive event gap, or disabled recording requires a refresh when recovering missed updates. The frontend keeps one replay request or retry timer active and holds the original @@ -256,19 +266,35 @@ longer be verified. An unrecoverable gap stays marked outdated across reconnects ### Event Cleanup -A periodic Celery task removes events older than 24 hours, independently of JWT refresh-token lifetime. It runs every minute, including when recording is disabled, with a 30-second work budget and at most 5,000 events per committed batch. Each deletion statement has a three-second timeout and a 250 ms lock timeout, preserving stricter database settings. Locked rows are left for a later run. Clients whose baseline has expired use the existing refresh fallback. - -Each batch commits separately, so earlier deletions survive a later failure. A -scheduled run skips cleanup while another task owns the nonblocking lease. The -retention target is not a hard maximum row age: locked rows or a sustained cleanup -backlog can remain until a later run. - -A surviving baseline older than the retention window cannot prove complete -history: cleanup may have skipped its lock while deleting newer expired events. -Replay checks the baseline's age as well as its existence before acknowledging it. - -The `(created_at, id)` index supports bounded expiration scans. Recipient indexes -are restricted to the shared `users` channel; page events retain the +A periodic Celery task compacts events older than `BASEROW_REALTIME_REPLAY_RETENTION_HOURS` (24 hours by default), independently of JWT refresh-token lifetime. It runs every ten minutes, including when recording is disabled, with a 240-second work budget and at most 5,000 events per committed batch. Each statement has a three-second timeout and a 250 ms lock timeout, tightened to the remaining budget without relaxing stricter database settings. Locked rows are left for a later run. + +Each exact audience gets a separate summary containing routing metadata and the +newest expired event IDs, without full business payloads. The summary keeps the +newest event and the newest event with a different ignored socket: any client +excludes at most one socket, so these two entries preserve its latest relevant +expired event. New browser sessions therefore do not create new summary rows for +an otherwise identical audience. Distinct audiences can still accumulate; summary +storage has no fixed size bound or expiry. Row-scoped channel groups such as +`table--row-` give one summary per edited row rather than per table, so +size this against row volume. + +Summary updates, deletion of all selected full events, and advancement of the +compacted high-water mark commit together. Each batch commits separately, so +earlier progress survives a later failure. Ordinary replay reads remain available +while a batch is uncommitted. A scheduled run skips cleanup while another task owns +the nonblocking 330-second lease. The retention target is not a hard maximum row +age: locked rows or a sustained cleanup backlog can remain until a later run. + +Replay also checks the age of relevant full events left behind by cleanup, so +locked rows or cleanup lag cannot make expired payloads replayable. Increasing the +retention window cannot restore already compacted payloads. Deleting events or +summaries outside this cleanup is unsupported without explicitly invalidating +existing replay cursors. + +The existing `(created_at, id)` index supports bounded expiration scans; no second +age index is added. Since expired originals are removed, retained summaries do not +accumulate in that scan. Recipient indexes are restricted to the shared `users` +channel; page events retain the `(channel_group, id)` index. Cleanup makes storage reusable through PostgreSQL vacuum; it does not normally shrink the table's allocated files. Monitor recording rate, committed cleanup progress, and database vacuum activity together; see @@ -287,5 +313,6 @@ an immediate `ANALYZE`. | Setting | Default | Purpose | |---|---|---| | `BASEROW_REALTIME_REPLAY_MAX_EVENTS` | 200 | Maximum number of missed events the server will replay. Beyond this, the client is told to refresh. Set to `0` to disable event recording and replay; retention cleanup continues. Clients learn replay availability during authentication and use refresh when missed events cannot be recovered. | +| `BASEROW_REALTIME_REPLAY_RETENTION_HOURS` | 24 | Maximum age in hours of full events available for replay. Must be a positive integer. Older relevant missed history requires a refresh; increasing the window cannot restore compacted payloads. | See [configuration.md](../installation/configuration.md) for the full settings reference. diff --git a/docs/testing/grid-view-test-plan.md b/docs/testing/grid-view-test-plan.md index 72dd2a012b..064708d00b 100644 --- a/docs/testing/grid-view-test-plan.md +++ b/docs/testing/grid-view-test-plan.md @@ -103,8 +103,9 @@ written. | Paste | group-by | formula | ✗ | TODO | TODO | TODO | TODO | — | Regular-field filter/sort create/update cases and saved filter/sort load cases -are parametrized to run in both flat and expanded group-by views. Dedicated -group-by rows above cover creating inside a group and changing a grouped value. +are parametrized to run in both flat and expanded Sections group-by views. +Columns-specific coverage is listed in section 9.6. Dedicated group-by rows above +cover creating inside a Sections group and changing a grouped value. ### Combined constraints (TODO) @@ -144,7 +145,8 @@ optimistic) and any-formula (deferred) configurations. ### 1.2.1a Add row that does not match an active simple-field filter and keep it selected -These simple-field filter cases run in both flat and expanded group-by views. +These simple-field filter cases run in both flat and expanded Sections group-by +views. - Empty row is appended at the bottom. - Empty primary and field cells are visible immediately. @@ -228,7 +230,8 @@ These simple-field filter cases run in both flat and expanded group-by views. ### 1.3.1a Add row with simple-field sort ASC and keep it selected -These simple-field sort cases run in both flat and expanded group-by views. +These simple-field sort cases run in both flat and expanded Sections group-by +views. - Empty row is appended at the bottom. - Empty primary and field cells are visible immediately. @@ -339,6 +342,8 @@ These simple-field sort cases run in both flat and expanded group-by views. ### 1.5.1 Add row from a group add-row line +These add-in-group scenarios exercise the per-leaf add-row lines in Sections. + - New row is inserted at the bottom of the selected group. - The group-by field is set to that group's value. - Other groups keep their rows and order. @@ -424,7 +429,8 @@ These simple-field sort cases run in both flat and expanded group-by views. ### 2.2.1a Edit filtered row to non-matching value and keep it selected -These simple-field filter cases run in both flat and expanded group-by views. +These simple-field filter cases run in both flat and expanded Sections group-by +views. - Typed value is visible immediately. - No row loading spinner is shown for this optimistic text update. @@ -535,7 +541,8 @@ These simple-field filter cases run in both flat and expanded group-by views. ### 2.3.1a Edit sorted row so it should move and keep it selected -These simple-field sort cases run in both flat and expanded group-by views. +These simple-field sort cases run in both flat and expanded Sections group-by +views. - Typed value is visible immediately. - No row loading spinner is shown for this optimistic text update. @@ -755,7 +762,7 @@ These simple-field sort cases run in both flat and expanded group-by views. ## Filters -These saved filter cases run in both flat and expanded group-by views. +These saved filter cases run in both flat and expanded Sections group-by views. ### 5.1.1 Load view with API-created filter @@ -774,7 +781,7 @@ These saved filter cases run in both flat and expanded group-by views. ## Sorts -These saved sort cases run in both flat and expanded group-by views. +These saved sort cases run in both flat and expanded Sections group-by views. ### 6.1.1 Sort ASC @@ -796,30 +803,32 @@ operation buckets — create in a group (1.5.x), changing a group-by value (2.2. 2.4.x), and group-by-affecting paste (4.7.x) — plus the parametrized flat/group-by runs, the same way filters and sorts do. -### 7.1.1 Grouped views load expanded and collapse or expand all groups +### 7.1.1 Sections views load expanded and collapse or expand all groups -- Grouped views initially render group banners expanded with their rows visible. +- A grouped view using Sections initially renders its group banners expanded with + their rows visible. - Collapsing all groups hides every group's rows and leaves the banners visible. - Expanding all groups restores the rows in their groups and keeps group counters. - Rows fetched on the initial expanded load stay in the store while collapsed, so re-expanding does not refetch them. -### 7.1.2 Collapse and expand a group +### 7.1.2 Collapse and expand a group in Sections - Collapsing one group hides only that group's rows. - Other expanded groups stay visible. - Expanding the group restores its rows in place. -### 7.2.1 Adding a group-by refreshes with one metadata request and one row request +### 7.2.1 Adding a group-by in Sections refreshes with one metadata and one row request -- A view already grouped by one field renders its group banners with counters. +- A Sections view already grouped by one field renders its group banners with + counters. - Adding a second group-by field issues exactly one group-by-data metadata request and exactly one grid rows request, not a request per group. - The group-by-data request includes `include_descendants=true`. - New nested group banners and their counters are visible after the refresh. - Total row count is unchanged. -### 7.3.1 Group headers show per-column aggregation summaries +### 7.3.1 Sections group headers show per-column aggregation summaries - When a column has a "Summarize" aggregation configured, each group header shows that column's aggregation computed over that group's rows, column-aligned under @@ -830,7 +839,7 @@ flat/group-by runs, the same way filters and sorts do. its bottom-footer value is unaffected. - The bottom footer still shows the overall total for the column. -### 7.3.2 Hovering a group header to pick or change an aggregation +### 7.3.2 Hovering a Sections group header to pick or change an aggregation - Hovering a group-by header reveals, per column, the aggregation value or a `+ Summarize` affordance on the hovered column (only the hovered column). @@ -841,10 +850,10 @@ flat/group-by runs, the same way filters and sorts do. update in place. The group layout (rows, counts, offsets) is not rebuilt; only the aggregation values change. -### 7.3.3 Live group aggregation updates on row changes +### 7.3.3 Live Sections group aggregation updates on row changes -- Editing a cell updates that row's group value (and its ancestor groups + the - footer total) in place, with no spinner — like a spreadsheet SUM. +- In Sections, editing a cell updates that row's group value (and its ancestor + groups + the footer total) in place, with no spinner — like a spreadsheet SUM. - Moving a row between groups (editing a group-by field) updates both the source and destination group values. - A second browser session viewing the same grouped view sees the values update @@ -904,9 +913,9 @@ flat/group-by runs, the same way filters and sorts do. - No row loading spinner is shown because row coloring is loaded from the saved view decoration. -### 9.1.2 Row coloring with group-by +### 9.1.2 Row coloring with Sections group-by -- Grid opens with the saved group-by applied. +- Grid opens with the saved group-by and Sections layout applied. - Group banners show each group value with its row count. - Row background uses the selected option color in the frozen-left section. - Row background uses the selected option color in the scrollable-right section. @@ -964,6 +973,82 @@ flat/group-by runs, the same way filters and sorts do. - Row identifier values are no longer visible. - Count values persist after reload. +### 9.6.1 Switch to Columns replaces banners with spanning group cells + +- Grouped by Team with three rows in two groups. +- The group layout control uses the user-facing labels "Sections" and "Columns"; + their stored values are `section` and `column`. +- Choosing "Columns" in the group-by menu sends one view PATCH with + `group_by_layout: "column"`. +- No `.grid-view__group-by-banner` remains; each group renders one + `.grid-view__group-span` in the frozen section whose `.grid-view__group-count` + shows the group's row count; all rows stay visible and ordered. +- The layout survives a reload. + +### 9.6.2 Second level adds a column; Sections restores the banners + +- Adding a second group-by in Columns renders two `.grid-view__head-group` + header cells and one correctly counted span for each nested group. +- Choosing "Sections" sends one view PATCH, restores the banners and removes every + span. + +### 9.6.3 Five Columns groups keep the data pane usable at a narrow viewport + +- A view grouped by the maximum five fields opens in Columns at a 1024px browser + viewport. +- All five group headers and spans remain visible, while their display widths are + proportionally fitted without going below the minimum column width. +- The right data pane remains 300px wide, contains the primary value, and exposes + its horizontal scrollbar. +- Group headers and body spans remain aligned, and resize handles are withheld + while responsive fitting is active so a drag cannot persist a misleading width. +- Widening the viewport restores every configured 200px group width without a + persistence mutation and restores the resize handles. + +### 9.6.4 Column width resize persists + +- At a wide viewport, dragging the group-column resize handle by 40px sends one + group-by PATCH containing `width: 240`. +- The header and every matching body span move from 200px to 240px together. +- The 240px width survives a reload. +- Starting a resize before responsive fitting activates and dragging beyond the + available group width still saves the requested width on mouseup. The active + handle remains mounted for the drag; widening the viewport restores the saved + width and resize handles. + +### 9.6.5 Row drag uses the Columns offset and persists + +- Dragging a row within a group renders the drag overlay after the 200px group + column and exposes a valid drop target. +- The move request targets the expected `before_id` and view. +- The new row order and group count survive a reload. + +### 9.6.6 Columns virtualizes thousands of uneven rows and groups + +- The fixture contains 3,000 rows grouped into 50 parent groups and 1,500 nested + leaf groups. Leaf groups alternate between one and three rows, while parents + alternate between 29 and 31 leaves, forcing sparse page estimates to converge + across uneven boundaries. +- Switching from collapsed Sections to Columns expands the rendered hierarchy + without overwriting the saved Sections collapse state. +- Deep wheel jumps across singleton/large leaf, parent, sparse-page and midpoint + boundaries render exact absolute count identifiers and matching parent/leaf + spans. +- The first parent span keeps its exact 57-row height and sticky label at a deep + offset. +- Fewer than 100 left rows, right rows and spans are mounted at both the start and + end. Attempted group and row data requests remain independently below 200 and + below 250 combined. +- Switching back to Sections restores the collapsed state; switching to Columns + again and reloading preserves the Columns layout, hydrates the first row, and + reaches a deep sparse row directly. +- No API response with an error status, non-cancellation transport failure, or + page error occurs. +- With 160 groups whose first and last 40 groups contain one row each and middle + 80 groups contain 100 rows each, jumping directly to row index 4,000 renders + that row. Loading a middle group page must not shift its known absolute row + offset or stop refinement while displaying a different row at that position. + ## Cell Selection ### 10.1.1 Click a cell @@ -998,6 +1083,14 @@ flat/group-by runs, the same way filters and sorts do. a 2x2 area, addressed by visible row index across group banners rather than raw row position. +### 10.1.7 Columns cell selection works across sparse group pages + +- In a Columns view with 120 one-row groups, load groups 0–39 and jump to groups + 80–119 without loading the middle page. +- Dragging or shift-clicking from row index 100 to 101 and an adjacent field + selects the expected 2x2 range. Correcting the selection after a refresh keeps + it intact; selection bounds use the full row count. + ### 10.2.1 Shift+Arrow expands selected range - Starting from one selected cell, Shift+ArrowRight expands the selected range @@ -1146,8 +1239,8 @@ flat/group-by runs, the same way filters and sorts do. valid drag targets. - Insertion slots in every other leaf group are unavailable, including when the destination differs only in writable grouped fields. - - Collapsed group headers and unloaded row placeholders are never drop - targets. + - In Sections, collapsed group headers are never drop targets; unloaded row + placeholders are never drop targets in either layout. ## TODO Coverage @@ -1178,19 +1271,23 @@ flat/group-by runs, the same way filters and sorts do. constraints. Regular keep-selected and deselect-before-confirmation are covered by 4.7.1a–4.7.1b. - 5.3.x: OR filter combinations and filter groups. -- 7.x: Group-by standalone behavior not yet automated — each group renders its - own add-row line and adding from a non-first group lands in that group; the - add-row line keeps the small row height at medium/large row heights (covered - by unit tests, not e2e); an empty grouped view keeps a top-level add-row line - and a row created from it lands in its value-derived group, selected and with - a filter warning when it does not match (covered by unit tests, not e2e); - group banners draw a vertical cell - separator at each field boundary in the scrollable-right section (covered by - unit tests, not e2e); collapse/expand state persists across reload and is - re-projected (kept on trailing-level add/remove, reset on reorder/replace); - nested multi-level group-by collapse/expand, counts, and add-row; - lazy-loading rows when scrolling deep into a large group (covered by unit - tests, not e2e). +- 7.x: Remaining Sections-specific standalone behavior not yet automated — each + leaf group renders its own add-row line and adding from a non-first group lands + in that group; the add-row line keeps the small row height at medium/large row + heights (covered by unit tests, not e2e); an empty grouped view keeps a + top-level add-row line and a row created from it lands in its value-derived + group, selected and with a filter warning when it does not match (covered by + unit tests, not e2e); Sections group headers draw a vertical cell separator at + each field boundary in the scrollable-right section (covered by unit tests, + not e2e); collapse/expand state persists across reload and is re-projected + (kept on trailing-level add/remove, reset on reorder/replace); nested + multi-level Sections collapse/expand, counts, and add-row; lazy-loading rows + when scrolling deep into a large group (covered by unit tests, not e2e). +- 9.6.x: Columns-specific behavior not yet automated — editable views expose one + trailing add-row line for the whole grid and no per-group add-row lines; adding + from it places the row in its value-derived group; read-only views do not show + it. Columns does not render per-group aggregation values or pickers, while the + overall footer total remains visible and updates after row changes. - 15.x: Ad-hoc grouping in the publicly shared grid — a visitor applying a different group-by rebuilds the group tree and loads rows for the new groups (covered by backend tests, not e2e). @@ -1207,10 +1304,6 @@ flat/group-by runs, the same way filters and sorts do. - The UI does not allow freezing more than 4 columns. - The UI does not allow freezing more columns than can fit in the available width. -- 9.6.x: Drag-and-drop rows when no sort is applied. - - Drop target is visible while dragging. - - Dropped row moves to the new visual position. - - Row order persists after reload. - 9.7.x: Drag-and-drop fields. - Field-drag preview is visible while dragging a field header. - Target position is visible while dragging a field header. diff --git a/e2e-tests/pages/database/gridPage.ts b/e2e-tests/pages/database/gridPage.ts index 714d031659..3c5a46b7bc 100644 --- a/e2e-tests/pages/database/gridPage.ts +++ b/e2e-tests/pages/database/gridPage.ts @@ -41,9 +41,7 @@ export class GridPage { async goTo(database: Database, table: Table, view?: View): Promise { const url = new URL( - `/database/${database.id}/table/${table.id}${ - view ? `/${view.id}` : "" - }`, + `/database/${database.id}/table/${table.id}${view ? `/${view.id}` : ""}`, this.baseUrl, ); url.searchParams.set("token", this.user.refreshToken); @@ -263,6 +261,31 @@ export class GridPage { .first(); } + groupSpanByValue(value: string): Locator { + return this.page.locator(".grid-view__left .grid-view__group-span", { + has: this.page + .locator(".grid-view__group-value") + .filter({ hasText: this.exactTextRegex(value) }), + }); + } + + async selectGroupLayout(label: "Sections" | "Columns"): Promise { + await this.openGroupByContext(); + await this.groupByContext() + .locator(".segment-control__button", { hasText: label }) + .click(); + await this.closeGroupByContext(); + } + + async expectGroupSpanCount(value: string, count: number): Promise { + const span = this.groupSpanByValue(value); + await expect(span).toHaveCount(1, { timeout: 10_000 }); + await expect(span.locator(".grid-view__group-count")).toHaveText( + this.exactTextRegex(String(count)), + { timeout: 10_000 }, + ); + } + groupByContextToggle(): Locator { return this.page .locator(".header__filter-link", { diff --git a/e2e-tests/tests/database/grid/grid_group_layout_stress.spec.ts b/e2e-tests/tests/database/grid/grid_group_layout_stress.spec.ts new file mode 100644 index 0000000000..030699f6f1 --- /dev/null +++ b/e2e-tests/tests/database/grid/grid_group_layout_stress.spec.ts @@ -0,0 +1,348 @@ +import { expect, test } from "../../baserowTest"; +import { createRows } from "../../../fixtures/database/rows"; +import { + type GridSetupResult, + setupGrid, +} from "../../../fixtures/database/gridSetup"; +import { patchView } from "../../../fixtures/database/view"; +import { GridPage } from "../../../pages/database/gridPage"; +import { baserowConfig } from "../../../playwright.config"; + +const ROW_COUNT = 3_000; +const LEAF_GROUP_COUNT = 1_500; +const PARENT_GROUP_COUNT = 50; +const BATCH_SIZE = 200; + +type LeafGroupFixture = { + index: number; + parentIndex: number; + rowStart: number; + rowCount: number; +}; + +type ParentGroupFixture = { + index: number; + leafStart: number; + leafCount: number; + rowStart: number; + rowCount: number; +}; + +function rowsBeforeLeafGroup(index: number): number { + return Math.floor(index / 2) * 4 + (index % 2 === 0 ? 0 : 1); +} + +const PARENT_GROUPS: ParentGroupFixture[] = (() => { + let leafStart = 0; + return Array.from({ length: PARENT_GROUP_COUNT }, (_, index) => { + // Alternating parent widths deliberately put leaf-group and row boundaries at + // different offsets throughout the dataset. Each pair still contains 60 + // leaves, so 50 parents cover exactly 1,500 leaves. + const leafCount = index % 2 === 0 ? 29 : 31; + const rowStart = rowsBeforeLeafGroup(leafStart); + const group = { + index, + leafStart, + leafCount, + rowStart, + rowCount: rowsBeforeLeafGroup(leafStart + leafCount) - rowStart, + }; + leafStart += leafCount; + return group; + }); +})(); + +const LEAF_GROUPS: LeafGroupFixture[] = PARENT_GROUPS.flatMap((parent) => + Array.from({ length: parent.leafCount }, (_, offset) => { + const index = parent.leafStart + offset; + return { + index, + parentIndex: parent.index, + rowStart: rowsBeforeLeafGroup(index), + rowCount: index % 2 === 0 ? 1 : 3, + }; + }), +); + +function padded(value: number): string { + return String(value).padStart(4, "0"); +} + +function leafGroupForRow(rowIndex: number): LeafGroupFixture { + const pairIndex = Math.floor(rowIndex / 4); + const groupIndex = pairIndex * 2 + (rowIndex % 4 === 0 ? 0 : 1); + const group = LEAF_GROUPS[groupIndex]; + if (group === undefined) { + throw new Error(`No leaf group fixture for row ${rowIndex}.`); + } + return group; +} + +test.describe("9.6 Column group layout at scale @slow", () => { + test.describe.configure({ mode: "serial" }); + test.setTimeout(180_000); + + let g: GridSetupResult; + + test.beforeAll(async () => { + g = await setupGrid({ + dbName: "GroupLayoutScaleDb", + fields: [ + { name: "Cluster", type: "text" }, + { name: "Team", type: "text" }, + ], + groupBys: [ + { fieldName: "Cluster", order: "ASC" }, + { fieldName: "Team", order: "ASC" }, + ], + }); + + for (let start = 0; start < ROW_COUNT; start += BATCH_SIZE) { + const rows = Array.from( + { length: Math.min(BATCH_SIZE, ROW_COUNT - start) }, + (_, offset) => { + const index = start + offset; + const group = leafGroupForRow(index); + return { + Name: `Scale row ${padded(index)}`, + Cluster: `Cluster ${padded(group.parentIndex)}`, + Team: `Group ${padded(group.index)}`, + }; + }, + ); + await createRows(g.user, g.table, rows); + } + + await patchView(g.user, g.view, { + group_by_layout: "section", + row_identifier_type: "count", + }); + }); + + test("9.6.6 virtualizes, switches layouts, and reaches uneven paged boundaries across thousands of rows and groups", async ({ + page, + }) => { + const grid = new GridPage(page, g.user); + const failedApiResponses: string[] = []; + const failedApiRequests: string[] = []; + const pageErrors: string[] = []; + const groupDataRequests: string[] = []; + const rowDataRequests: string[] = []; + const apiPrefix = new URL( + "/api/", + baserowConfig.PUBLIC_BACKEND_URL, + ).toString(); + const groupDataPath = `/api/database/views/grid/${g.view.id}/group-by-data/`; + const rowDataPath = `/api/database/views/grid/${g.view.id}/`; + const rowHeight = 33; + + const expectVisibleGroupSpanCount = async ( + value: string, + count: number, + ) => { + await grid.expectGroupSpanCount(value, count); + await expect(grid.groupSpanByValue(value)).toBeVisible({ + timeout: 15_000, + }); + }; + + const expectVirtualizedDom = async () => { + const [spanCount, leftRowCount, rightRowCount] = await Promise.all([ + page.locator(".grid-view__left .grid-view__group-span").count(), + grid.leftRows().count(), + grid.rightRows().count(), + ]); + expect(spanCount).toBeGreaterThan(0); + expect(spanCount).toBeLessThan(100); + expect(leftRowCount).toBeGreaterThan(0); + expect(leftRowCount).toBeLessThan(100); + expect(rightRowCount).toBeGreaterThan(0); + expect(rightRowCount).toBeLessThan(100); + expect(leftRowCount).toBe(rightRowCount); + }; + + const scrollToRow = async (index: number) => { + const target = grid + .leftRows() + .filter({ hasText: `Scale row ${padded(index)}` }) + .first(); + const body = page.locator(".grid-view__left .grid-view__body"); + await page + .locator(".grid-view__right .grid-view__body") + .hover({ position: { x: 10, y: 10 } }); + const currentScrollTop = await body.evaluate( + (element) => element.scrollTop, + ); + await page.mouse.wheel(0, index * rowHeight - currentScrollTop); + await expect(target).toBeVisible({ timeout: 15_000 }); + return target; + }; + + // Count every attempted group/row GET, including requests that a layout change + // legitimately cancels, so the upper bounds reflect actual browser work. + page.on("request", (request) => { + if (request.method() !== "GET") { + return; + } + const url = new URL(request.url()); + if (url.pathname === groupDataPath) { + groupDataRequests.push(request.url()); + } else if ( + url.pathname === rowDataPath && + url.searchParams.has("limit") + ) { + rowDataRequests.push(request.url()); + } + }); + page.on("response", (response) => { + if (response.url().startsWith(apiPrefix) && response.status() >= 400) { + failedApiResponses.push(`${response.status()} ${response.url()}`); + } + }); + page.on("requestfailed", (request) => { + const failure = request.failure()?.errorText ?? "failed"; + // Layout changes and the final reload deliberately cancel stale virtual-row + // prefetches. Those browser-level aborts are expected; transport failures are not. + const wasCancelled = /ERR_ABORTED|NS_BINDING_ABORTED/i.test(failure); + if (request.url().startsWith(apiPrefix) && !wasCancelled) { + failedApiRequests.push(`${failure} ${request.url()}`); + } + }); + page.on("pageerror", (error) => pageErrors.push(error.message)); + + // Keep the primary field in the frozen section even when the optional assistant + // panel is open, so the page object's primary-cell helpers remain deterministic. + await page.setViewportSize({ width: 1_920, height: 1_080 }); + await grid.goTo(g.database, g.table); + await grid.collapseAllGroupsFromContext(); + await grid.expectGroupByBanner( + "Cluster 0000", + PARENT_GROUPS[0].rowCount, + true, + ); + await expect(grid.leftRows()).toHaveCount(0); + await expect(grid.rightRows()).toHaveCount(0); + + // Columns always expands the hierarchy for display, without losing the saved + // collapse-all state that must be restored when returning to Sections. + await grid.selectGroupLayout("Columns"); + await expectVisibleGroupSpanCount( + "Cluster 0000", + PARENT_GROUPS[0].rowCount, + ); + await expectVisibleGroupSpanCount("Group 0000", LEAF_GROUPS[0].rowCount); + await grid.expectPrimaryText(0, "Scale row 0000"); + await grid.expectRowIdentifierText(0, "1"); + + await expectVirtualizedDom(); + + // Exercise one- and three-row leaves, an uneven parent boundary, two sparse + // page boundaries, and the middle of the 3,000-row dataset. + for (const index of [40, 56, 57, 199, 200, 1_499, 1_500]) { + const group = leafGroupForRow(index); + const parent = PARENT_GROUPS[group.parentIndex]; + const sampledRow = await scrollToRow(index); + // Scrolling starts over the right rows. Once the target row replaces the row + // under the pointer, its identifier is intentionally swapped for the hover + // checkbox, so move away before checking the count. + await page.mouse.move(0, 0); + await expect( + sampledRow.locator(".grid-view__row-count-content"), + ).toHaveText(String(index + 1)); + await expectVisibleGroupSpanCount( + `Group ${padded(group.index)}`, + group.rowCount, + ); + await expectVisibleGroupSpanCount( + `Cluster ${padded(parent.index)}`, + parent.rowCount, + ); + + if (index === 40) { + const parentSpan = grid.groupSpanByValue("Cluster 0000"); + const parentBox = await parentSpan.boundingBox(); + const stickyBox = await parentSpan + .locator(".grid-view__group-cell--sticky") + .boundingBox(); + const bodyBox = await page + .locator(".grid-view__left .grid-view__body") + .boundingBox(); + expect(parentBox?.height).toBe(PARENT_GROUPS[0].rowCount * rowHeight); + expect(Math.abs((stickyBox?.y ?? 0) - (bodyBox?.y ?? 0))).toBeLessThan( + 2, + ); + } + } + + const finalRow = await scrollToRow(ROW_COUNT - 1); + await page.mouse.move(0, 0); + await expect(finalRow.locator(".grid-view__row-count-content")).toHaveText( + String(ROW_COUNT), + ); + await expectVisibleGroupSpanCount( + `Group ${padded(LEAF_GROUP_COUNT - 1)}`, + LEAF_GROUPS[LEAF_GROUP_COUNT - 1].rowCount, + ); + await expectVisibleGroupSpanCount( + `Cluster ${padded(PARENT_GROUP_COUNT - 1)}`, + PARENT_GROUPS[PARENT_GROUP_COUNT - 1].rowCount, + ); + + await expectVirtualizedDom(); + + await grid.selectGroupLayout("Sections"); + await grid.expectGroupByBanner( + `Cluster ${padded(PARENT_GROUP_COUNT - 1)}`, + PARENT_GROUPS[PARENT_GROUP_COUNT - 1].rowCount, + true, + ); + await expect(grid.leftRows()).toHaveCount(0); + await expect(grid.rightRows()).toHaveCount(0); + await expect(page.locator(".grid-view__group-span")).toHaveCount(0); + + await grid.selectGroupLayout("Columns"); + await scrollToRow(ROW_COUNT - 1); + await grid.goTo(g.database, g.table); + await expectVisibleGroupSpanCount( + "Cluster 0000", + PARENT_GROUPS[0].rowCount, + ); + await grid.expectPrimaryText(0, "Scale row 0000"); + await grid.expectRowIdentifierText(0, "1"); + + // A reload must hydrate the initial row window, then be able to jump directly + // back into a deep sparse page without walking every preceding group. + const deepRowIndex = 2_501; + const deepGroup = leafGroupForRow(deepRowIndex); + const deepParent = PARENT_GROUPS[deepGroup.parentIndex]; + const deepRow = await scrollToRow(deepRowIndex); + await page.mouse.move(0, 0); + await expect(deepRow.locator(".grid-view__row-count-content")).toHaveText( + String(deepRowIndex + 1), + ); + await expectVisibleGroupSpanCount( + `Group ${padded(deepGroup.index)}`, + deepGroup.rowCount, + ); + await expectVisibleGroupSpanCount( + `Cluster ${padded(deepParent.index)}`, + deepParent.rowCount, + ); + await expectVirtualizedDom(); + + expect(groupDataRequests.length).toBeGreaterThan(0); + expect(groupDataRequests.length).toBeLessThan(200); + expect(rowDataRequests.length).toBeGreaterThan(0); + expect(rowDataRequests.length).toBeLessThan(200); + expect(groupDataRequests.length + rowDataRequests.length).toBeLessThan(250); + expect( + rowDataRequests.some((requestUrl) => { + const offset = new URL(requestUrl).searchParams.get("offset"); + return offset !== null && Number(offset) > 1_000; + }), + ).toBe(true); + expect(failedApiResponses).toEqual([]); + expect(failedApiRequests).toEqual([]); + expect(pageErrors).toEqual([]); + }); +}); diff --git a/e2e-tests/tests/database/grid/grid_view_options.spec.ts b/e2e-tests/tests/database/grid/grid_view_options.spec.ts index 55f3abee1a..e2aec29468 100644 --- a/e2e-tests/tests/database/grid/grid_view_options.spec.ts +++ b/e2e-tests/tests/database/grid/grid_view_options.spec.ts @@ -100,6 +100,41 @@ async function waitForViewPatch( await waitForApiPatch(page, `/api/database/views/${viewId}/`, action); } +async function expectSingleViewPatch( + page: Page, + viewId: number, + expectedBody: Record, + action: () => Promise, +): Promise { + const path = `/api/database/views/${viewId}/`; + const matchingRequests: Request[] = []; + const isMatchingRequest = (request: Request) => { + const url = new URL(request.url()); + return url.pathname === path && request.method() === "PATCH"; + }; + const onRequest = (request: Request) => { + if (isMatchingRequest(request)) { + matchingRequests.push(request); + } + }; + + page.on("request", onRequest); + try { + const responsePromise = page.waitForResponse((response) => + isMatchingRequest(response.request()), + ); + await action(); + const response = await responsePromise; + + expect(response.ok()).toBe(true); + expect(response.request().postDataJSON()).toEqual(expectedBody); + await page.waitForTimeout(100); + expect(matchingRequests).toHaveLength(1); + } finally { + page.off("request", onRequest); + } +} + async function waitForFieldOptionsPatch( page: Page, viewId: number, @@ -960,8 +995,363 @@ test.describe("9.5 Row identifier type", () => { // section 15 Public shared grid view // ----------------------------------------------------------------------------- +test.describe("9.6 Group layout", () => { + async function setupGroupLayoutGrid(page: Page) { + const g = await setupGrid({ + dbName: "GroupLayoutDb", + fields: [ + { name: "Team", type: "text" }, + { name: "Role", type: "text" }, + ], + rows: [ + { Name: "Alice", Team: "A", Role: "Developer" }, + { Name: "Ada", Team: "A", Role: "Designer" }, + { Name: "Bob", Team: "B", Role: "QA" }, + ], + groupBys: [{ fieldName: "Team", order: "ASC" }], + }); + const grid = new GridPage(page, g.user); + await grid.goTo(g.database, g.table); + return { g, grid }; + } + + test("9.6.1 switching to Columns replaces banners with spanning group cells and persists", async ({ + page, + }) => { + const { g, grid } = await setupGroupLayoutGrid(page); + await expect(grid.groupByBannerByValue("A")).toBeVisible({ + timeout: 10_000, + }); + + await expectSingleViewPatch( + page, + g.view.id, + { group_by_layout: "column" }, + () => grid.selectGroupLayout("Columns"), + ); + await expect(page.locator(".grid-view__group-by-banner")).toHaveCount(0); + await grid.expectGroupSpanCount("A", 2); + await grid.expectGroupSpanCount("B", 1); + await expect( + page.locator(".grid-view__left .grid-view__group-span"), + ).toHaveCount(2); + await grid.expectRowCount(3); + for (const name of ["Alice", "Ada", "Bob"]) { + const cell = page.locator(".grid-view__body .grid-field-text", { + hasText: new RegExp(`^\\s*${name}\\s*$`), + }); + await expect(cell).toHaveCount(1); + await expect(cell).toBeVisible(); + } + + await grid.goTo(g.database, g.table); + await grid.expectGroupSpanCount("A", 2); + await grid.expectGroupSpanCount("B", 1); + await expect(page.locator(".grid-view__group-by-banner")).toHaveCount(0); + }); + + test("9.6.2 a second group-by adds a second column and Sections restores the banners", async ({ + page, + }) => { + const { g, grid } = await setupGroupLayoutGrid(page); + await expectSingleViewPatch( + page, + g.view.id, + { group_by_layout: "column" }, + () => grid.selectGroupLayout("Columns"), + ); + + const groupByCreated = page.waitForResponse((response) => { + const url = new URL(response.url()); + return ( + url.pathname === `/api/database/views/${g.view.id}/group_bys/` && + response.request().method() === "POST" + ); + }); + await grid.addGroupBy("Role"); + expect((await groupByCreated).ok()).toBe(true); + await expect( + page.locator(".grid-view__left .grid-view__head-group"), + ).toHaveCount(2); + await grid.expectGroupSpanCount("A", 2); + await grid.expectGroupSpanCount("B", 1); + await grid.expectGroupSpanCount("Designer", 1); + await grid.expectGroupSpanCount("Developer", 1); + await grid.expectGroupSpanCount("QA", 1); + + await expectSingleViewPatch( + page, + g.view.id, + { group_by_layout: "section" }, + () => grid.selectGroupLayout("Sections"), + ); + await grid.expectGroupByBanner("A", 2); + await grid.expectGroupByBanner("B", 1); + await grid.expectGroupByBanner("Designer", 1); + await grid.expectGroupByBanner("Developer", 1); + await grid.expectGroupByBanner("QA", 1); + await expect(page.locator(".grid-view__group-span")).toHaveCount(0); + + await grid.goTo(g.database, g.table); + await grid.expectGroupByBanner("A", 2); + await grid.expectGroupByBanner("B", 1); + await grid.expectGroupByBanner("Designer", 1); + await grid.expectGroupByBanner("Developer", 1); + await grid.expectGroupByBanner("QA", 1); + await expect(page.locator(".grid-view__group-span")).toHaveCount(0); + }); + + test("9.6.3 five Columns groups keep a usable data pane at a narrow viewport", async ({ + page, + }) => { + const groupFields = Array.from({ length: 5 }, (_, index) => ({ + name: `Group ${index + 1}`, + type: "text" as const, + })); + const g = await setupGrid({ + dbName: "ResponsiveGroupLayoutDb", + fields: groupFields, + rows: [ + { + Name: "Responsive row", + ...Object.fromEntries( + groupFields.map(({ name }, index) => [name, `Value ${index + 1}`]), + ), + }, + ], + groupBys: groupFields.map(({ name }) => ({ + fieldName: name, + order: "ASC", + })), + }); + await patchView(g.user, g.view, { group_by_layout: "column" }); + await page.setViewportSize({ width: 1_024, height: 800 }); + + const unexpectedViewPatches: string[] = []; + const onViewMutation = (request: Request) => { + const url = new URL(request.url()); + if ( + request.method() === "PATCH" && + url.pathname.startsWith("/api/database/views/") + ) { + unexpectedViewPatches.push(url.pathname); + } + }; + page.on("request", onViewMutation); + + const grid = new GridPage(page, g.user); + await grid.goTo(g.database, g.table); + + const gridRoot = page.locator(".grid-view"); + const rightSection = page.locator(".grid-view__right"); + const groupHeaders = page.locator( + ".grid-view__left .grid-view__head-group", + ); + const groupSpans = page.locator(".grid-view__left .grid-view__group-span"); + await expect(groupHeaders).toHaveCount(5); + await expect(groupSpans).toHaveCount(5); + await expect( + page.locator(".grid-view__head-group-width-handle"), + ).toHaveCount(0); + await expect( + rightSection.locator(".grid-field-text", { + hasText: /^\s*Responsive row\s*$/, + }), + ).toBeVisible(); + await expect(page.locator(".scrollbars__horizontal-wrapper")).toBeVisible(); + + await expect + .poll(async () => { + const [gridBox, rightBox, widths, spanWidths] = await Promise.all([ + gridRoot.boundingBox(), + rightSection.boundingBox(), + groupHeaders.evaluateAll((elements) => + elements.map((element) => element.getBoundingClientRect().width), + ), + groupSpans.evaluateAll((elements) => + elements.map((element) => element.getBoundingClientRect().width), + ), + ]); + return { + rightWidth: Math.round(rightBox?.width ?? 0), + widthsFit: + widths.length === 5 && + widths.every((width) => width >= 78 && width < 200), + contained: + !!gridBox && + !!rightBox && + rightBox.x + rightBox.width <= gridBox.x + gridBox.width + 1, + spansAligned: + spanWidths.length === widths.length && + spanWidths.every( + (width, index) => Math.abs(width - widths[index]) < 1, + ), + }; + }) + .toEqual({ + rightWidth: 300, + widthsFit: true, + contained: true, + spansAligned: true, + }); + + // Responsive widths are display-only: the configured 200px widths return when + // space is available, without another view mutation. + await page.setViewportSize({ width: 1_920, height: 1_080 }); + await expect + .poll(() => + groupHeaders.evaluateAll((elements) => + elements.map((element) => + Math.round(element.getBoundingClientRect().width), + ), + ), + ) + .toEqual([200, 200, 200, 200, 200]); + await expect( + page.locator(".grid-view__head-group-width-handle"), + ).toHaveCount(5); + await page.waitForTimeout(100); + expect(unexpectedViewPatches).toEqual([]); + page.off("request", onViewMutation); + }); + + test("9.6.4 a group column can be resized and its width persists", async ({ + page, + }) => { + await page.setViewportSize({ width: 1_920, height: 1_080 }); + const { g, grid } = await setupGroupLayoutGrid(page); + await expectSingleViewPatch( + page, + g.view.id, + { group_by_layout: "column" }, + () => grid.selectGroupLayout("Columns"), + ); + + const header = page + .locator(".grid-view__left .grid-view__head-group") + .first(); + const span = grid.groupSpanByValue("A"); + const handle = page.locator(".grid-view__head-group-width-handle").first(); + await expect(header).toHaveCSS("width", "200px"); + await expect(span).toHaveCSS("width", "200px"); + await expect(handle).toBeVisible(); + + const matchingRequests: Request[] = []; + const isGroupWidthPatch = (request: Request) => { + const url = new URL(request.url()); + return ( + request.method() === "PATCH" && + /^\/api\/database\/views\/group_by\/\d+\/$/.test(url.pathname) + ); + }; + const onRequest = (request: Request) => { + if (isGroupWidthPatch(request)) { + matchingRequests.push(request); + } + }; + page.on("request", onRequest); + try { + const responsePromise = page.waitForResponse((response) => + isGroupWidthPatch(response.request()), + ); + const box = await handle.boundingBox(); + if (box === null) { + throw new Error("Expected the group width handle to have a box."); + } + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.down(); + await page.mouse.move( + box.x + box.width / 2 + 40, + box.y + box.height / 2, + { steps: 5 }, + ); + await page.mouse.up(); + + const response = await responsePromise; + expect(response.ok()).toBe(true); + expect(response.request().postDataJSON()).toEqual({ width: 240 }); + await page.waitForTimeout(100); + expect(matchingRequests).toHaveLength(1); + } finally { + page.off("request", onRequest); + } + + await expect(header).toHaveCSS("width", "240px"); + await expect(span).toHaveCSS("width", "240px"); + await grid.goTo(g.database, g.table); + await expect(header).toHaveCSS("width", "240px"); + await expect(grid.groupSpanByValue("A")).toHaveCSS("width", "240px"); + }); + + test("9.6.5 a row can be dragged within a group with the Columns offset and persists", async ({ + page, + }) => { + await page.setViewportSize({ width: 1_920, height: 1_080 }); + const { g, grid } = await setupGroupLayoutGrid(page); + await expectSingleViewPatch( + page, + g.view.id, + { group_by_layout: "column" }, + () => grid.selectGroupLayout("Columns"), + ); + await grid.expectPrimaryText(0, "Alice"); + await grid.expectPrimaryText(1, "Ada"); + + await grid.hoverRow(1); + const sourceHandle = grid.rowDragHandleAt(1); + await expect(sourceHandle).toBeVisible(); + const [sourceBox, targetBox] = await Promise.all([ + sourceHandle.boundingBox(), + grid.leftRowAt(0).boundingBox(), + ]); + if (sourceBox === null || targetBox === null) { + throw new Error( + "Expected the source handle and target row to have boxes.", + ); + } + + const movedRowId = g.rowIds[1]; + const movePath = `/api/database/rows/table/${g.table.id}/${movedRowId}/move/`; + const moveResponse = page.waitForResponse((response) => { + const url = new URL(response.url()); + return ( + response.request().method() === "PATCH" && url.pathname === movePath + ); + }); + + await page.mouse.move( + sourceBox.x + sourceBox.width / 2, + sourceBox.y + sourceBox.height / 2, + ); + await page.mouse.down(); + const dragging = page.locator(".grid-view__row-dragging-container"); + await expect(dragging).toBeVisible(); + await expect(dragging).toHaveCSS("left", "200px"); + await page.mouse.move(sourceBox.x + sourceBox.width / 2, targetBox.y + 4, { + steps: 5, + }); + await expect(page.locator(".grid-view__row-target")).toBeVisible(); + await page.mouse.up(); + + const response = await moveResponse; + expect(response.ok()).toBe(true); + const requestUrl = new URL(response.url()); + expect(requestUrl.searchParams.get("before_id")).toBe(String(g.rowIds[0])); + expect(requestUrl.searchParams.get("view")).toBe(String(g.view.id)); + await grid.expectPrimaryText(0, "Ada"); + await grid.expectPrimaryText(1, "Alice"); + await grid.expectGroupSpanCount("A", 2); + + await grid.goTo(g.database, g.table); + await grid.expectPrimaryText(0, "Ada"); + await grid.expectPrimaryText(1, "Alice"); + await grid.expectGroupSpanCount("A", 2); + }); +}); + test.describe("15.1 Public shared grid", () => { let g: Setup; + let groupedColumns: Setup; test.beforeAll(async () => { g = await setupGrid({ @@ -973,6 +1363,21 @@ test.describe("15.1 Public shared grid", () => { ], }); await patchView(g.user, g.view, { public: true }); + + groupedColumns = await setupGrid({ + dbName: "PublicGroupedColumnsGridDb", + fields: [{ name: "Team", type: "text" }], + rows: [ + { Name: "Alice", Team: "A" }, + { Name: "Ada", Team: "A" }, + { Name: "Bob", Team: "B" }, + ], + groupBys: [{ fieldName: "Team", order: "ASC" }], + }); + await patchView(groupedColumns.user, groupedColumns.view, { + public: true, + group_by_layout: "column", + }); }); test("15.1.1 public shared grid renders rows without edit controls", async ({ @@ -1014,4 +1419,50 @@ test.describe("15.1 Public shared grid", () => { }), ).toHaveCount(0); }); + + test("15.1.2 public shared Columns grid renders grouped rows without edit controls", async ({ + page, + }) => { + const slug = groupedColumns.view.slug; + if (!slug) { + throw new Error("Expected grouped grid view to include a public slug."); + } + + await page.goto( + `${baserowConfig.PUBLIC_WEB_FRONTEND_URL}/public/grid/${slug}`, + { + waitUntil: "domcontentloaded", + }, + ); + + await expect(page.locator(".grid-view__right")).toBeVisible({ + timeout: 15_000, + }); + await expect( + page.locator(".grid-view__left .grid-view__head-group"), + ).toHaveCount(1); + await expect(page.locator(".grid-view__group-by-banner")).toHaveCount(0); + + const grid = new GridPage(page, groupedColumns.user); + await grid.expectGroupSpanCount("A", 2); + await grid.expectGroupSpanCount("B", 1); + for (const name of ["Alice", "Ada", "Bob"]) { + await expect( + page.locator(".grid-view__left .grid-field-text", { + hasText: new RegExp(`^\\s*${name}\\s*$`), + }), + ).toBeVisible(); + } + + await expect(page.locator(".grid-view__add-row")).toHaveCount(0); + await page + .locator(".grid-view__left .grid-view__row") + .first() + .click({ button: "right" }); + await expect( + page.locator(".context__menu:visible .context__menu-item-link", { + hasText: /Delete row|Insert row|Duplicate row/, + }), + ).toHaveCount(0); + }); }); diff --git a/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py b/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py index 83e9b132ae..ae8a1e4fa8 100644 --- a/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py +++ b/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py @@ -1,11 +1,43 @@ from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_FIELDS from baserow.core.ai_provider.registries import AIProviderModelFeatureType from baserow.core.generative_ai.registries import generative_ai_model_type_registry +from baserow.core.models import Workspace +from baserow_premium.fields.models import AIField class AIFieldsAIProviderModelFeatureType(AIProviderModelFeatureType): type = AI_PROVIDER_FEATURE_AI_FIELDS + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the AI fields selecting one provider model. + + A field is skipped as soon as any ancestor is trashed, because the whole + subtree disappears with it. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier the fields persist. + :param workspace: The workspace to narrow to, or None for the instance + scope, which counts every workspace. + :return: The number of fields referencing the model. + """ + + queryset = AIField.objects.filter( + ai_generative_ai_type=provider_type, + ai_generative_ai_model=model_identifier, + table__trashed=False, + table__database__trashed=False, + table__database__workspace__trashed=False, + ) + if workspace is not None: + queryset = queryset.filter(table__database__workspace=workspace) + return queryset.count() + def get_workspace_availability(self, workspace, state=None) -> dict: models = generative_ai_model_type_registry.get_enabled_models_per_type( workspace, feature_type=self.type, state=state diff --git a/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py b/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py new file mode 100644 index 0000000000..f18ea5a036 --- /dev/null +++ b/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py @@ -0,0 +1,91 @@ +import pytest + +from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_FIELDS +from baserow.core.ai_provider.registries import ( + ai_provider_model_feature_type_registry, +) + + +def get_feature_type(): + return ai_provider_model_feature_type_registry.get(AI_PROVIDER_FEATURE_AI_FIELDS) + + +@pytest.mark.django_db +def test_ai_fields_count_model_references(premium_data_fixture): + table = premium_data_fixture.create_database_table() + workspace = table.database.workspace + other_table = premium_data_fixture.create_database_table() + other_workspace = other_table.database.workspace + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="another-model", + ) + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="anthropic", + ai_generative_ai_model="field-model", + ) + trashed_field = premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + trashed_field.trashed = True + trashed_field.save() + premium_data_fixture.create_ai_field( + table=other_table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + feature_type = get_feature_type() + + assert feature_type.count_model_references("openai", "field-model", workspace) == 1 + assert ( + feature_type.count_model_references("openai", "field-model", other_workspace) + == 1 + ) + assert feature_type.count_model_references("openai", "field-model", None) == 2 + assert feature_type.count_model_references("openai", "unknown-model", None) == 0 + assert feature_type.count_model_references("mistral", "field-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_fields_count_model_references_ignores_trashed_ancestors( + premium_data_fixture, +): + table = premium_data_fixture.create_database_table() + database = table.database + workspace = database.workspace + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + feature_type = get_feature_type() + assert feature_type.count_model_references("openai", "field-model", None) == 1 + + table.trashed = True + table.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 + assert feature_type.count_model_references("openai", "field-model", workspace) == 0 + + table.trashed = False + table.save() + database.trashed = True + database.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 + + database.trashed = False + database.save() + workspace.trashed = True + workspace.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 diff --git a/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js b/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js index 1abf254651..e7f070314a 100644 --- a/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js +++ b/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js @@ -87,7 +87,7 @@ describe('FieldAISubForm component', () => { expect(wrapper.vm.isFormValid()).toBe(true) }) - test('a disabled model is removed before it can be selected', async () => { + test('a disabled model stays visible but cannot be selected', async () => { const wrapper = await mountComponent( { formula: "'hello'", @@ -100,10 +100,15 @@ describe('FieldAISubForm component', () => { } ) - const optionNames = wrapper - .findAll('.select__item-name-text') - .map((item) => item.text()) - expect(optionNames).not.toContain('disabled-model') + const disabledOption = wrapper + .findAll('.select__item') + .find( + (option) => + option.find('.select__item-name-text').text() === 'disabled-model' + ) + expect(disabledOption).toBeTruthy() + expect(disabledOption.classes()).toContain('disabled') + wrapper.vm.submit() await wrapper.vm.$nextTick() expect(wrapper.emitted('submitted')).toBeUndefined() diff --git a/web-frontend/modules/core/assets/scss/components/group_bys.scss b/web-frontend/modules/core/assets/scss/components/group_bys.scss index 191e64edc8..0bade580ca 100644 --- a/web-frontend/modules/core/assets/scss/components/group_bys.scss +++ b/web-frontend/modules/core/assets/scss/components/group_bys.scss @@ -183,3 +183,23 @@ @include flex-align-items(20px); } + +.group-bys__layout { + justify-content: space-between; + padding: 10px 14px; + border-bottom: 1px solid $palette-neutral-200; + flex-shrink: 0; + + @include flex-align-items(12px); +} + +.group-bys__layout-label { + font-size: 13px; + font-weight: 600; + color: $palette-neutral-900; +} + +.group-bys__layout .segment-control__button { + flex: none; + padding: 0 14px; +} diff --git a/web-frontend/modules/core/assets/scss/components/views/grid.scss b/web-frontend/modules/core/assets/scss/components/views/grid.scss index 71ec2889a6..5543593a2a 100644 --- a/web-frontend/modules/core/assets/scss/components/views/grid.scss +++ b/web-frontend/modules/core/assets/scss/components/views/grid.scss @@ -483,6 +483,50 @@ color: $palette-neutral-900; } +.grid-view__group-columns { + @include absolute(0, auto, auto, 0); + + z-index: 2; + background-color: $palette-neutral-50; +} + +.grid-view__group-span { + position: absolute; + box-sizing: border-box; + border-bottom: 1px solid $palette-neutral-400; + background-color: $palette-neutral-50; +} + +.grid-view__group-cell--sticky { + position: sticky; + top: 0; + overflow: hidden; +} + +.grid-view__group-value { + @extend %ellipsis; + + min-width: 0; + flex: 1; + font-size: 13px; +} + +.grid-view__group-value-empty { + color: $palette-neutral-700; +} + +.grid-view__group-count { + @include rounded($rounded); + + flex: none; + margin-left: 8px; + padding: 0 5px; + font-size: 11px; + line-height: 18px; + font-weight: 600; + background-color: $palette-neutral-200; +} + .grid-view__group-by-banner { position: absolute; left: 0; @@ -789,6 +833,16 @@ background: $palette-neutral-0; } +.grid-view__group-by-rows-row--group-end::after { + @include absolute(auto, 0, 0, 0); + + content: ''; + z-index: 1; + height: 1px; + background-color: $palette-neutral-400; + pointer-events: none; +} + .grid-view__right .grid-view__group-by-rows-row .grid-view__column:last-child { border-right-color: $palette-neutral-400; } diff --git a/web-frontend/modules/core/components/HorizontalResize.vue b/web-frontend/modules/core/components/HorizontalResize.vue index 217825f253..58e0859eff 100644 --- a/web-frontend/modules/core/components/HorizontalResize.vue +++ b/web-frontend/modules/core/components/HorizontalResize.vue @@ -31,7 +31,7 @@ export default { default: false, }, }, - emits: ['move', 'update'], + emits: ['move', 'update', 'dragging'], data() { return { dragging: false, @@ -55,6 +55,7 @@ export default { window.addEventListener('mousemove', this.$el.moveEvent) window.addEventListener('mouseup', this.$el.upEvent) document.body.classList.add('resizing-horizontal') + this.$emit('dragging', true) }, move(event) { event.preventDefault() @@ -79,6 +80,7 @@ export default { window.removeEventListener('mousemove', this.$el.moveEvent) window.removeEventListener('mouseup', this.$el.upEvent) document.body.classList.remove('resizing-horizontal') + this.$emit('dragging', false) if (newWidth === this.startWidth) { return diff --git a/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue b/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue index 264ff2bb14..299337838b 100644 --- a/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue +++ b/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue @@ -1,5 +1,10 @@ diff --git a/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue b/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue new file mode 100644 index 0000000000..1c70a6346e --- /dev/null +++ b/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue @@ -0,0 +1,76 @@ + + + diff --git a/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue b/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue index a1a6c0cb81..7140ba4521 100644 --- a/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue +++ b/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue @@ -19,10 +19,10 @@
@@ -130,6 +130,11 @@ export default { required: false, default: () => false, }, + groupByWidths: { + type: Array, + required: false, + default: () => [], + }, includeAddField: { type: Boolean, required: false, @@ -198,6 +203,9 @@ export default { const field = allFieldsInTable.find((f) => f.id === groupBy.field) return field }, + getGroupByWidth(groupBy, index) { + return this.groupByWidths[index] ?? groupBy.width + }, }, } diff --git a/web-frontend/modules/database/components/view/grid/GridViewSection.vue b/web-frontend/modules/database/components/view/grid/GridViewSection.vue index 1f33eab688..9725d0a739 100644 --- a/web-frontend/modules/database/components/view/grid/GridViewSection.vue +++ b/web-frontend/modules/database/components/view/grid/GridViewSection.vue @@ -18,16 +18,19 @@ class="grid-view__group-by-divider" :style="{ left: left + 'px' }" >
- +
+
+
{{ $t('gridView.rowCount', { count }) }}
@@ -198,6 +214,7 @@ import GridViewHead from '@baserow/modules/database/components/view/grid/GridVie import GridViewPlaceholder from '@baserow/modules/database/components/view/grid/GridViewPlaceholder' import GridViewRows from '@baserow/modules/database/components/view/grid/GridViewRows' import GridViewGroupByRows from '@baserow/modules/database/components/view/grid/GridViewGroupByRows' +import GridViewGroupByColumns from '@baserow/modules/database/components/view/grid/GridViewGroupByColumns' import GridViewRowAdd from '@baserow/modules/database/components/view/grid/GridViewRowAdd' import gridViewHelpers from '@baserow/modules/database/mixins/gridViewHelpers' import GridViewFieldFooter from '@baserow/modules/database/components/view/grid/GridViewFieldFooter' @@ -211,6 +228,7 @@ export default { GridViewPlaceholder, GridViewRows, GridViewGroupByRows, + GridViewGroupByColumns, GridViewRowAdd, GridViewFieldFooter, }, @@ -254,6 +272,11 @@ export default { required: false, default: () => false, }, + groupByWidths: { + type: Array, + required: false, + default: () => [], + }, includeAddField: { type: Boolean, required: false, @@ -321,6 +344,9 @@ export default { fieldsLeftOffset: 0, resizeObserver: null, horizontalScrollEvent: null, + // Keep the active handle mounted until mouseup can persist its width, even + // when dragging past the available space activates responsive fitting. + resizingGroupWidth: false, } }, computed: { @@ -338,7 +364,7 @@ export default { width += this.gridViewRowDetailsWidth } if (this.includeGroupBy) { - width += this.activeGroupByWidth + width += this.groupColumnsWidth } // The add button has a width of 100 and we reserve 100 at the right side. @@ -348,20 +374,38 @@ export default { return width }, + renderedGroupByWidths() { + return this.activeGroupBys.map( + (groupBy, index) => this.groupByWidths[index] ?? groupBy.width + ) + }, + groupByWidthsAreResponsivelyFitted() { + const configuredWidth = this.activeGroupBys.reduce( + (total, groupBy) => + total + Math.max(groupBy.width, this.GRID_VIEW_MIN_FIELD_WIDTH), + 0 + ) + return this.groupColumnsWidth < configuredWidth - 0.01 + }, + groupColumnsWidth() { + if (!this.includeGroupBy) { + return 0 + } + return this.renderedGroupByWidths.reduce( + (total, width) => total + width, + 0 + ) + }, groupByDividers() { if (!this.includeGroupBy) { return [] } let last = 0 - const dividers = this.activeGroupBys - .filter((groupBy, index) => index < this.activeGroupBys.length - 1) - .map((groupBy) => { - last += groupBy.width - return { groupBy, left: last } - }) - - return dividers + return this.activeGroupBys.map((groupBy, index) => { + last += this.renderedGroupByWidths[index] + return { groupBy, left: last } + }) }, useGroupByRows() { return this.activeGroupBys.length > 0 diff --git a/web-frontend/modules/database/locales/en.json b/web-frontend/modules/database/locales/en.json index e8314e98f5..b66816cbc1 100644 --- a/web-frontend/modules/database/locales/en.json +++ b/web-frontend/modules/database/locales/en.json @@ -601,6 +601,7 @@ "viewSettingsRowHeightSize": "row height", "viewSettingsFrozenColumnCount": "frozen columns", "viewSettingsRowIdentifierType": "row identifier", + "viewSettingsGroupByLayout": "group layout", "filters": "Filters", "sorts": "Sorts", "groupBys": "Groups", @@ -643,6 +644,9 @@ "maxGroupBysReached": "You can group by up to {count} fields.", "collapseAllGroups": "Collapse all", "expandAllGroups": "Expand all", + "layout": "Group layout", + "layoutSection": "Sections", + "layoutColumn": "Columns", "hiddenFieldWarning": "One or more group bys reference hidden fields that won't be visible to editors and lower roles." }, "viewGroupBy": { diff --git a/web-frontend/modules/database/mixins/gridViewGroupByValue.js b/web-frontend/modules/database/mixins/gridViewGroupByValue.js new file mode 100644 index 0000000000..a008db7f10 --- /dev/null +++ b/web-frontend/modules/database/mixins/gridViewGroupByValue.js @@ -0,0 +1,83 @@ +// Host components provide the `groupByField`, `groupPath` and `groupDisplay` computeds. +export default { + computed: { + fieldType() { + if (!this.groupByField) { + return null + } + return this.$registry.get('field', this.groupByField.type) + }, + groupValue() { + const field = this.groupByField + if (!field) { + return null + } + return this.groupPath[`field_${field.id}`] + }, + displayValue() { + const field = this.groupByField + const display = this.groupDisplay + if (!field || !display) { + return undefined + } + const key = `field_${field.id}` + return key in display ? display[key] : undefined + }, + rowValueForGroup() { + const field = this.groupByField + if (!field || !this.fieldType) { + return null + } + // Reference fields (selects, links, collaborators) only render from `display`. + if (this.displayValue !== undefined) { + return this.displayValue + } + return this.fieldType.getRowValueFromGroupValue(field, this.groupValue) + }, + isEmptyValue() { + const value = this.rowValueForGroup + if (Array.isArray(value)) { + return value.length === 0 + } + return value === null || value === undefined || value === '' + }, + groupByComponent() { + if (this.isEmptyValue || !this.groupByField || !this.fieldType) { + return null + } + if (typeof this.fieldType.getGroupByComponent !== 'function') { + return null + } + return this.fieldType.getGroupByComponent(this.groupByField) + }, + fallbackValueText() { + const value = this.rowValueForGroup + if (value === null || value === undefined) { + return '' + } + if (typeof value === 'boolean') { + return value ? 'true' : 'false' + } + if (this.fieldType?.toHumanReadableString) { + try { + const text = this.fieldType.toHumanReadableString( + this.groupByField, + value + ) + if (typeof text === 'string') { + return text + } + } catch (_) { + // Fall back to the generic object/string rendering below. + } + } + if (typeof value === 'object') { + if ('value' in value) { + return value.value + } + return JSON.stringify(value) + } + return String(value) + }, + }, +} diff --git a/web-frontend/modules/database/store/view/grid.js b/web-frontend/modules/database/store/view/grid.js index 5615d14be0..075d84dc03 100644 --- a/web-frontend/modules/database/store/view/grid.js +++ b/web-frontend/modules/database/store/view/grid.js @@ -44,6 +44,8 @@ import { renderViewport, visibleGroupPagesInViewport, visibleSectionsInViewport, + GROUP_BY_LAYOUT_SECTION, + GROUP_BY_LAYOUT_COLUMN, } from '@baserow/modules/database/utils/gridGroupByRender' import { getGroupByCollapseAllState, @@ -92,6 +94,11 @@ const REFRESH_ROW_DELAY_MS = 1000 // geometry is unaffected. High enough that a viewport's working set never evicts, so // collapse then re-expand never refetches. const GROUP_BY_MAX_RETAINED_SECTIONS = 200 +// Loading a sparse group page can change the estimated height of earlier unloaded +// pages, moving the same viewport onto another page. Refine until the geometry +// converges, but cap the number of sequential network round trips for corrupt or +// adversarial page data. The per-request seen sets below also prevent duplicate work. +const GROUP_BY_MAX_VIEWPORT_REFINEMENT_PASSES = 100 const DEFAULT_FLAT_VIEW = { group_bys: [], sortings: [], @@ -240,6 +247,8 @@ function getGroupByLayoutFromState(state) { state.rowHeight, state.bufferRequestSize, state.activeGroupBys, + state.groupByLayout, + state.count, ] const cached = groupByLayoutCacheByState.get(state) if ( @@ -258,6 +267,8 @@ function getGroupByLayoutFromState(state) { fields: getGroupByFieldRefsFromState(state), rowHeight: state.rowHeight, pageSize: state.bufferRequestSize, + layout: state.groupByLayout, + rootRowCount: state.count, }) groupByLayoutCacheByState.set(state, { key: cacheKey, value: layout }) @@ -535,6 +546,8 @@ function getGroupBySnapshotLayout(snapshot, activeGroupBys, fields, state) { fields: groupByFields, rowHeight: state.rowHeight, pageSize: state.bufferRequestSize, + layout: state.groupByLayout, + rootRowCount: state.count, }) } @@ -965,6 +978,7 @@ export const state = () => ({ count: 0, // The height of a single row. rowHeight: 33, + groupByLayout: GROUP_BY_LAYOUT_SECTION, // The distance to the top in pixels the visible rows should have. rowsTop: 0, // The amount of rows that must be visible above and under the middle row. @@ -1718,6 +1732,12 @@ export const mutations = { UPDATE_ROW_HEIGHT(state, value) { state.rowHeight = value }, + SET_GROUP_BY_LAYOUT(state, value) { + state.groupByLayout = + value === GROUP_BY_LAYOUT_COLUMN + ? GROUP_BY_LAYOUT_COLUMN + : GROUP_BY_LAYOUT_SECTION + }, ADD_CHECKBOX_SELECTED_ROW(state, rowId) { if (!state.checkboxSelectedRows.includes(rowId)) { state.checkboxSelectedRows.push(rowId) @@ -2167,7 +2187,11 @@ export const actions = { fields ) const viewport = getGroupByViewport(getters, scrollTop) - const useDepthPages = shouldUseGroupByDepthPages(state.groupBy) + // Columns always render every level expanded, even when the saved Section + // collapse state is collapse-all. Its fetch strategy must match that layout. + const forceExpandedLayout = getters.isGroupByColumnLayout + const useDepthPages = + !forceExpandedLayout && shouldUseGroupByDepthPages(state.groupBy) let layout = getters.getGroupByLayout if ( @@ -2179,7 +2203,8 @@ export const actions = { view, fields, adhocFiltering: getters.getAdhocFiltering, - includeDescendants: state.groupBy.collapse?.mode === 'expand', + includeDescendants: + forceExpandedLayout || state.groupBy.collapse?.mode === 'expand', descendantLimit: getters.getBufferRequestSize, descendantRowBudget: getGroupByDescendantRowBudget(getters, scrollTop), signal, @@ -2192,7 +2217,11 @@ export const actions = { const seenPageRequests = new Set() const seenDepthRequests = new Set() - for (let depth = 0; depth < groupByFields.length; depth += 1) { + for ( + let pass = 0; + pass < GROUP_BY_MAX_VIEWPORT_REFINEMENT_PASSES; + pass += 1 + ) { if (useDepthPages) { const pageToFetch = getVisibleGroupDepthPageToFetch({ layout, @@ -2214,6 +2243,9 @@ export const actions = { adhocFiltering: getters.getAdhocFiltering, signal, }) + if (isGroupByRequestStale(getters, state, groupByGeneration, signal)) { + return [] + } layout = getters.getGroupByLayout continue } @@ -2347,6 +2379,9 @@ export const actions = { { commit, dispatch, getters }, { path, view, fields } ) { + if (getters.isGroupByColumnLayout) { + return + } const groupByFields = getGroupByFieldsFromActiveGroupBys( getters.getActiveGroupBys, fields @@ -2368,6 +2403,9 @@ export const actions = { { commit, dispatch, getters }, { view, fields, collapse } ) { + if (getters.isGroupByColumnLayout) { + return + } commit('SET_GROUP_BY_COLLAPSE', getGroupByCollapseAllState(collapse)) commit('CLEAR_AREA_SELECTION') const scrollTop = getClampedGroupByScrollTop(getters) @@ -3080,6 +3118,13 @@ export const actions = { groupByCollapse = getGroupByCollapseAllState(false) } + // Keep the persisted collapse state in the snapshot so switching back to + // Sections restores it. Only the fetch decisions are forced to expand for Columns. + const forceExpandedLayout = getters.isGroupByColumnLayout + const effectiveExpandAll = + forceExpandedLayout || + (groupByCollapse.mode === 'expand' && groupByCollapse.paths.length === 0) + commit('INVALIDATE_GROUP_BY_REQUESTS') const groupByGeneration = state.groupBy.generation || 0 const snapshot = { @@ -3093,9 +3138,7 @@ export const actions = { // first page is independent of the new tree and can be fetched in parallel with the // skeleton. Mixed/collapsed states tie their offsets to the tree, so they can't. const parallelExpandRows = - scrollTop === 0 && - groupByCollapse.mode === 'expand' && - groupByCollapse.paths.length === 0 + scrollTop === 0 && effectiveExpandAll ? GridService($client).fetchRows({ gridId, offset: 0, @@ -3124,7 +3167,8 @@ export const actions = { filters: getFilters(view, getters.getAdhocFiltering), offset: 0, limit: getters.getBufferRequestSize, - includeDescendants: groupByCollapse.mode === 'expand', + includeDescendants: + forceExpandedLayout || groupByCollapse.mode === 'expand', descendantLimit: getters.getBufferRequestSize, descendantRowBudget: getGroupByDescendantRowBudget(getters), groupBy: getGroupBy(rootGetters, gridId, getters.getAdhocGrouping), @@ -3949,7 +3993,7 @@ export const actions = { const startRowIndex = getters.getMultiSelectStartRowIndex const startFieldIndex = getters.getMultiSelectStartFieldIndex - const maxRowIndex = getters.getRowsLength + getters.getBufferStartIndex - 1 + const maxRowIndex = getters.getSelectionMaxRowIndex const maxFieldIndex = getters.getNumberOfVisibleFields - 1 if (headRowIndex > maxRowIndex || headFieldIndex > maxFieldIndex) { @@ -5284,7 +5328,7 @@ export const actions = { } if ( - rowIndex > getters.getRowsLength + getters.getBufferStartIndex - 1 || + rowIndex > getters.getSelectionMaxRowIndex || fieldIndex > getters.getNumberOfVisibleFields - 1 ) { return @@ -6419,6 +6463,9 @@ export const actions = { setRowHeight({ commit, dispatch, getters }, value) { commit('UPDATE_ROW_HEIGHT', value) }, + setGroupByLayout({ commit }, value) { + commit('SET_GROUP_BY_LAYOUT', value) + }, toggleCheckboxRowSelection({ commit, dispatch, state, getters }, { row }) { const { $registry, $client, $i18n, $config } = this const rowId = row.id @@ -6497,6 +6544,14 @@ export const getters = { } return state.rows.length }, + getSelectionMaxRowIndex(state, getters) { + // Columns use absolute row offsets, including unloaded group pages. Sections + // number only the expanded sections, while flat grids use their row buffer. + if (getters.isGroupByMode && getters.isGroupByColumnLayout) { + return state.count - 1 + } + return getters.getRowsLength + getters.getBufferStartIndex - 1 + }, getPlaceholderHeight(state) { return state.count * state.rowHeight }, @@ -6780,6 +6835,12 @@ export const getters = { getGroupByCollapse(state) { return state.groupBy.collapse }, + getGroupByLayoutMode(state) { + return state.groupByLayout + }, + isGroupByColumnLayout(state) { + return state.groupByLayout === GROUP_BY_LAYOUT_COLUMN + }, // Takes no field arg so the memoization in getGroupByLayoutFromState holds: a // per-call fields array would be a new reference every time and defeat the cache. getGroupByLayout(state) { diff --git a/web-frontend/modules/database/utils/gridGroupByRender.js b/web-frontend/modules/database/utils/gridGroupByRender.js index 929b9f11eb..57e866b084 100644 --- a/web-frontend/modules/database/utils/gridGroupByRender.js +++ b/web-frontend/modules/database/utils/gridGroupByRender.js @@ -5,6 +5,102 @@ export const GROUP_GAP = 8 // matching the fixed-height ungrouped `GridViewRowAdd`. export const ADD_ROW_HEIGHT = ROW_HEIGHT +export const GROUP_BY_LAYOUT_SECTION = 'section' +export const GROUP_BY_LAYOUT_COLUMN = 'column' + +const SECTION_GEOMETRY = Object.freeze({ + layout: GROUP_BY_LAYOUT_SECTION, + headerHeight: HEADER_HEIGHT, + groupGap: GROUP_GAP, + addRowHeight: ADD_ROW_HEIGHT, + addRowPerGroup: true, + unloadedGroupHeight: HEADER_HEIGHT, +}) + +const EXPAND_ALL = Object.freeze({ mode: 'expand', paths: [] }) + +export function getLayoutGeometry(layout, rowHeight = ROW_HEIGHT) { + if (layout !== GROUP_BY_LAYOUT_COLUMN) { + return SECTION_GEOMETRY + } + return { + layout: GROUP_BY_LAYOUT_COLUMN, + headerHeight: 0, + groupGap: 0, + addRowHeight: ADD_ROW_HEIGHT, + addRowPerGroup: false, + // A row per unloaded group keeps the placeholder tall enough to reach the viewport. + unloadedGroupHeight: rowHeight, + } +} + +function layoutGeometryOf(layout) { + return layout?.geometry || SECTION_GEOMETRY +} + +function pushGroupHeader(items, geometry, rowHeight, node) { + if (geometry.headerHeight === 0) { + items.push({ + type: 'groupSpan', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + y: node.y, + height: node.rowCount * rowHeight, + }) + return node.y + } + items.push({ + type: 'header', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + aggregationRowCount: node.aggregationRowCount, + y: node.y, + height: geometry.headerHeight, + collapsed: node.collapsed, + gapAbove: node.gapAbove, + aggregations: node.aggregations, + }) + return node.y + geometry.headerHeight +} + +function pushGroupAddRow(items, geometry, node) { + if (!geometry.addRowPerGroup) { + return node.y + } + items.push({ + type: 'addRow', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + y: node.y, + height: geometry.addRowHeight, + }) + return node.y + geometry.addRowHeight +} + +function pushTrailingAddRow(items, geometry, y, rootPageLoaded) { + // Without per-group add-row lines the grid must always end with one. + const needed = geometry.addRowPerGroup + ? rootPageLoaded && items.length === 0 + : rootPageLoaded || items.length > 0 + if (!needed) { + return y + } + items.push({ + type: 'addRow', + depth: 0, + path: {}, + y, + height: geometry.addRowHeight, + }) + return y + geometry.addRowHeight +} + const GROUP_BANNER_DEPTH_INDENT_PX = 24 const GROUP_BANNER_BASE_GUTTER = 12 const GROUP_BANNER_CHEVRON_WIDTH = 24 @@ -104,9 +200,23 @@ export function buildLayout({ fields, rowHeight = ROW_HEIGHT, pageSize = GROUP_PAGE_SIZE, + layout = GROUP_BY_LAYOUT_SECTION, + rootRowCount = null, }) { + const geometry = getLayoutGeometry(layout, rowHeight) + const effectiveCollapse = + geometry.layout === GROUP_BY_LAYOUT_COLUMN ? EXPAND_ALL : collapse + if (pages !== null) { - return buildPagedLayout({ pages, collapse, fields, rowHeight, pageSize }) + return buildPagedLayout({ + pages, + collapse: effectiveCollapse, + fields, + rowHeight, + pageSize, + geometry, + rootRowCount, + }) } const items = [] @@ -114,11 +224,11 @@ export function buildLayout({ let visibleRowCount = 0 if (!nodes || nodes.length === 0 || !fields || fields.length === 0) { - return { items, totalHeight: 0, totalRowCount: 0 } + return { items, totalHeight: 0, totalRowCount: 0, geometry } } const maxDepth = fields.length - 1 - const exceptionKeys = collapseExceptionKeys(collapse, fields) + const exceptionKeys = collapseExceptionKeys(effectiveCollapse, fields) let skipDescendantsAtDepth = -1 let lastPlacedDepth = null @@ -139,13 +249,13 @@ export function buildLayout({ // its parent's banner (a first child) sits flush against it. const gapAbove = lastPlacedDepth !== null && lastPlacedDepth >= node.depth if (gapAbove) { - y += GROUP_GAP + y += geometry.groupGap } lastPlacedDepth = node.depth const collapsed = pathCollapsedAgainst( node.path, - collapse.mode, + effectiveCollapse.mode, exceptionKeys, fields ) @@ -153,20 +263,17 @@ export function buildLayout({ const aggregationRowCount = node.aggregationRowCount ?? node.aggregation_row_count ?? rowCount - items.push({ - type: 'header', + y = pushGroupHeader(items, geometry, rowHeight, { depth: node.depth, path: node.path, display: node.display, rowCount, aggregationRowCount, y, - height: HEADER_HEIGHT, collapsed, gapAbove, aggregations: node.aggregations ?? null, }) - y += HEADER_HEIGHT if (collapsed) { skipDescendantsAtDepth = node.depth @@ -188,20 +295,19 @@ export function buildLayout({ }) y += sectionHeight visibleRowCount += rowCount - items.push({ - type: 'addRow', + y = pushGroupAddRow(items, geometry, { depth: node.depth, path: node.path, display: node.display, rowCount, y, - height: ADD_ROW_HEIGHT, }) - y += ADD_ROW_HEIGHT } } - return { items, totalHeight: y, totalRowCount: visibleRowCount } + y = pushTrailingAddRow(items, geometry, y, true) + + return { items, totalHeight: y, totalRowCount: visibleRowCount, geometry } } function getPage(pages, parentPath, fields) { @@ -214,38 +320,41 @@ function getSortedLoadedIndexes(page) { .sort((a, b) => a - b) } -function unloadedGroupRangeHeight(startIndex, endIndex) { +function unloadedGroupRangeHeight( + startIndex, + endIndex, + geometry, + rowSlotCount = null +) { const count = Math.max(0, endIndex - startIndex) if (count === 0) { return 0 } + if (geometry.layout === GROUP_BY_LAYOUT_COLUMN && rowSlotCount !== null) { + return Math.max(0, rowSlotCount) * geometry.unloadedGroupHeight + } const gaps = count - (startIndex === 0 ? 1 : 0) - return count * HEADER_HEIGHT + gaps * GROUP_GAP + return count * geometry.unloadedGroupHeight + gaps * geometry.groupGap } -function placeholderStartIndexAtOffset(item, offset) { +function placeholderStartIndexAtOffset(item, offset, geometry) { const startIndex = item.siblingStartIndex const endIndex = item.siblingEndIndex const clampedOffset = Math.max(0, Math.min(offset, item.height)) + const slotHeight = geometry.unloadedGroupHeight + const stride = slotHeight + geometry.groupGap if (startIndex === 0) { - if (clampedOffset < HEADER_HEIGHT) { + if (clampedOffset < slotHeight) { return startIndex } return Math.min( endIndex, - startIndex + - 1 + - Math.floor( - (clampedOffset - HEADER_HEIGHT) / (HEADER_HEIGHT + GROUP_GAP) - ) + startIndex + 1 + Math.floor((clampedOffset - slotHeight) / stride) ) } - return Math.min( - endIndex, - startIndex + Math.floor(clampedOffset / (HEADER_HEIGHT + GROUP_GAP)) - ) + return Math.min(endIndex, startIndex + Math.floor(clampedOffset / stride)) } function pushUnloadedGroupPlaceholder({ @@ -256,8 +365,15 @@ function pushUnloadedGroupPlaceholder({ endIndex, globalStartIndex, y, + geometry, + rowSlotCount = null, }) { - const height = unloadedGroupRangeHeight(startIndex, endIndex) + const height = unloadedGroupRangeHeight( + startIndex, + endIndex, + geometry, + rowSlotCount + ) if (height <= 0) { return y } @@ -282,13 +398,15 @@ function buildPagedLayout({ fields, rowHeight = ROW_HEIGHT, pageSize = GROUP_PAGE_SIZE, + geometry = SECTION_GEOMETRY, + rootRowCount = null, }) { const items = [] let y = 0 let visibleRowCount = 0 if (!fields || fields.length === 0) { - return { items, totalHeight: 0, totalRowCount: 0 } + return { items, totalHeight: 0, totalRowCount: 0, geometry } } const maxDepth = fields.length - 1 @@ -307,7 +425,13 @@ function buildPagedLayout({ const advanceGlobalSiblingCount = (d, count) => globalSiblingCountByDepth.set(d, globalSiblingCount(d) + count) - const walkPage = (parentPath, depth, fallbackSiblingCount = 0) => { + const walkPage = ( + parentPath, + depth, + fallbackSiblingCount = 0, + parentRowCount = null, + parentRowOffset = null + ) => { const pageCacheKey = pathKey(parentPath, fields) if (visitedPageKeys.has(pageCacheKey)) { return @@ -318,7 +442,61 @@ function buildPagedLayout({ page?.totalSiblingCount ?? page?.total_sibling_count ?? fallbackSiblingCount + const pageY = y const loadedIndexes = getSortedLoadedIndexes(page) + const validLoadedIndexes = loadedIndexes.filter( + (loadedIndex) => loadedIndex >= 0 && loadedIndex < totalSiblingCount + ) + let remainingUnloadedSiblingCount = null + let remainingUnloadedRowCount = null + if ( + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + Number.isFinite(parentRowCount) + ) { + const loadedRowCount = validLoadedIndexes.reduce((total, loadedIndex) => { + const loadedNode = page?.nodes?.[loadedIndex] + return total + (loadedNode?.rowCount ?? loadedNode?.row_count ?? 0) + }, 0) + remainingUnloadedSiblingCount = Math.max( + 0, + totalSiblingCount - validLoadedIndexes.length + ) + // Server groups are non-empty, so reserve at least one row for each sibling + // when optimistic/stale counts momentarily disagree. Otherwise distribute the + // parent's rows not accounted for by loaded siblings across the missing ranges. + remainingUnloadedRowCount = Math.max( + remainingUnloadedSiblingCount, + parentRowCount - loadedRowCount + ) + } + + const takeUnloadedRowSlots = ( + siblingCount, + gapSiblingCount, + gapRowCount + ) => { + const anchoredGap = + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + Number.isFinite(gapRowCount) + const siblingBudget = anchoredGap + ? gapSiblingCount + : remainingUnloadedSiblingCount + const rowBudget = anchoredGap + ? Math.max(gapSiblingCount, gapRowCount) + : remainingUnloadedRowCount + if (siblingBudget === null || rowBudget === null) { + return null + } + + const extraRows = Math.max(0, rowBudget - siblingBudget) + const rowSlots = + siblingCount + Math.floor((extraRows * siblingCount) / siblingBudget) + if (remainingUnloadedSiblingCount !== null) { + remainingUnloadedSiblingCount -= siblingCount + remainingUnloadedRowCount -= rowSlots + } + return rowSlots + } let loadedPointer = 0 let index = 0 // Whether a sibling was already placed at this depth. Drives the depth-0 gap, which a @@ -332,6 +510,22 @@ function buildPagedLayout({ loadedIndex === undefined ? totalSiblingCount : loadedIndex, Math.ceil((index + 1) / pageSize) * pageSize ) + const unloadedSiblingCount = unloadedEnd - index + // A loaded node's absolute offset anchors the entire preceding gap. Only + // estimate page boundaries inside that gap: distributing rows across gaps + // on both sides of a loaded node would move it away from its actual rows. + const nextNode = page?.nodes?.[loadedIndex] + const gapEndRowOffset = + loadedIndex !== undefined + ? (nextNode?.rowOffset ?? nextNode?.row_offset) + : Number.isFinite(parentRowOffset) && + Number.isFinite(parentRowCount) + ? parentRowOffset + parentRowCount + : null + const gapRowCount = + Number.isFinite(parentRowOffset) && Number.isFinite(gapEndRowOffset) + ? gapEndRowOffset - parentRowOffset - (y - pageY) / rowHeight + : null y = pushUnloadedGroupPlaceholder({ items, parentPath, @@ -340,8 +534,14 @@ function buildPagedLayout({ endIndex: unloadedEnd, globalStartIndex: globalSiblingCount(depth), y, + geometry, + rowSlotCount: takeUnloadedRowSlots( + unloadedSiblingCount, + (loadedIndex ?? totalSiblingCount) - index, + gapRowCount + ), }) - advanceGlobalSiblingCount(depth, unloadedEnd - index) + advanceGlobalSiblingCount(depth, unloadedSiblingCount) index = unloadedEnd placedSibling = true continue @@ -365,7 +565,7 @@ function buildPagedLayout({ } const gapAbove = placedSibling if (gapAbove) { - y += GROUP_GAP + y += geometry.groupGap } placedSibling = true @@ -380,24 +580,22 @@ function buildPagedLayout({ node.aggregationRowCount ?? node.aggregation_row_count ?? rowCount const childrenCount = node.childrenCount ?? node.children_count ?? 0 - items.push({ - type: 'header', + y = pushGroupHeader(items, geometry, rowHeight, { depth: node.depth ?? depth, path: node.path, display: node.display, rowCount, aggregationRowCount, y, - height: HEADER_HEIGHT, collapsed, gapAbove, aggregations: node.aggregations ?? null, }) - y += HEADER_HEIGHT if (!collapsed) { if ((node.depth ?? depth) === maxDepth) { const sectionHeight = rowCount * rowHeight + const absoluteRowOffset = node.rowOffset ?? node.row_offset items.push({ type: 'rowSection', depth: node.depth ?? depth, @@ -406,23 +604,31 @@ function buildPagedLayout({ rowCount, y, height: sectionHeight, - firstGlobalRowOffset: visibleRowCount, - absoluteRowOffset: node.rowOffset ?? node.row_offset ?? 0, + firstGlobalRowOffset: + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + absoluteRowOffset !== undefined && + absoluteRowOffset !== null + ? absoluteRowOffset + : visibleRowCount, + absoluteRowOffset: absoluteRowOffset ?? 0, }) y += sectionHeight visibleRowCount += rowCount - items.push({ - type: 'addRow', + y = pushGroupAddRow(items, geometry, { depth: node.depth ?? depth, path: node.path, display: node.display, rowCount, y, - height: ADD_ROW_HEIGHT, }) - y += ADD_ROW_HEIGHT } else { - walkPage(node.path, (node.depth ?? depth) + 1, childrenCount) + walkPage( + node.path, + (node.depth ?? depth) + 1, + childrenCount, + rowCount, + node.rowOffset ?? node.row_offset ?? null + ) } } @@ -432,25 +638,12 @@ function buildPagedLayout({ } } - walkPage({}, 0) + walkPage({}, 0, 0, rootRowCount, 0) - // A loaded but group-less view (no rows, or every group emptied) would otherwise - // render nothing to click, so keep one top-level add-row line available. Gated on - // the loaded root page so nothing flashes while the first request is in flight, - // and on a fully empty layout so it never stacks under loading placeholders. const rootPageLoaded = getPage(pages, {}, fields) !== null - if (rootPageLoaded && items.length === 0) { - items.push({ - type: 'addRow', - depth: 0, - path: {}, - y, - height: ADD_ROW_HEIGHT, - }) - y += ADD_ROW_HEIGHT - } + y = pushTrailingAddRow(items, geometry, y, rootPageLoaded) - return { items, totalHeight: y, totalRowCount: visibleRowCount } + return { items, totalHeight: y, totalRowCount: visibleRowCount, geometry } } /** @@ -574,7 +767,8 @@ export function visibleGroupDepthPageInViewport( const visibleStart = placeholderStartIndexAtOffset( item, - Math.max(top, item.y) - item.y + Math.max(top, item.y) - item.y, + layoutGeometryOf(layout) ) // globalSiblingStartIndex maps the placeholder's per-parent start into the // depth-wide sibling space the server offsets on, so batch-fetching the right page. @@ -607,6 +801,7 @@ export function renderViewport({ const items = [] const top = viewport.scrollTop const bottom = viewport.scrollTop + viewport.clientHeight + const geometry = layoutGeometryOf(layout) for (const item of layout.items) { const itemBottom = item.y + item.height @@ -617,6 +812,19 @@ export function renderViewport({ break } + if (item.type === 'groupSpan') { + items.push({ + type: 'groupSpan', + depth: item.depth, + path: item.path, + display: item.display, + rowCount: item.rowCount, + y: item.y, + height: item.height, + }) + continue + } + if (item.type === 'header') { items.push({ type: 'header', @@ -646,16 +854,26 @@ export function renderViewport({ } if (item.type === 'groupPlaceholder') { + const rangeBottom = item.y + item.height + if (geometry.headerHeight === 0) { + const visibleTop = Math.max(top, item.y) + items.push({ + type: 'groupRangePlaceholder', + depth: item.depth, + y: visibleTop, + height: Math.min(bottom, rangeBottom) - visibleTop, + }) + continue + } // Fill the unloaded region with a staircase of skeleton headers, one per level the // group still nests through (the level count is known before the data loads), so // the structure shows instead of blank space while the descendant request resolves. - const rangeBottom = item.y + item.height const maxDepth = Math.max((fields?.length ?? 1) - 1, item.depth) const levelsBelow = maxDepth - item.depth + 1 // Sibling groups are laid out with a gap between them (see // `unloadedGroupRangeHeight`), so step by the same stride or the staircase // over-produces slots across the gapped range. - const slotStep = HEADER_HEIGHT + GROUP_GAP + const slotStep = geometry.unloadedGroupHeight + geometry.groupGap const firstSlotIndex = Math.max(0, Math.floor((top - item.y) / slotStep)) let slotIndex = firstSlotIndex for ( @@ -667,7 +885,7 @@ export function renderViewport({ type: 'groupSkeleton', depth: item.depth + (slotIndex % levelsBelow), y: slotY, - height: Math.min(HEADER_HEIGHT, rangeBottom - slotY), + height: Math.min(geometry.unloadedGroupHeight, rangeBottom - slotY), }) } continue @@ -697,6 +915,9 @@ export function renderViewport({ sectionKey, position: i, globalRowOffset: item.firstGlobalRowOffset + i, + groupEnd: + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + i === item.rowCount - 1, }) } else { items.push({ @@ -735,6 +956,9 @@ export function resolveGroupByRowMoveTarget({ const sourceSectionKey = pathKey(sourcePath, fields) for (const item of layout.items) { + if (item.type === 'groupSpan') { + continue + } if (contentY < item.y || contentY >= item.y + item.height) { continue } diff --git a/web-frontend/modules/database/utils/gridGroupByWidths.js b/web-frontend/modules/database/utils/gridGroupByWidths.js new file mode 100644 index 0000000000..53e049ba28 --- /dev/null +++ b/web-frontend/modules/database/utils/gridGroupByWidths.js @@ -0,0 +1,32 @@ +/** + * Returns the widths to use when rendering group-by columns in the available space. + * The configured widths are left untouched so they can be restored exactly when the + * viewport grows again. If the columns need to shrink, only the space above the + * minimum width is scaled, which also preserves columns explicitly set to the + * minimum. + */ +export function fitGroupByWidths(groupBys, availableWidth, minimumWidth) { + const widths = groupBys.map((groupBy) => + Math.max(groupBy.width, minimumWidth) + ) + const totalWidth = widths.reduce((total, width) => total + width, 0) + + if ( + availableWidth === null || + availableWidth === undefined || + totalWidth <= availableWidth + ) { + return widths + } + + const minimumTotalWidth = minimumWidth * widths.length + if (availableWidth <= minimumTotalWidth) { + return widths.map(() => minimumWidth) + } + + const flexibleWidth = totalWidth - minimumTotalWidth + const availableFlexibleWidth = availableWidth - minimumTotalWidth + const scale = availableFlexibleWidth / flexibleWidth + + return widths.map((width) => minimumWidth + (width - minimumWidth) * scale) +} diff --git a/web-frontend/modules/database/viewTypes.js b/web-frontend/modules/database/viewTypes.js index ffa356c86a..bcd9ca93b9 100644 --- a/web-frontend/modules/database/viewTypes.js +++ b/web-frontend/modules/database/viewTypes.js @@ -614,7 +614,12 @@ export class GridViewType extends ViewType { } getCopyableViewSettings() { - return ['row_height_size', 'frozen_column_count', 'row_identifier_type'] + return [ + 'row_height_size', + 'frozen_column_count', + 'row_identifier_type', + 'group_by_layout', + ] } getName() { @@ -671,6 +676,10 @@ export class GridViewType extends ViewType { storePrefix + 'view/grid/setRowHeight', GRID_VIEW_SIZE_TO_ROW_HEIGHT_MAPPING[view.row_height_size] ) + await store.dispatch( + storePrefix + 'view/grid/setGroupByLayout', + view.group_by_layout || 'section' + ) await store.dispatch(storePrefix + 'view/grid/fetchInitial', { gridId: view.id, fields, diff --git a/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js b/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js index d8854e92a3..5cc703e7d5 100644 --- a/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js +++ b/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js @@ -1,6 +1,8 @@ import { flushPromises } from '@vue/test-utils' import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import AIProviderConfirmModal from '@baserow/modules/core/components/ai/AIProviderConfirmModal' +import AIProviderModelFeatureSelector from '@baserow/modules/core/components/ai/AIProviderModelFeatureSelector' import AIProviderModelFormModal from '@baserow/modules/core/components/ai/AIProviderModelFormModal' import { TestApp } from '@baserow/test/helpers/testApp' @@ -16,6 +18,243 @@ describe('AIProviderModelFormModal', () => { vi.restoreAllMocks() }) + const mountEditForm = async (usage, modelValues = {}) => { + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockImplementation(async (action) => { + if (action === 'aiProvider/fetchModelUsage') { + return usage + } + if (action === 'aiProvider/updateModel') { + return { id: 2, model_identifier: 'gpt-5.6' } + } + }) + const wrapper = await testApp.mount(AIProviderModelFormModal, { + props: { + provider: { id: 1, provider_type: 'openai', models: [] }, + model: { + id: 2, + model_identifier: 'gpt-5.6', + feature_types: ['ai_agent', 'ai_fields', 'kuma'], + ...modelValues, + }, + }, + }) + await wrapper.vm.show() + await flushPromises() + return { wrapper, dispatch } + } + + const uncheckFeatures = async (wrapper, featureTypes) => { + wrapper + .findComponent(AIProviderModelFeatureSelector) + .vm.$emit('update:modelValue', featureTypes) + await flushPromises() + } + + test('confirms before saving when a feature in use is unchecked', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [ + { featureType: 'ai_fields', count: 2 }, + { featureType: 'ai_agent', count: 0 }, + ], + }) + await uncheckFeatures(wrapper, ['ai_agent', 'kuma']) + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/fetchModelUsage', { + modelId: 2, + }) + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + expect(confirmModal.props('title')).toBe( + 'aiProviderAdmin.modelFeaturesRemovedTitle' + ) + expect(confirmModal.props('message')).toMatch( + /^aiProviderAdmin\.modelInUse.* aiProviderAdmin\.modelFeaturesRemovedDescription$/ + ) + + confirmModal.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { + model_identifier: 'gpt-5.6', + feature_types: ['ai_agent', 'kuma'], + }, + }) + }) + + test('saves without a confirmation when the unchecked feature is unused', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [ + { featureType: 'ai_fields', count: 2 }, + { featureType: 'ai_agent', count: 0 }, + ], + }) + await uncheckFeatures(wrapper, ['ai_fields', 'kuma']) + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(wrapper.findComponent(AIProviderConfirmModal).exists()).toBe(false) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { + model_identifier: 'gpt-5.6', + feature_types: ['ai_fields', 'kuma'], + }, + }) + }) + + test('does not look up usage when no feature is unchecked', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [{ featureType: 'ai_fields', count: 2 }], + }) + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/fetchModelUsage', + expect.anything() + ) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { + model_identifier: 'gpt-5.6', + feature_types: ['ai_agent', 'ai_fields', 'kuma'], + }, + }) + }) + + test('confirms the save when the usage lookup fails', async () => { + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockImplementation(async (action) => { + if (action === 'aiProvider/fetchModelUsage') { + throw new Error('Usage unavailable') + } + }) + const wrapper = await testApp.mount(AIProviderModelFormModal, { + props: { + provider: { id: 1, provider_type: 'openai', models: [] }, + model: { + id: 2, + model_identifier: 'gpt-5.6', + feature_types: ['ai_agent', 'ai_fields', 'kuma'], + }, + }, + }) + await wrapper.vm.show() + await uncheckFeatures(wrapper, ['ai_agent', 'kuma']) + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(wrapper.findComponent(AIProviderConfirmModal).exists()).toBe(true) + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + }) + + test('confirms a rename while consumers still point at the old identifier', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [ + { featureType: 'ai_fields', count: 2 }, + { featureType: 'ai_agent', count: 1 }, + ], + }) + wrapper.vm.values.model_identifier = 'gpt-5.7' + await flushPromises() + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + expect(confirmModal.props('title')).toBe( + 'aiProviderAdmin.modelIdentifierRenamedTitle' + ) + expect(confirmModal.props('message')).toMatch( + /^aiProviderAdmin\.modelInUse.* aiProviderAdmin\.modelIdentifierRenamedDescription$/ + ) + + confirmModal.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { + model_identifier: 'gpt-5.7', + feature_types: ['ai_agent', 'ai_fields', 'kuma'], + }, + }) + }) + + test('saves a rename nothing depends on without a confirmation', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [{ featureType: 'ai_fields', count: 0 }], + }) + wrapper.vm.values.model_identifier = 'gpt-5.7' + await flushPromises() + + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + expect(wrapper.findComponent(AIProviderConfirmModal).exists()).toBe(false) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { + model_identifier: 'gpt-5.7', + feature_types: ['ai_agent', 'ai_fields', 'kuma'], + }, + }) + }) + + test('keeps the confirmation open across a re-render and closes both on save', async () => { + const { wrapper, dispatch } = await mountEditForm({ + usage: [{ featureType: 'ai_fields', count: 2 }], + }) + await uncheckFeatures(wrapper, ['ai_agent', 'kuma']) + await wrapper.find('.actions button').trigger('click') + await flushPromises() + + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + const uid = confirmModal.vm.$.uid + expect(confirmModal.vm.$refs.modal.open).toBe(true) + + wrapper.vm.loading = true + await flushPromises() + wrapper.vm.loading = false + await flushPromises() + + const reRendered = wrapper.findComponent(AIProviderConfirmModal) + expect(reRendered.vm.$.uid).toBe(uid) + expect(reRendered.vm.$refs.modal.open).toBe(true) + + reRendered.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + expect(wrapper.emitted('saved')).toBeTruthy() + expect(wrapper.vm.$refs.modal.open).toBe(false) + expect(document.body.classList.contains('prevent-scroll')).toBe(false) + }) + test('shows discovered models, excludes configured models, and accepts a suggestion', async () => { const dispatch = vi .spyOn(testApp.store, 'dispatch') diff --git a/web-frontend/test/unit/core/components/aiProviderWorkspaceSettings.spec.js b/web-frontend/test/unit/core/components/aiProviderWorkspaceSettings.spec.js index 4643af3d37..6b5b989dbb 100644 --- a/web-frontend/test/unit/core/components/aiProviderWorkspaceSettings.spec.js +++ b/web-frontend/test/unit/core/components/aiProviderWorkspaceSettings.spec.js @@ -2,6 +2,7 @@ import { flushPromises } from '@vue/test-utils' import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' import AIProviderActionsMenu from '@baserow/modules/core/components/ai/AIProviderActionsMenu' +import AIProviderConfirmModal from '@baserow/modules/core/components/ai/AIProviderConfirmModal' import AIProviderFeatureSettings from '@baserow/modules/core/components/ai/AIProviderFeatureSettings' import AIProviderWorkspaceSettings from '@baserow/modules/core/components/workspace/AIProviderWorkspaceSettings' import { TestApp } from '@baserow/test/helpers/testApp' @@ -18,6 +19,188 @@ describe('AIProviderWorkspaceSettings', () => { vi.restoreAllMocks() }) + const enabledModel = () => ({ + id: 21, + model_identifier: 'workspace-model', + is_enabled: true, + last_test_status: null, + feature_types: ['ai_fields', 'ai_agent'], + }) + + const mountWithModel = async (model, usage) => { + testApp.store.commit('aiProvider/SET_WORKSPACE_ID', 42) + testApp.store.commit('aiProvider/SET_LOADED', true) + testApp.store.commit('aiProvider/SET_PROVIDERS', [ + { + id: 2, + provider_type: 'openai', + is_active: true, + workspace_enabled: true, + read_only: false, + models: [model], + }, + ]) + testApp.store.commit('aiProvider/SET_PROVIDER_TYPES', [ + { type: 'openai', name: 'OpenAI', uses_api_key: true, extra_fields: [] }, + ]) + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockImplementation(async (action) => { + if (action === 'aiProvider/fetchModelUsage') { + return typeof usage === 'function' ? usage() : usage + } + }) + const wrapper = await testApp.mount(AIProviderWorkspaceSettings, { + props: { workspace: { id: 42 } }, + }) + await flushPromises() + return { wrapper, dispatch } + } + + const selectModelAction = async (wrapper, action) => { + wrapper + .find('.ai-provider-model__actions') + .findComponent(AIProviderActionsMenu) + .vm.$emit('select', action) + await flushPromises() + } + + test('warns with the usage counts before disabling a model in use', async () => { + const model = enabledModel() + const { wrapper, dispatch } = await mountWithModel(model, { + usage: [ + { featureType: 'ai_fields', count: 3 }, + { featureType: 'ai_agent', count: 0 }, + ], + }) + + await selectModelAction(wrapper, 'toggle') + + expect(dispatch).toHaveBeenCalledWith('aiProvider/fetchModelUsage', { + modelId: 21, + workspaceId: 42, + }) + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + expect(confirmModal.props('title')).toBe( + 'aiProviderAdmin.disableModelTitle' + ) + expect(confirmModal.props('message')).toContain( + 'aiProviderAdmin.modelInUse' + ) + expect(confirmModal.props('confirmLabel')).toBe('aiProviderAdmin.disable') + + confirmModal.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 21, + workspaceId: 42, + values: { is_enabled: false }, + }) + }) + + test('disables a model nothing uses without a confirmation', async () => { + const model = enabledModel() + const { wrapper, dispatch } = await mountWithModel(model, { + usage: [ + { featureType: 'ai_fields', count: 0 }, + { featureType: 'ai_agent', count: 0 }, + ], + }) + + await selectModelAction(wrapper, 'toggle') + + expect(wrapper.findComponent(AIProviderConfirmModal).exists()).toBe(false) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 21, + workspaceId: 42, + values: { is_enabled: false }, + }) + }) + + test('confirms the disable when the usage lookup fails', async () => { + const model = enabledModel() + const { wrapper, dispatch } = await mountWithModel(model, () => { + throw new Error('Usage unavailable') + }) + + await selectModelAction(wrapper, 'toggle') + + expect(wrapper.findComponent(AIProviderConfirmModal).props('message')).toBe( + 'aiProviderAdmin.modelUsageUnknown aiProviderAdmin.disableModelDescription' + ) + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/updateModel', + expect.anything() + ) + }) + + test('does not look up usage when enabling a model', async () => { + const model = { ...enabledModel(), is_enabled: false } + const { wrapper, dispatch } = await mountWithModel(model, { + usage: [], + }) + + await selectModelAction(wrapper, 'toggle') + + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/fetchModelUsage', + expect.anything() + ) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 21, + workspaceId: 42, + values: { is_enabled: true }, + }) + }) + + test('names the usage counts before deleting a model in use', async () => { + const model = enabledModel() + const { wrapper, dispatch } = await mountWithModel(model, { + usage: [ + { featureType: 'ai_fields', count: 1 }, + { featureType: 'ai_agent', count: 2 }, + ], + }) + + await selectModelAction(wrapper, 'delete') + + expect(dispatch).not.toHaveBeenCalledWith( + 'aiProvider/deleteModel', + expect.anything() + ) + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + expect(confirmModal.props('title')).toBe('aiProviderAdmin.deleteModelTitle') + expect(confirmModal.props('message')).toMatch( + /^aiProviderAdmin\.modelInUse.* aiProviderAdmin\.deleteModelDescription$/ + ) + + confirmModal.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/deleteModel', { + modelId: 21, + workspaceId: 42, + }) + }) + + test('keeps the plain delete message when nothing uses the model', async () => { + const model = enabledModel() + const { wrapper } = await mountWithModel(model, { + usage: [{ featureType: 'ai_fields', count: 0 }], + }) + + await selectModelAction(wrapper, 'delete') + + expect(wrapper.findComponent(AIProviderConfirmModal).props('message')).toBe( + 'aiProviderAdmin.deleteModelDescription' + ) + }) + test('keeps legacy Kuma controls available without providers', async () => { const featureSetting = { feature_type: 'kuma', diff --git a/web-frontend/test/unit/core/components/selectAIModelForm.spec.js b/web-frontend/test/unit/core/components/selectAIModelForm.spec.js new file mode 100644 index 0000000000..aeace0cd25 --- /dev/null +++ b/web-frontend/test/unit/core/components/selectAIModelForm.spec.js @@ -0,0 +1,204 @@ +import { defineComponent } from 'vue' +import { mountSuspended } from '@nuxt/test-utils/runtime' +import { enableAutoUnmount, flushPromises } from '@vue/test-utils' +import { afterEach, describe, expect, test, vi } from 'vitest' + +import SelectAIModelForm from '@baserow/modules/core/components/ai/SelectAIModelForm' + +const FormGroupStub = defineComponent({ + name: 'FormGroup', + props: { + label: { type: String, default: null }, + error: { type: Boolean, default: false }, + }, + template: + '
', +}) +const DropdownStub = defineComponent({ + name: 'Dropdown', + props: { modelValue: { type: [String, Number], default: null } }, + emits: ['update:modelValue'], + methods: { + select(value) { + this.$emit('update:modelValue', value === undefined ? null : value) + }, + }, + template: + '', +}) +const DropdownItemStub = defineComponent({ + name: 'DropdownItem', + props: { + name: { type: String, required: true }, + value: { type: String, required: true }, + disabled: { type: Boolean, default: false }, + }, + template: + '', +}) +const PassthroughStub = defineComponent({ template: '
' }) + +enableAutoUnmount(afterEach) + +const openAIModelType = { + getType: () => 'openai', + getName: () => 'OpenAI', + getMaxTemperature: () => 2, +} +const anthropicModelType = { + getType: () => 'anthropic', + getName: () => 'Anthropic', + getMaxTemperature: () => 1, +} +const modelTypes = { openai: openAIModelType, anthropic: anthropicModelType } + +const workspace = { + id: 1, + generative_ai_models_enabled: { openai: ['legacy-model'] }, + ai_features: { ai_fields: { models: { openai: ['gpt-4'] } } }, +} + +async function mountForm({ + featureFlagEnabled = true, + defaultValues = {}, + workspace: workspaceValue = workspace, +} = {}) { + const wrapper = await mountSuspended(SelectAIModelForm, { + props: { + database: { id: 100, workspace: { id: 1 } }, + featureType: 'ai_fields', + defaultValues, + }, + global: { + stubs: { + FormGroup: FormGroupStub, + Dropdown: DropdownStub, + DropdownItem: DropdownItemStub, + FormInput: PassthroughStub, + }, + mocks: { + $t: (key) => key, + $featureFlagIsEnabled: () => featureFlagEnabled, + $store: { + dispatch: vi.fn().mockResolvedValue(undefined), + getters: { + 'workspace/get': () => workspaceValue, + 'settings/get': () => ({}), + }, + }, + $registry: { + getAll: () => modelTypes, + get: (namespace, type) => modelTypes[type], + }, + }, + }, + }) + await flushPromises() + return wrapper +} + +const typeField = (wrapper) => + wrapper.get('[data-label="selectAIModelForm.AIType"]') +const modelField = (wrapper) => + wrapper.get('[data-label="selectAIModelForm.AIModel"]') + +describe('SelectAIModelForm', () => { + test('keeps a saved model that is no longer available visible and invalid', async () => { + const wrapper = await mountForm({ + defaultValues: { + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'retired-model', + }, + }) + + const field = modelField(wrapper) + expect(field.get('select').element.value).toBe('retired-model') + expect( + field.get('[data-value="retired-model"]').attributes('aria-disabled') + ).toBe('true') + expect(field.get('[data-value="gpt-4"]').exists()).toBe(true) + expect(field.attributes('data-error')).toBe('true') + expect(field.text()).toContain('selectAIModelForm.modelUnavailable') + + wrapper.vm.touch() + expect(wrapper.vm.isFormValid()).toBe(false) + }) + + test('keeps a saved provider that is no longer available visible', async () => { + const wrapper = await mountForm({ + defaultValues: { + ai_generative_ai_type: 'anthropic', + ai_generative_ai_model: 'claude', + }, + }) + + const field = typeField(wrapper) + expect(field.get('select').element.value).toBe('anthropic') + expect(field.get('[data-value="anthropic"]').text()).toBe('Anthropic') + expect(field.get('[data-value="openai"]').text()).toBe('OpenAI') + }) + + test('names a saved provider the registry no longer knows by its type', async () => { + const wrapper = await mountForm({ + defaultValues: { + ai_generative_ai_type: 'removed-provider', + ai_generative_ai_model: 'removed-model', + }, + }) + + expect( + typeField(wrapper).get('[data-value="removed-provider"]').text() + ).toBe('removed-provider') + expect( + modelField(wrapper).get('[data-value="removed-model"]').exists() + ).toBe(true) + }) + + test('leaves an available selection untouched', async () => { + const wrapper = await mountForm({ + defaultValues: { + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'gpt-4', + }, + }) + + const field = modelField(wrapper) + expect(field.findAll('option').map((option) => option.text())).toEqual([ + 'gpt-4', + ]) + expect(field.get('[data-value="gpt-4"]').attributes('aria-disabled')).toBe( + 'false' + ) + expect(field.attributes('data-error')).toBe('false') + expect(field.text()).not.toContain('selectAIModelForm.modelUnavailable') + + wrapper.vm.touch() + expect(wrapper.vm.isFormValid()).toBe(true) + }) + + test('selects the first available type and model for an empty form', async () => { + const wrapper = await mountForm() + + expect(wrapper.vm.values.ai_generative_ai_type).toBe('openai') + expect(wrapper.vm.values.ai_generative_ai_model).toBe('gpt-4') + }) + + test('does not retain an unavailable model when the flag is disabled', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: false, + defaultValues: { + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'retired-model', + }, + }) + + const field = modelField(wrapper) + expect(field.findAll('option').map((option) => option.text())).toEqual([ + 'legacy-model', + ]) + expect(field.attributes('data-error')).toBe('false') + + wrapper.vm.touch() + expect(wrapper.vm.isFormValid()).toBe(false) + }) +}) diff --git a/web-frontend/test/unit/core/pages/admin/aiProviders.spec.js b/web-frontend/test/unit/core/pages/admin/aiProviders.spec.js index bf0e2cdd4f..b40e4e678c 100644 --- a/web-frontend/test/unit/core/pages/admin/aiProviders.spec.js +++ b/web-frontend/test/unit/core/pages/admin/aiProviders.spec.js @@ -1,5 +1,7 @@ import { flushPromises } from '@vue/test-utils' +import AIProviderActionsMenu from '@baserow/modules/core/components/ai/AIProviderActionsMenu' +import AIProviderConfirmModal from '@baserow/modules/core/components/ai/AIProviderConfirmModal' import AdminAIProviders from '@baserow/modules/core/pages/admin/aiProviders' import { TestApp } from '@baserow/test/helpers/testApp' @@ -120,6 +122,98 @@ describe('AdminAIProviders', () => { expect(wrapper.find('.skeleton').exists()).toBe(true) }) + const mountWithModel = async (model, usage) => { + testApp.store.commit('aiProvider/SET_WORKSPACE_ID', null) + testApp.store.commit('aiProvider/SET_LOADED', true) + testApp.store.commit('aiProvider/SET_PROVIDERS', [ + { + id: 1, + provider_type: 'openai', + is_active: true, + extra_settings: {}, + models: [model], + }, + ]) + testApp.store.commit('aiProvider/SET_PROVIDER_TYPES', [ + { type: 'openai', name: 'OpenAI', uses_api_key: true, extra_fields: [] }, + ]) + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockImplementation(async (action) => { + if (action === 'aiProvider/fetchModelUsage') { + return usage + } + }) + const wrapper = await testApp.mount(AdminAIProviders) + await flushPromises() + return { wrapper, dispatch } + } + + const selectModelAction = async (wrapper, action) => { + wrapper + .find('.ai-provider-model__actions') + .findComponent(AIProviderActionsMenu) + .vm.$emit('select', action) + await flushPromises() + } + + test('warns with the instance-wide usage counts before disabling a model', async () => { + const { wrapper, dispatch } = await mountWithModel( + { + id: 2, + model_identifier: 'gpt-5.6', + is_enabled: true, + last_test_status: null, + }, + { + usage: [{ featureType: 'ai_fields', count: 3 }], + } + ) + + await selectModelAction(wrapper, 'toggle') + + expect(dispatch).toHaveBeenCalledWith('aiProvider/fetchModelUsage', { + modelId: 2, + }) + const confirmModal = wrapper.findComponent(AIProviderConfirmModal) + expect(confirmModal.props('title')).toBe( + 'aiProviderAdmin.disableModelTitle' + ) + expect(confirmModal.props('message')).toMatch( + /^aiProviderAdmin\.modelInUse.* aiProviderAdmin\.disableModelDescription$/ + ) + + confirmModal.vm.$emit('confirm') + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { is_enabled: false }, + }) + }) + + test('disables a model nothing depends on without a confirmation', async () => { + const { wrapper, dispatch } = await mountWithModel( + { + id: 2, + model_identifier: 'gpt-5.6', + is_enabled: true, + last_test_status: null, + }, + { + usage: [{ featureType: 'ai_fields', count: 0 }], + } + ) + + await selectModelAction(wrapper, 'toggle') + + expect(wrapper.findComponent(AIProviderConfirmModal).exists()).toBe(false) + expect(dispatch).toHaveBeenCalledWith('aiProvider/updateModel', { + modelId: 2, + values: { is_enabled: false }, + }) + }) + test('tests every provider model in one request', async () => { let finishRequest const dispatch = vi.fn( diff --git a/web-frontend/test/unit/core/store/aiProvider.spec.js b/web-frontend/test/unit/core/store/aiProvider.spec.js index f01abad5d8..16495ec4b9 100644 --- a/web-frontend/test/unit/core/store/aiProvider.spec.js +++ b/web-frontend/test/unit/core/store/aiProvider.spec.js @@ -32,6 +32,14 @@ describe('AI provider store', () => { create: vi.fn().mockResolvedValue({ data: { id: 1 } }), update: vi.fn().mockResolvedValue({ data: { id: 1, is_active: false } }), updateModel: vi.fn().mockResolvedValue({ data: { id: 2 } }), + fetchModelUsage: vi.fn().mockResolvedValue({ + data: { + usage: [ + { feature_type: 'ai_fields', count: 3 }, + { feature_type: 'ai_agent', count: 0 }, + ], + }, + }), testModels: vi.fn().mockResolvedValue({ data: { results: [ @@ -483,6 +491,43 @@ describe('AI provider store', () => { ) }) + test('fetchModelUsage maps the response to camel case', async () => { + const result = await actions.fetchModelUsage.call( + { $client: {} }, + { commit: vi.fn(), state: makeState() }, + { modelId: 2 } + ) + + expect(service.fetchModelUsage).toHaveBeenCalledWith(2) + expect(result).toEqual({ + usage: [ + { featureType: 'ai_fields', count: 3 }, + { featureType: 'ai_agent', count: 0 }, + ], + }) + }) + + test('fetchModelUsage accepts a bare model id and defaults to instance scope', async () => { + await actions.fetchModelUsage.call( + { $client: {} }, + { commit: vi.fn(), state: makeState() }, + 2 + ) + + expect(aiProviderService).toHaveBeenLastCalledWith({}, null) + expect(service.fetchModelUsage).toHaveBeenCalledWith(2) + }) + + test('fetchModelUsage scopes the request to the given workspace', async () => { + await actions.fetchModelUsage.call( + { $client: {} }, + { commit: vi.fn(), state: makeState() }, + { modelId: 2, workspaceId: 42 } + ) + + expect(aiProviderService).toHaveBeenLastCalledWith({}, 42) + }) + test('model mutations preserve the provider and update only the model', () => { const state = makeState() state.providers = [{ id: 1, models: [{ id: 2, model_identifier: 'old' }] }] diff --git a/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap b/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap index cd1ece5f1f..ecce9616b1 100644 --- a/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap +++ b/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap @@ -448,13 +448,11 @@ exports[`Public View Page Tests > Can see a publicly shared grid view 1`] = ` class="grid-view__placeholder" style="width: 72px;" > -
+
+
Can see a publicly shared grid view 1`] = `
+
@@ -662,6 +661,7 @@ exports[`Public View Page Tests > Can see a publicly shared grid view 1`] = ` />
+
Can see a publicly shared grid view 1`] = ` class="grid-view__foot" > +
Adding a row to a table increases the row count
-
+
+
@@ -177,6 +178,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
gridView.rowCount - 1
@@ -253,6 +255,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -281,6 +284,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -494,8 +498,9 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
-
+
+
@@ -516,6 +521,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
gridView.rowCount - 2
@@ -592,6 +598,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -620,6 +627,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
diff --git a/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap b/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap index 48fc1acfbc..192bb08db9 100644 --- a/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap +++ b/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap @@ -63,13 +63,11 @@ exports[`GridView component with decoration > Default component with first_cell class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with first_cell
+
@@ -346,6 +345,7 @@ exports[`GridView component with decoration > Default component with first_cell />
+
Default component with first_cell class="grid-view__foot" > +
Default component with row wrapper class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with row wrapper
+
@@ -874,6 +874,7 @@ exports[`GridView component with decoration > Default component with row wrapper />
+
Default component with row wrapper class="grid-view__foot" > +
Default component with unavailable class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with unavailable
+
@@ -1402,6 +1403,7 @@ exports[`GridView component with decoration > Default component with unavailable />
+
Default component with unavailable class="grid-view__foot" > +
{ const fields = [ @@ -54,6 +56,170 @@ describe('GridView component', () => { expect(sortedFields.map((field) => field.id)).toEqual([1, 3]) }) + test('frozen columns fall back when group columns would crowd the viewport', () => { + const context = { + $refs: { gridView: { clientWidth: 800 } }, + fields: [{ id: 1, name: 'Primary', primary: true }], + fieldOptions: { 1: { hidden: false, order: 0 } }, + frozenColumnCount: 1, + gridViewRowDetailsWidth: 72, + groupColumnsWidth: 0, + getFieldWidth: () => 200, + canFitFrozenColumns: true, + } + + GridView.methods.checkCanFitFrozenColumns.call(context) + expect(context.canFitFrozenColumns).toBe(true) + + context.groupColumnsWidth = 400 + GridView.methods.checkCanFitFrozenColumns.call(context) + expect(context.canFitFrozenColumns).toBe(false) + }) + + test.each([ + [1024, 130.4], + // The default 240px application sidebar leaves a 784px grid at a 1024px + // browser viewport. + [784, 82.4], + ])( + 'five group columns leave a usable data pane at a %ipx grid width', + (gridViewWidth, expectedGroupWidth) => { + const groupBys = Array.from({ length: 5 }, (_, index) => ({ + id: index, + width: 200, + })) + const groupByWidths = GridView.computed.groupByWidths.call({ + isColumnLayout: true, + activeGroupBys: groupBys, + gridViewWidth, + gridViewRowDetailsWidth: 72, + }) + const groupColumnsWidth = GridView.computed.groupColumnsWidth.call({ + groupByWidths, + }) + const leftWidth = GridView.computed.leftWidth.call({ + leftFieldsWidth: 0, + gridViewRowDetailsWidth: 72, + groupColumnsWidth, + }) + + expect(groupByWidths).toEqual(Array(5).fill(expectedGroupWidth)) + expect(gridViewWidth - leftWidth).toBeCloseTo(300) + expect(groupBys.map(({ width }) => width)).toEqual(Array(5).fill(200)) + } + ) + + test('effective group width changes recalculate grid geometry', () => { + const fieldsUpdated = vi.fn() + const nextTick = vi.fn((callback) => callback()) + const context = { fieldsUpdated, $nextTick: nextTick } + + GridView.watch.groupColumnsWidth.call(context, 240, 200) + + expect(nextTick).toHaveBeenCalledOnce() + expect(fieldsUpdated).toHaveBeenCalledOnce() + + GridView.watch.groupColumnsWidth.call(context, 240, 240) + expect(nextTick).toHaveBeenCalledOnce() + expect(fieldsUpdated).toHaveBeenCalledOnce() + + expect( + GridView.computed.groupColumnsWidth.call({ groupByWidths: [] }) + ).toBe(0) + }) + + test.each([ + [[82.4, 82.4], [200, 200], true], + [[200, 200], [200, 200], false], + [[78, 200], [40, 200], false], + ])( + 'group width resize handles responsive state for rendered widths %j', + (renderedGroupByWidths, configuredWidths, expected) => { + expect( + GridViewSection.computed.groupByWidthsAreResponsivelyFitted.call({ + activeGroupBys: configuredWidths.map((width) => ({ width })), + renderedGroupByWidths, + groupColumnsWidth: renderedGroupByWidths.reduce( + (total, width) => total + width, + 0 + ), + GRID_VIEW_MIN_FIELD_WIDTH: 78, + }) + ).toBe(expected) + } + ) + + test('row dragging starts after group columns only in column layout', async () => { + const testApp = new TestApp() + try { + const mockServer = testApp.mockServer + const store = testApp.store + const table = mockServer.createTable() + const { application } = await mockServer.createAppAndWorkspace(table) + const groupBys = [ + { + id: 10, + field: 2, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + }, + ] + const view = mockServer.createGridView(application, table, { groupBys }) + view.group_by_layout = 'column' + mockServer.createFields(application, table, [ + { + id: 1, + name: 'Name', + order: 0, + type: 'text', + primary: true, + text_default: '', + }, + { + id: 2, + name: 'Team', + order: 1, + type: 'text', + primary: false, + text_default: '', + }, + ]) + await store.dispatch('field/fetchAll', { table }) + const mountedFields = store.getters['field/getAll'] + const primary = store.getters['field/getPrimary'] + mockServer.createGridRows(view, mountedFields, []) + await store.dispatch('page/view/grid/fetchInitial', { + gridId: view.id, + fields: mountedFields, + primary, + }) + await store.dispatch('page/view/grid/updateActiveGroupBys', groupBys) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + + const wrapper = await testApp.mount(GridView, { + props: { + fields: mountedFields, + view, + table, + database: application, + readOnly: false, + storePrefix: 'page/', + row: null, + }, + }) + + expect(wrapper.findComponent(GridViewRowDragging).props('offset')).toBe( + 200 + ) + await wrapper.setProps({ view: { ...view, group_by_layout: 'section' } }) + expect(wrapper.findComponent(GridViewRowDragging).props('offset')).toBe(0) + } finally { + await testApp.afterEach() + } + }) + // The post-drag click of a multi-select lands on the rows container; that must not // cancel the selection. Regression for group-by, whose rows use their own containers. const runCancel = (targetClass) => { diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js new file mode 100644 index 0000000000..7efc824688 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js @@ -0,0 +1,112 @@ +import { TestApp } from '@baserow/test/helpers/testApp' +import GridViewGroupByColumns from '@baserow/modules/database/components/view/grid/GridViewGroupByColumns' + +describe('GridViewGroupByColumns component', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Team', type: 'text', primary: true }, + { id: 2, name: 'Role', type: 'text', primary: false }, + ] + const groupBys = [ + { id: 10, field: 1, order: 'ASC', type: 'default', width: 120 }, + { id: 11, field: 2, order: 'ASC', type: 'default', width: 90 }, + ] + + beforeEach(() => { + testApp = new TestApp() + store = testApp.store + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + const seedGroups = async (treeNodes) => { + store.commit('page/view/grid/APPLY_GROUP_BY_STATE', { + activeGroupBys: groupBys, + groupBy: { + treeNodes, + pages: {}, + absoluteRows: {}, + revision: 0, + generation: 0, + aggregationsLoading: false, + aggregationsLoadingPaths: [], + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + sectionRows: {}, + rowLocations: {}, + sectionAccessOrder: [], + addRowHoverPathKey: null, + offsetsServerConfirmed: true, + }, + }) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + } + + const mountColumns = (props = {}) => + testApp.mount(GridViewGroupByColumns, { + props: { + allFieldsInTable: fields, + workspaceId: 1, + storePrefix: 'page/', + ...props, + }, + }) + + test('renders one cell per visible span, positioned per level, with value and count', async () => { + await seedGroups([ + { path: { field_1: 'A' }, depth: 0, row_count: 3 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 2 }, + { path: { field_1: 'A', field_2: 'Y' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns() + + const spans = wrapper.findAll('.grid-view__group-span') + expect(spans).toHaveLength(3) + expect(spans[0].attributes('style')).toContain('left: 0px') + expect(spans[0].attributes('style')).toContain('width: 120px') + expect(spans[0].attributes('style')).toContain('height: 99px') + expect(spans[1].attributes('style')).toContain('left: 120px') + expect(spans[1].attributes('style')).toContain('width: 90px') + expect(spans[2].attributes('style')).toContain('top: 66px') + expect(spans[0].find('.grid-view__group-value').text()).toBe('A') + expect(spans[0].find('.grid-view__group-count').text()).toBe('3') + expect(spans[1].find('.grid-view__group-value').text()).toBe('X') + expect(spans[2].find('.grid-view__group-count').text()).toBe('1') + expect( + wrapper.find('.grid-view__group-columns').attributes('style') + ).toContain('width: 210px') + }) + + test('renders the empty label for a group without a value', async () => { + await seedGroups([ + { path: { field_1: '' }, depth: 0, row_count: 1 }, + { path: { field_1: '', field_2: '' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns() + + const empties = wrapper.findAll('.grid-view__group-value-empty') + expect(empties).toHaveLength(2) + expect(empties[0].text()).toBe('gridViewGroupByBanner.emptyValue') + }) + + test('uses responsive widths without changing the configured group widths', async () => { + await seedGroups([ + { path: { field_1: 'A' }, depth: 0, row_count: 1 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns({ groupByWidths: [80, 100] }) + + const spans = wrapper.findAll('.grid-view__group-span') + expect(spans[0].attributes('style')).toContain('width: 80px') + expect(spans[1].attributes('style')).toContain('left: 80px') + expect(spans[1].attributes('style')).toContain('width: 100px') + expect( + wrapper.find('.grid-view__group-columns').attributes('style') + ).toContain('width: 180px') + expect(groupBys.map(({ width }) => width)).toEqual([120, 90]) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js new file mode 100644 index 0000000000..a4befb1db2 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js @@ -0,0 +1,107 @@ +import GridViewGroupByRows from '@baserow/modules/database/components/view/grid/GridViewGroupByRows' +import { pathKey } from '@baserow/modules/database/utils/gridGroupByRender' +import { TestApp } from '@baserow/test/helpers/testApp' + +describe('GridViewGroupByRows component', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Team', type: 'text', primary: true, text_default: '' }, + ] + const groupBys = [ + { id: 10, field: 1, order: 'ASC', type: 'default', width: 200 }, + ] + const makeRow = (id, team) => ({ + id, + field_1: team, + _: { + loading: false, + selected: false, + hover: false, + matchFilters: true, + matchSortings: true, + matchSearch: true, + fieldSearchMatches: [], + persistentId: id, + }, + }) + + beforeEach(() => { + testApp = new TestApp() + store = testApp.store + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + test('continues column group boundaries across row cells only in Columns', async () => { + const rowsA = [makeRow(1, 'A'), makeRow(2, 'A')] + const rowsB = [makeRow(3, 'B')] + store.commit('page/view/grid/APPLY_GROUP_BY_STATE', { + activeGroupBys: groupBys, + count: 3, + groupBy: { + treeNodes: [ + { path: { field_1: 'A' }, depth: 0, row_count: 2 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + ], + pages: {}, + absoluteRows: {}, + revision: 0, + generation: 0, + aggregationsLoading: false, + aggregationsLoadingPaths: [], + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + sectionRows: { + [pathKey({ field_1: 'A' }, fields)]: rowsA, + [pathKey({ field_1: 'B' }, fields)]: rowsB, + }, + rowLocations: {}, + sectionAccessOrder: [], + addRowHoverPathKey: null, + offsetsServerConfirmed: true, + }, + }) + store.commit('page/view/grid/SET_WINDOW_HEIGHT', 1000) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + + const wrapper = await testApp.mount(GridViewGroupByRows, { + props: { + renderedFields: fields, + visibleFields: fields, + allVisibleFields: fields, + allFieldsInTable: fields, + decorationsByPlace: {}, + groupColumnsWidth: 200, + view: { + id: 1, + table: {}, + sortings: [], + row_identifier_type: 'count', + }, + includeRowDetails: true, + readOnly: false, + workspaceId: 1, + storePrefix: 'page/', + }, + global: { + stubs: { GridViewRow: true, GridViewGroupByBanner: true }, + }, + }) + + expect(wrapper.findAll('.grid-view__group-by-rows-row')).toHaveLength(3) + expect( + wrapper.findAll('.grid-view__group-by-rows-row--group-end') + ).toHaveLength(2) + + await store.dispatch('page/view/grid/setGroupByLayout', 'section') + await wrapper.vm.$nextTick() + + expect( + wrapper.findAll('.grid-view__group-by-rows-row--group-end') + ).toHaveLength(0) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js new file mode 100644 index 0000000000..06fc742e51 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js @@ -0,0 +1,147 @@ +import { h, nextTick, ref } from 'vue' +import flushPromises from 'flush-promises' + +import { TestApp } from '@baserow/test/helpers/testApp' +import GridViewSection from '@baserow/modules/database/components/view/grid/GridViewSection' +import { fitGroupByWidths } from '@baserow/modules/database/utils/gridGroupByWidths' +import { GRID_VIEW_MIN_FIELD_WIDTH } from '@baserow/modules/database/constants' + +describe('GridViewSection group column resizing', () => { + let testApp + + beforeEach(() => { + testApp = new TestApp() + }) + + afterEach(async () => { + window.dispatchEvent(new MouseEvent('mouseup')) + await testApp.afterEach() + }) + + const mountSection = async () => { + const availableWidth = ref(250) + const groupBy = { + id: 10, + view: 1, + field: 1, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + } + const view = { id: 1, group_bys: [groupBy], group_by_layout: 'column' } + await testApp.store.dispatch('page/view/grid/updateActiveGroupBys', [ + groupBy, + ]) + testApp.mock.onPatch('/database/views/group_by/10/').reply(200, {}) + + const wrapper = await testApp.mount( + { + render() { + return h(GridViewSection, { + view, + database: { id: 1, workspace: { id: 1 } }, + table: { id: 1 }, + visibleFields: [], + allVisibleFields: [], + allFieldsInTable: [], + decorationsByPlace: {}, + includeGroupBy: true, + readOnly: false, + storePrefix: 'page/', + groupByWidths: fitGroupByWidths( + testApp.store.getters['page/view/grid/getActiveGroupBys'], + availableWidth.value, + GRID_VIEW_MIN_FIELD_WIDTH + ), + }) + }, + }, + { + global: { + mocks: { $hasPermission: () => true }, + stubs: { + GridViewHead: true, + GridViewGroupByColumns: true, + GridViewGroupByRows: true, + }, + }, + } + ) + return { wrapper, availableWidth } + } + + const moveMouse = async (type, clientX) => { + window.dispatchEvent(new MouseEvent(type, { clientX })) + await nextTick() + } + + test.each([ + { positions: [240] }, + { positions: [270] }, + { positions: [270, 240] }, + ])( + 'persists the final width after dragging through $positions', + async ({ positions }) => { + const { wrapper, availableWidth } = await mountSection() + const finalWidth = positions.at(-1) + await wrapper + .get('.grid-view__head-group-width-handle') + .trigger('mousedown', { clientX: 200 }) + + for (const position of positions) { + await moveMouse('mousemove', position) + } + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain(`left: ${Math.min(finalWidth, 250)}px`) + expect( + wrapper.find('.grid-view__head-group-width-handle.dragging').exists() + ).toBe(true) + expect(testApp.mock.history.patch).toHaveLength(0) + + await moveMouse('mouseup', finalWidth) + await flushPromises() + + expect(testApp.mock.history.patch).toHaveLength(1) + expect(JSON.parse(testApp.mock.history.patch[0].data)).toEqual({ + width: finalWidth, + }) + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + finalWidth <= 250 + ) + + availableWidth.value = 400 + await nextTick() + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain(`left: ${finalWidth}px`) + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + true + ) + } + ) + + test('finishes a drag returned to its original width without persisting', async () => { + const { wrapper, availableWidth } = await mountSection() + await wrapper + .get('.grid-view__head-group-width-handle') + .trigger('mousedown', { clientX: 200 }) + + await moveMouse('mousemove', 270) + await moveMouse('mousemove', 200) + await moveMouse('mouseup', 200) + await flushPromises() + + expect(testApp.mock.history.patch).toHaveLength(0) + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain('left: 200px') + + availableWidth.value = 150 + await nextTick() + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + false + ) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js b/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js new file mode 100644 index 0000000000..178b30c020 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js @@ -0,0 +1,113 @@ +import flushPromises from 'flush-promises' +import { vi } from 'vitest' + +import { TestApp } from '@baserow/test/helpers/testApp' +import ViewGroupByContext from '@baserow/modules/database/components/view/ViewGroupByContext' + +describe('ViewGroupByContext', () => { + let testApp = null + + const database = { id: 1, workspace: { id: 1 } } + const fields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text', primary: false }, + ] + const makeView = (groupByLayout) => ({ + id: 1, + ownership_type: 'collaborative', + group_bys: [ + { + id: 10, + field: 2, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + }, + ], + group_by_layout: groupByLayout, + }) + + beforeEach(() => { + testApp = new TestApp() + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + const mountContext = async (view) => { + const wrapper = await testApp.mount(ViewGroupByContext, { + props: { database, view, fields, readOnly: false, disableGroupBy: false }, + }) + await wrapper.vm.show(document.body) + await wrapper.vm.$nextTick() + return wrapper + } + + const contextElement = () => document.body.querySelector('.context.group-bys') + const segmentLabels = () => + [...contextElement().querySelectorAll('.segment-control__button')].map( + (button) => button.textContent.trim() + ) + const buttonTexts = () => + [...contextElement().querySelectorAll('.group-bys__footer-actions button')] + .map((button) => button.textContent.trim()) + .filter(Boolean) + + test('section layout shows the layout switch and the collapse buttons', async () => { + await mountContext(makeView('section')) + + expect(segmentLabels()).toEqual([ + 'viewGroupByContext.layoutSection', + 'viewGroupByContext.layoutColumn', + ]) + expect(buttonTexts()).toEqual( + expect.arrayContaining([ + 'viewGroupByContext.collapseAllGroups', + 'viewGroupByContext.expandAllGroups', + ]) + ) + }) + + test('column layout hides the collapse buttons', async () => { + await mountContext(makeView('column')) + + expect(buttonTexts()).not.toEqual( + expect.arrayContaining(['viewGroupByContext.collapseAllGroups']) + ) + expect(buttonTexts()).not.toEqual( + expect.arrayContaining(['viewGroupByContext.expandAllGroups']) + ) + const active = contextElement().querySelector( + '.segment-control__button--active' + ) + expect(active.textContent.trim()).toBe('viewGroupByContext.layoutColumn') + }) + + test.each([ + ['Sections', 'column', 'section', 0], + ['Columns', 'section', 'column', 1], + ])( + 'choosing %s updates the view setting', + async (_label, currentLayout, selectedLayout, index) => { + const view = makeView(currentLayout) + await mountContext(view) + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockResolvedValue(undefined) + + const selected = [ + ...contextElement().querySelectorAll('.segment-control__button'), + ][index] + selected.click() + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('view/update', { + view, + values: { group_by_layout: selectedLayout }, + readOnly: false, + }) + } + ) +}) diff --git a/web-frontend/test/unit/database/store/view/grid.spec.js b/web-frontend/test/unit/database/store/view/grid.spec.js index 1077042ada..45f7d06dcc 100644 --- a/web-frontend/test/unit/database/store/view/grid.spec.js +++ b/web-frontend/test/unit/database/store/view/grid.spec.js @@ -10332,3 +10332,491 @@ describe('Grid view store', () => { expect(rowsOrderByParam).toBe('field_1') }) }) + +describe('Grid view store group-by layout mode', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + ] + const groupBys = [ + { id: 10, field: 2, order: 'ASC', type: 'default', width: 200 }, + ] + const view = { + id: 1, + filters: [], + filter_groups: [], + filter_type: 'AND', + sortings: [], + group_bys: groupBys, + } + const seed = (targetStore, extra = {}) => { + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: groupBys, + rowHeight: 33, + windowHeight: 100, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + }, + groupBy: { + ...gridStore.state().groupBy, + treeNodes: [{ path: { field_2: 'A' }, depth: 0, row_count: 2 }], + collapse: { mode: 'expand', paths: [] }, + }, + ...extra, + }) + targetStore.replaceState({ ...targetStore.state, grid: state }) + } + + beforeEach(() => { + testApp = new TestApp() + store = testApp.createStore({ modules: { grid: gridStore } }) + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + test('setGroupByLayout switches the layout builder and forces expand-all', async () => { + seed(store) + store.commit('grid/SET_GROUP_BY_COLLAPSE', { mode: 'collapse', paths: [] }) + expect( + store.getters['grid/getGroupByLayout'].items.map((item) => item.type) + ).toEqual(['header']) + + await store.dispatch('grid/setGroupByLayout', 'column') + + expect(store.getters['grid/isGroupByColumnLayout']).toBe(true) + expect( + store.getters['grid/getGroupByLayout'].items.map((item) => item.type) + ).toEqual(['groupSpan', 'rowSection', 'addRow']) + }) + + test('collapse actions are no-ops in column layout', async () => { + const fetchGroupByRowsByScrollTop = vi.fn().mockResolvedValue([]) + const groupByStore = testApp.createStore({ + modules: { + grid: { + ...gridStore, + actions: { ...gridStore.actions, fetchGroupByRowsByScrollTop }, + }, + }, + }) + seed(groupByStore, { groupByLayout: 'column' }) + + await groupByStore.dispatch('grid/toggleGroupCollapse', { + path: { field_2: 'A' }, + view, + fields, + }) + await groupByStore.dispatch('grid/setGroupByCollapseAll', { + view, + fields, + collapse: true, + }) + + expect(groupByStore.getters['grid/getGroupByCollapse']).toEqual({ + mode: 'expand', + paths: [], + }) + expect(fetchGroupByRowsByScrollTop).not.toHaveBeenCalled() + }) + + test.each(['column', 'section'])( + '%s layout selects loaded rows beyond an unloaded group page', + async (layout) => { + const nodes = Object.fromEntries( + [0, 80].flatMap((offset) => + Array.from({ length: 40 }, (_, index) => { + const position = offset + index + return [ + position, + { + path: { field_2: `Group ${position}` }, + depth: 0, + row_count: 1, + sibling_index: position, + row_offset: position, + }, + ] + }) + ) + ) + seed(store, { + groupByLayout: layout, + count: 120, + groupBy: { + ...gridStore.state().groupBy, + pages: { '': { parentPath: {}, totalSiblingCount: 120, nodes } }, + }, + }) + for (const rowId of [100, 101]) { + store.commit('grid/SET_GROUP_BY_SECTION_ROWS', { + sectionKey: groupPathKey(2, `Group ${rowId}`), + startPosition: 0, + rows: [ + { + id: rowId, + field_2: `Group ${rowId}`, + _: { selected: false, selectedFieldId: -1 }, + }, + ], + }) + } + + await store.dispatch('grid/multiSelectStart', { + rowId: 100, + fieldIndex: 0, + }) + await store.dispatch('grid/multiSelectHold', { + rowId: 101, + fieldIndex: 1, + }) + + const firstRowIndex = layout === 'column' ? 100 : 60 + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + expect( + store.getters['grid/getSelectedRows'].map((row) => row.id) + ).toEqual([100, 101]) + + await store.dispatch('grid/correctMultiSelect') + + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + expect( + store.getters['grid/getSelectedRows'].map((row) => row.id) + ).toEqual([100, 101]) + + await store.dispatch('grid/updateMultipleSelectIndexes', { + position: 'tail', + rowIndex: layout === 'column' ? 120 : 80, + fieldIndex: 1, + }) + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + } + ) + + test('column layout initially fetches descendants despite a saved collapse-all state', async () => { + const nestedFields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + { id: 3, name: 'Role', type: 'text' }, + ] + const nestedGroupBys = [ + { field: 2, order: 'ASC', type: 'default', width: 200 }, + { field: 3, order: 'ASC', type: 'default', width: 200 }, + ] + const nestedView = { ...view, group_bys: nestedGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: nestedGroupBys, + groupByLayout: 'column', + count: 2, + rowHeight: 33, + rowPadding: 0, + windowHeight: 330, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + 3: { hidden: false, order: 2 }, + }, + groupBy: { + ...gridStore.state().groupBy, + pages: {}, + treeNodes: [], + collapse: { mode: 'collapse', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const groupByRequests = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + groupByRequests.push(config.params) + return [ + 200, + { + pages: [ + { + parent: {}, + groups: [ + { + path: { field_2: 'A' }, + depth: 0, + row_count: 2, + children_count: 1, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + }, + { + parent: { field_2: 'A' }, + groups: [ + { + path: { field_2: 'A', field_3: 'Dev' }, + depth: 1, + row_count: 2, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + }, + ], + }, + ] + }) + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply(200, { + count: 2, + results: [ + { + id: 10, + order: '1.00', + field_1: 'Alice', + field_2: 'A', + field_3: 'Dev', + }, + { id: 11, order: '2.00', field_1: 'Ada', field_2: 'A', field_3: 'Dev' }, + ], + }) + + await store.dispatch('grid/fetchGroupByRowsByScrollTop', { + gridId: 1, + view: nestedView, + fields: nestedFields, + scrollTop: 0, + }) + + expect(groupByRequests).toHaveLength(1) + expect(groupByRequests[0].get('include_descendants')).toBe('true') + expect(groupByRequests[0].get('depth')).toBe(null) + expect(store.state.grid.groupBy.collapse).toEqual({ + mode: 'collapse', + paths: [], + }) + }) + + test('column viewport refines sparse group pages until rows are visible', async () => { + const sparseGroupBys = [ + { id: 10, field: 2, order: 'ASC', type: 'default', width: 200 }, + ] + const sparseView = { ...view, group_bys: sparseGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: sparseGroupBys, + groupByLayout: 'column', + count: 12000, + bufferRequestSize: 40, + rowHeight: 33, + rowPadding: 0, + windowHeight: 33, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + }, + groupBy: { + ...gridStore.state().groupBy, + pages: { + '': { + parentPath: {}, + totalSiblingCount: 120, + nodes: {}, + }, + }, + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const groupPageOffsets = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + const [{ offset }] = JSON.parse(config.params.get('parents')) + groupPageOffsets.push(offset) + const isLastPage = offset === 80 + const rowCount = isLastPage ? 1 : 100 + const rowOffset = isLastPage ? 11960 : 7960 + return [ + 200, + { + pages: [ + { + parent: {}, + groups: Array.from({ length: 40 }, (_, index) => ({ + path: { field_2: `Group ${offset + index}` }, + depth: 0, + row_count: rowCount, + sibling_index: offset + index, + row_offset: rowOffset + index * rowCount, + })), + offset, + limit: 40, + group_count: 120, + }, + ], + }, + ] + }) + + const rowRequests = [] + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply((config) => { + rowRequests.push(config.params) + return [ + 200, + { + count: 12000, + results: [ + { + id: 8001, + order: '8001.00', + field_1: 'Visible row', + field_2: 'Group 40', + }, + ], + }, + ] + }) + + await store.dispatch('grid/fetchGroupByRowsByScrollTop', { + gridId: 1, + view: sparseView, + fields, + scrollTop: 8000 * 33, + }) + + // Loading the initially visible final page changes the estimated heights of the + // earlier sparse pages. The same viewport then falls in page 40 and must be refined + // again before row ranges can be resolved. + expect(groupPageOffsets).toEqual([80, 40]) + expect(rowRequests).toHaveLength(1) + expect(rowRequests[0].get('offset')).toBe('7960') + }) + + test('column layout refresh uses expanded paging but preserves section collapse state', async () => { + const nestedFields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + { id: 3, name: 'Role', type: 'text' }, + ] + const nestedGroupBys = [ + { field: 2, order: 'ASC', type: 'default', width: 200 }, + { field: 3, order: 'ASC', type: 'default', width: 200 }, + ] + const nestedView = { ...view, group_bys: nestedGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: nestedGroupBys, + groupByLayout: 'column', + count: 2, + rowHeight: 33, + rowPadding: 0, + windowHeight: 330, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + 3: { hidden: false, order: 2 }, + }, + groupBy: { + ...gridStore.state().groupBy, + collapse: { mode: 'collapse', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const rootPage = { + parent: {}, + groups: [ + { + path: { field_2: 'A' }, + depth: 0, + row_count: 2, + children_count: 1, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + } + const childPage = { + parent: { field_2: 'A' }, + groups: [ + { + path: { field_2: 'A', field_3: 'Dev' }, + depth: 1, + row_count: 2, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + } + const groupByRequests = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + groupByRequests.push(config.params) + const pages = + config.params.get('include_descendants') === 'true' + ? [rootPage, childPage] + : config.params.get('depth') !== null + ? [childPage] + : [rootPage] + return [200, { pages }] + }) + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply(200, { + count: 2, + results: [ + { + id: 10, + order: '1.00', + field_1: 'Alice', + field_2: 'A', + field_3: 'Dev', + }, + { id: 11, order: '2.00', field_1: 'Ada', field_2: 'A', field_3: 'Dev' }, + ], + }) + + await store.dispatch('grid/refreshActiveGroupBys', { + view: nestedView, + fields: nestedFields, + scrollTop: 0, + preserveScroll: true, + }) + + expect(groupByRequests).toHaveLength(1) + expect(groupByRequests[0].get('include_descendants')).toBe('true') + expect(groupByRequests[0].get('depth')).toBe(null) + expect(store.state.grid.groupBy.collapse).toEqual({ + mode: 'collapse', + paths: [], + }) + }) +}) diff --git a/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js b/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js index 38fd321a6e..1c609920fa 100644 --- a/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js +++ b/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js @@ -1,3 +1,5 @@ +import { readFileSync } from 'fs' +import { resolve } from 'path' import { TestApp } from '@baserow/test/helpers/testApp' import { expect } from 'vitest' @@ -13,6 +15,26 @@ import { ViewSettingsCopyOptionType, } from '@baserow/modules/database/copyViewConfigurationOptionTypes' +// Read rather than imported: the i18n loader turns an imported locale file +// into compiled message ASTs, which the copy below can't be read off of. +const en = JSON.parse( + readFileSync( + resolve(process.cwd(), 'modules/database/locales/en.json'), + 'utf8' + ) +) + +const translate = (key, params = {}) => { + const translation = key + .split('.') + .reduce((value, segment) => value[segment], en) + return Object.entries(params).reduce( + (value, [name, replacement]) => + value.replace(`{${name}}`, String(replacement)), + translation + ) +} + describe('getEnabledCopyOptionKeys', () => { let testApp = null @@ -143,6 +165,19 @@ describe('getEnabledCopyOptionKeys', () => { ) ).not.toContain('field_widths') }) + + test('grid view settings label includes the group layout setting', () => { + const viewSettingsOption = new ViewSettingsCopyOptionType({ + app: { + $registry: testApp.getRegistry(), + $i18n: { t: translate }, + }, + }) + + expect(viewSettingsOption.getName(view('grid'))).toBe( + 'Settings (row height, frozen columns, row identifier, group layout)' + ) + }) }) describe('copyViewConfiguration', () => { diff --git a/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js b/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js index 829a7ab495..96a416e85c 100644 --- a/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js +++ b/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js @@ -3,6 +3,8 @@ import { ROW_HEIGHT, ADD_ROW_HEIGHT, GROUP_GAP, + GROUP_BY_LAYOUT_SECTION, + GROUP_BY_LAYOUT_COLUMN, buildLayout, pathKey, renderViewport, @@ -981,6 +983,505 @@ describe('gridGroupByRender', () => { const renderedHeader = items.find((item) => item.type === 'header') expect(renderedHeader.display).toEqual(display) }) + + describe('column layout', () => { + const fields = [textField(1), textField(2)] + const nodes = [ + { path: { field_1: 'A' }, depth: 0, row_count: 3 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 2 }, + { path: { field_1: 'A', field_2: 'Y' }, depth: 1, row_count: 1 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + { path: { field_1: 'B', field_2: 'X' }, depth: 1, row_count: 1 }, + ] + const column = { mode: 'expand', paths: [] } + + test('emits spans and gap-free row sections, no headers or per-group add rows', () => { + const layout = buildLayout({ + nodes, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + expect(layout.items.map((i) => i.type)).toEqual([ + 'groupSpan', + 'groupSpan', + 'rowSection', + 'groupSpan', + 'rowSection', + 'groupSpan', + 'groupSpan', + 'rowSection', + 'addRow', + ]) + const spans = layout.items.filter((i) => i.type === 'groupSpan') + expect(spans.map((s) => [s.depth, s.y, s.height])).toEqual([ + [0, 0, 3 * ROW_HEIGHT], + [1, 0, 2 * ROW_HEIGHT], + [1, 2 * ROW_HEIGHT, ROW_HEIGHT], + [0, 3 * ROW_HEIGHT, ROW_HEIGHT], + [1, 3 * ROW_HEIGHT, ROW_HEIGHT], + ]) + const addRow = layout.items.at(-1) + expect(addRow).toMatchObject({ + type: 'addRow', + path: {}, + y: 4 * ROW_HEIGHT, + height: ADD_ROW_HEIGHT, + }) + expect(layout.totalHeight).toBe(4 * ROW_HEIGHT + ADD_ROW_HEIGHT) + expect(layout.geometry.headerHeight).toBe(0) + }) + + test('ignores collapse state', () => { + const layout = buildLayout({ + nodes, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + collapse: { mode: 'collapse', paths: [] }, + }) + expect(layout.items.filter((i) => i.type === 'rowSection')).toHaveLength( + 3 + ) + }) + + test('sizes unloaded sibling ranges by one row per group', () => { + const layout = buildLayout({ + pages: { '': { parentPath: {}, totalSiblingCount: 50, nodes: {} } }, + collapse: column, + fields: [textField(1)], + layout: GROUP_BY_LAYOUT_COLUMN, + pageSize: 40, + }) + const placeholders = layout.items.filter( + (i) => i.type === 'groupPlaceholder' + ) + expect(placeholders.map((p) => p.height)).toEqual([ + 40 * ROW_HEIGHT, + 10 * ROW_HEIGHT, + ]) + expect( + visibleGroupDepthPageInViewport( + layout, + { scrollTop: 41 * ROW_HEIGHT, clientHeight: ROW_HEIGHT }, + 40 + ) + ).toEqual({ depth: 0, offset: 40, limit: 40 }) + }) + + test('uses the total row count to size sparse root group pages', () => { + const layout = buildLayout({ + pages: { + '': { parentPath: {}, totalSiblingCount: 1500, nodes: {} }, + }, + collapse: column, + fields: [textField(1)], + rootRowCount: 3000, + layout: GROUP_BY_LAYOUT_COLUMN, + pageSize: 40, + }) + const placeholders = layout.items.filter( + (item) => item.type === 'groupPlaceholder' + ) + + expect(placeholders).toHaveLength(Math.ceil(1500 / 40)) + expect( + placeholders.reduce((height, item) => height + item.height, 0) + ).toBe(3000 * ROW_HEIGHT) + expect(layout.totalHeight).toBe(3000 * ROW_HEIGHT + ADD_ROW_HEIGHT) + }) + + test.each([0, 1])( + 'anchors sparse loaded groups between uneven gaps at depth %s', + (depth) => { + const parentPath = depth === 0 ? {} : { field_1: 'Parent' } + const parentRowOffset = depth === 0 ? 0 : 20 + const groupField = `field_${depth + 1}` + const groupFields = fields.slice(0, depth + 1) + const makePage = (offset) => + Object.fromEntries( + Array.from({ length: 40 }, (_, index) => { + const position = offset + index + return [ + position, + { + path: { ...parentPath, [groupField]: `Group ${position}` }, + depth, + row_count: position < 40 ? 1 : 100, + sibling_index: position, + row_offset: + parentRowOffset + + (position < 40 ? position : 40 + (position - 40) * 100), + }, + ] + }) + ) + const pages = { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'Earlier' }, + depth: 0, + row_count: 20, + children_count: 1, + row_offset: 0, + }, + 1: { + path: parentPath, + depth: 0, + row_count: 8080, + children_count: 160, + row_offset: parentRowOffset, + }, + }, + }, + [pathKey(parentPath, groupFields)]: { + parentPath, + totalSiblingCount: 160, + nodes: makePage(0), + }, + } + const build = () => + buildLayout({ + pages, + fields: groupFields, + collapse: column, + layout: GROUP_BY_LAYOUT_COLUMN, + rootRowCount: parentRowOffset + 8080, + }) + const viewport = { + scrollTop: (parentRowOffset + 3980) * ROW_HEIGHT, + clientHeight: 60 * ROW_HEIGHT, + } + expect(visibleGroupPagesInViewport(build(), viewport)).toEqual([ + { parentPath, offset: 80, limit: 40 }, + ]) + + // Hydrating the estimated page must anchor it at its actual row offset. + // Otherwise it covers this viewport incorrectly and refinement stops early. + Object.assign( + pages[pathKey(parentPath, groupFields)].nodes, + makePage(80) + ) + const layout = build() + const firstLoadedSection = layout.items.find( + (item) => + item.type === 'rowSection' && item.path[groupField] === 'Group 80' + ) + expect(firstLoadedSection.y).toBe((parentRowOffset + 4040) * ROW_HEIGHT) + expect(visibleGroupPagesInViewport(layout, viewport)).toEqual([ + { parentPath, offset: 40, limit: 40 }, + ]) + expect(layout.totalHeight).toBe( + (parentRowOffset + 8080) * ROW_HEIGHT + ADD_ROW_HEIGHT + ) + + Object.assign( + pages[pathKey(parentPath, groupFields)].nodes, + makePage(40) + ) + const hydratedLayout = build() + const targetViewport = { + scrollTop: (parentRowOffset + 4000) * ROW_HEIGHT, + clientHeight: ROW_HEIGHT, + } + expect(visibleGroupPagesInViewport(hydratedLayout, viewport)).toEqual( + [] + ) + const [section] = visibleSectionsInViewport( + hydratedLayout, + targetViewport, + groupFields + ) + expect(section.absoluteRowOffset + section.startPosition).toBe( + parentRowOffset + 4000 + ) + } + ) + + test('reserves a loaded parent row span while its child page is sparse', () => { + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 100, + children_count: 2, + sibling_index: 0, + row_offset: 0, + }, + 1: { + path: { field_1: 'B' }, + depth: 0, + row_count: 1, + children_count: 1, + sibling_index: 1, + row_offset: 100, + }, + }, + }, + }, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const [a, b] = layout.items.filter( + (item) => item.type === 'groupSpan' && item.depth === 0 + ) + + expect(b.y).toBe(a.y + a.height) + expect(layout.totalHeight).toBe(101 * ROW_HEIGHT + ADD_ROW_HEIGHT) + expect( + visibleGroupPagesInViewport( + layout, + { scrollTop: 50 * ROW_HEIGHT, clientHeight: ROW_HEIGHT }, + 40 + ) + ).toEqual([{ parentPath: { field_1: 'A' }, offset: 0, limit: 40 }]) + }) + + test('distributes missing parent rows across multiple child placeholder pages', () => { + const parentPath = { field_1: 'A' } + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 1, + nodes: { + 0: { + path: parentPath, + depth: 0, + row_count: 100, + children_count: 50, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + [pathKey(parentPath, fields)]: { + parentPath, + totalSiblingCount: 50, + nodes: { + 0: { + path: { ...parentPath, field_2: 'Loaded' }, + depth: 1, + row_count: 10, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + }, + collapse: column, + fields, + pageSize: 40, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const placeholders = layout.items.filter( + (item) => item.type === 'groupPlaceholder' + ) + + expect(placeholders).toHaveLength(2) + expect( + placeholders.reduce((height, item) => height + item.height, 0) + ).toBe(90 * ROW_HEIGHT) + expect(layout.totalHeight).toBe(100 * ROW_HEIGHT + ADD_ROW_HEIGHT) + }) + + test('uses the absolute row offset for identifiers after a sparse branch', () => { + const bPath = { field_1: 'B' } + const bLeafPath = { ...bPath, field_2: 'Only' } + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 100, + children_count: 2, + sibling_index: 0, + row_offset: 0, + }, + 1: { + path: bPath, + depth: 0, + row_count: 1, + children_count: 1, + sibling_index: 1, + row_offset: 100, + }, + }, + }, + [pathKey(bPath, fields)]: { + parentPath: bPath, + totalSiblingCount: 1, + nodes: { + 0: { + path: bLeafPath, + depth: 1, + row_count: 1, + sibling_index: 0, + row_offset: 100, + }, + }, + }, + }, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const bSection = layout.items.find( + (item) => item.type === 'rowSection' && item.path.field_1 === 'B' + ) + const [renderedRow] = renderViewport({ + layout, + sectionRows: buildSectionRows( + [{ path: bLeafPath, rows: [{ id: 101 }] }], + fields + ), + viewport: { scrollTop: bSection.y, clientHeight: ROW_HEIGHT }, + fields, + }).filter((item) => item.type === 'row') + + expect(renderedRow.globalRowOffset).toBe(100) + }) + + test('marks only the last row of each group as a column boundary', () => { + const f = [textField(1)] + const groupedNodes = [ + { path: { field_1: 'A' }, depth: 0, row_count: 2 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + ] + const sectionRows = buildSectionRows( + [ + { path: { field_1: 'A' }, rows: [{ id: 1 }, { id: 2 }] }, + { path: { field_1: 'B' }, rows: [{ id: 3 }] }, + ], + f + ) + const renderRows = (layout) => + renderViewport({ + layout, + sectionRows, + viewport: { scrollTop: 0, clientHeight: 1000 }, + fields: f, + }).filter((item) => item.type === 'row') + + const columnRows = renderRows( + buildLayout({ + nodes: groupedNodes, + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + ) + expect(columnRows.map((item) => item.groupEnd)).toEqual([ + false, + true, + true, + ]) + + const sectionLayoutRows = renderRows( + buildLayout({ + nodes: groupedNodes, + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_SECTION, + }) + ) + expect(sectionLayoutRows.every((item) => item.groupEnd === false)).toBe( + true + ) + }) + + test('renderViewport clips spans and renders unloaded ranges as one placeholder block', () => { + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 3, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 4, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + }, + collapse: column, + fields: [textField(1)], + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const items = renderViewport({ + layout, + sectionRows: new Map(), + fields: [textField(1)], + viewport: { scrollTop: ROW_HEIGHT, clientHeight: 5 * ROW_HEIGHT }, + }) + expect(items[0]).toMatchObject({ + type: 'groupSpan', + y: 0, + height: 4 * ROW_HEIGHT, + }) + expect(items.filter((i) => i.type === 'placeholder')).toHaveLength(3) + expect( + items.find((i) => i.type === 'groupRangePlaceholder') + ).toMatchObject({ y: 4 * ROW_HEIGHT, height: 2 * ROW_HEIGHT }) + expect(items.some((i) => i.type === 'groupSkeleton')).toBe(false) + }) + + test('resolveGroupByRowMoveTarget ignores spans and resolves the end-of-group slot', () => { + const f = [textField(1)] + const rows = [{ id: 1 }, { id: 2 }] + const layout = buildLayout({ + nodes: [{ path: { field_1: 'A' }, depth: 0, row_count: 2 }], + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const sectionRows = buildSectionRows( + [{ path: { field_1: 'A' }, rows }], + f + ) + const common = { + layout, + sectionRows, + fields: f, + sourcePath: { field_1: 'A' }, + allowCrossGroup: false, + } + expect( + resolveGroupByRowMoveTarget({ ...common, contentY: 5 }) + ).toMatchObject({ before: rows[0], position: 0 }) + expect( + resolveGroupByRowMoveTarget({ ...common, contentY: 2 * ROW_HEIGHT - 3 }) + ).toMatchObject({ before: null, position: 2 }) + }) + + test('section layout output is unchanged', () => { + const section = buildLayout({ nodes, collapse: column, fields }) + const explicit = buildLayout({ + nodes, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_SECTION, + }) + expect(explicit.items).toEqual(section.items) + expect(section.items.map((i) => i.type)).toContain('header') + expect(section.items.some((i) => i.type === 'groupSpan')).toBe(false) + }) + }) }) describe('buildLayout cycle safety', () => { diff --git a/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js b/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js new file mode 100644 index 0000000000..fc53866623 --- /dev/null +++ b/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js @@ -0,0 +1,29 @@ +import { fitGroupByWidths } from '@baserow/modules/database/utils/gridGroupByWidths' + +describe('fitGroupByWidths', () => { + test('keeps configured widths when they fit or the grid is not measured', () => { + const groupBys = [{ width: 90 }, { width: 180 }, { width: 270 }] + + expect(fitGroupByWidths(groupBys, 600, 78)).toEqual([90, 180, 270]) + expect(fitGroupByWidths(groupBys, null, 78)).toEqual([90, 180, 270]) + }) + + test('scales only space above the minimum and does not mutate configured widths', () => { + const groupBys = [ + { width: 78 }, + { width: 120 }, + { width: 200 }, + { width: 260 }, + { width: 342 }, + ] + const configuredWidths = groupBys.map(({ width }) => width) + const renderedWidths = fitGroupByWidths(groupBys, 652, 78) + + expect(renderedWidths[0]).toBe(78) + expect(renderedWidths.every((width) => width >= 78)).toBe(true) + expect( + renderedWidths.reduce((total, width) => total + width, 0) + ).toBeCloseTo(652) + expect(groupBys.map(({ width }) => width)).toEqual(configuredWidths) + }) +})