From be57a256482ecec2931196991ce48394933b964c Mon Sep 17 00:00:00 2001 From: Davide Silvestri <75379892+silvestrid@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:30:22 +0200 Subject: [PATCH 1/4] fix: compact expired realtime replay events (#6074) * feat: compact realtime history into audience summaries * test: cover retention guard and replay refresh reasons Adds the settings-import test for a non-positive retention window, pins the refresh_reason of three replay refusals, and names the per-row channel group as the summary growth driver. --- backend/src/baserow/config/settings/base.py | 8 + backend/src/baserow/ws/history.py | 224 +++++++++ .../migrations/0003_realtime_event_history.py | 168 +++++++ backend/src/baserow/ws/models.py | 78 ++- backend/src/baserow/ws/realtime_events.py | 306 ++++++----- backend/src/baserow/ws/replay.py | 8 +- backend/src/baserow/ws/tasks.py | 3 +- .../config/test_settings_validation.py | 39 ++ backend/tests/baserow/ws/conftest.py | 17 +- .../baserow/ws/test_ws_history_migration.py | 262 ++++++++++ .../baserow/ws/test_ws_index_migration.py | 16 +- .../baserow/ws/test_ws_realtime_cleanup.py | 116 ++++- .../baserow/ws/test_ws_realtime_events.py | 42 +- .../baserow/ws/test_ws_realtime_history.py | 474 ++++++++++++++++++ .../baserow/ws/test_ws_replay_executor.py | 4 +- .../baserow/ws/test_ws_storage_telemetry.py | 3 +- ...esh_prompts_when_reconnecting_after_o.json | 9 + docker-compose.no-caddy.yml | 1 + docker-compose.yml | 1 + docs/installation/configuration.md | 3 +- docs/installation/monitoring.md | 17 +- docs/technical/websockets.md | 61 ++- 22 files changed, 1650 insertions(+), 210 deletions(-) create mode 100644 backend/src/baserow/ws/history.py create mode 100644 backend/src/baserow/ws/migrations/0003_realtime_event_history.py create mode 100644 backend/tests/baserow/config/test_settings_validation.py create mode 100644 backend/tests/baserow/ws/test_ws_history_migration.py create mode 100644 backend/tests/baserow/ws/test_ws_realtime_history.py create mode 100644 changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json diff --git a/backend/src/baserow/config/settings/base.py b/backend/src/baserow/config/settings/base.py index 5e63780a9e..dab26cfcdb 100644 --- a/backend/src/baserow/config/settings/base.py +++ b/backend/src/baserow/config/settings/base.py @@ -1863,3 +1863,11 @@ def install_cachalot(): BASEROW_REALTIME_REPLAY_MAX_EVENTS = int( os.getenv("BASEROW_REALTIME_REPLAY_MAX_EVENTS", 200) ) + +REALTIME_REPLAY_RETENTION_HOURS = int( + os.getenv("BASEROW_REALTIME_REPLAY_RETENTION_HOURS", 24) +) +if REALTIME_REPLAY_RETENTION_HOURS <= 0: + raise ImproperlyConfigured( + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS must be a positive integer." + ) diff --git a/backend/src/baserow/ws/history.py b/backend/src/baserow/ws/history.py new file mode 100644 index 0000000000..95107cb6ba --- /dev/null +++ b/backend/src/baserow/ws/history.py @@ -0,0 +1,224 @@ +"""Keep compact delivery evidence when full realtime events expire.""" + +import json +from time import monotonic + +from django.db import connection, transaction + +# Preserve delivery and recovery rules, including individual recipients' event +# types, but discard business data. Socket exclusions are summarized separately. +_ROUTING_SQL = """ +jsonb_build_object( + 'type', payload->'type', + 'send_to_all_users', COALESCE(payload->'send_to_all_users', 'false'), + 'user_ids', ( + SELECT COALESCE(jsonb_agg(DISTINCT value ORDER BY value), '[]') + FROM jsonb_array_elements( + CASE WHEN jsonb_typeof(payload->'user_ids') = 'array' + THEN payload->'user_ids' ELSE '[]' END + ) + ), + 'exclude_user_ids', ( + SELECT COALESCE(jsonb_agg(DISTINCT value ORDER BY value), '[]') + FROM jsonb_array_elements( + CASE WHEN jsonb_typeof(payload->'exclude_user_ids') = 'array' + THEN payload->'exclude_user_ids' ELSE '[]' END + ) + ), + 'payload', jsonb_build_object('type', payload #> '{payload,type}'), + 'payload_map', ( + SELECT COALESCE(jsonb_object_agg( + recipient, jsonb_build_object('type', message->'type') + ), '{}') + FROM jsonb_each( + CASE WHEN jsonb_typeof(payload->'payload_map') = 'object' + THEN payload->'payload_map' ELSE '{}' END + ) AS messages(recipient, message) + ) +) +""" + +_SELECT_EXPIRED_SQL = f""" +WITH candidates AS MATERIALIZED ( + SELECT id, channel_group, payload, target_user_ids, all_users + FROM ws_realtime_events WHERE created_at < %s + ORDER BY created_at, id LIMIT %s FOR UPDATE SKIP LOCKED +), routes AS MATERIALIZED ( + SELECT id, channel_group, {_ROUTING_SQL} AS route, + target_user_ids, all_users, + CASE WHEN jsonb_typeof(payload->'ignore_web_socket_id') = 'string' + THEN payload->>'ignore_web_socket_id' END AS ignored_socket + FROM candidates +) +SELECT id, sha256(convert_to(jsonb_build_array( + channel_group, route, target_user_ids, all_users)::text, 'UTF8')), + channel_group, route::text, target_user_ids, all_users, ignored_socket +FROM routes +""".strip() # noqa: S608 + +_UPSERT_HISTORY_SQL = """ +INSERT INTO ws_realtime_event_history_summary AS history ( + route_key, channel_group, payload, target_user_ids, all_users, + latest_event_id, latest_socket_id, previous_event_id, previous_socket_id +) +SELECT decode(route_key, 'hex'), channel_group, payload_text::jsonb, + target_user_ids, all_users, + latest_event_id, latest_socket_id, previous_event_id, previous_socket_id +FROM jsonb_to_recordset(%s::jsonb) AS batch( + route_key text, channel_group text, payload_text text, + target_user_ids integer[], all_users boolean, + latest_event_id bigint, latest_socket_id text, + previous_event_id bigint, previous_socket_id text +) +ORDER BY route_key +ON CONFLICT (route_key) DO UPDATE SET + (latest_event_id, latest_socket_id, previous_event_id, previous_socket_id) = ( + SELECT (array_agg(event_id ORDER BY event_id DESC))[1], + (array_agg(socket_id ORDER BY event_id DESC))[1], + (array_agg(event_id ORDER BY event_id DESC))[2], + (array_agg(socket_id ORDER BY event_id DESC))[2] + FROM ( + SELECT DISTINCT ON (socket_id) event_id, socket_id + FROM (VALUES + (history.latest_event_id, history.latest_socket_id), + (history.previous_event_id, history.previous_socket_id), + (excluded.latest_event_id, excluded.latest_socket_id), + (excluded.previous_event_id, excluded.previous_socket_id) + ) AS evidence(event_id, socket_id) + WHERE event_id IS NOT NULL + ORDER BY socket_id, event_id DESC + ) AS latest_per_socket + ) +RETURNING route_key, channel_group, payload::text, target_user_ids, all_users +""".strip() + +_DELETE_EXPIRED_SQL = "DELETE FROM ws_realtime_events WHERE id = ANY(%s)" + +_ADVANCE_HIGH_WATER_SQL = """ +UPDATE ws_realtime_event_history_state +SET compacted_event_id = GREATEST(compacted_event_id, %s) +WHERE id = 1 +""".strip() + +_CONFIGURE_TRANSACTION_SQL = """ +SELECT + set_config('statement_timeout', CASE + WHEN current_setting('statement_timeout')::interval = interval '0' + OR current_setting('statement_timeout')::interval > %s::interval + THEN %s ELSE current_setting('statement_timeout') END, true), + set_config('lock_timeout', CASE + WHEN current_setting('lock_timeout')::interval = interval '0' + OR current_setting('lock_timeout')::interval > %s::interval + THEN %s ELSE current_setting('lock_timeout') END, true) +""".strip() + + +class _CleanupDeadlineExceeded(Exception): + pass + + +def _make_executor(cursor, deadline, statement_timeout_ms, lock_timeout_ms): + """Apply transaction-local limits, tightening them as the budget runs out.""" + + configured_timeout_ms = None + + def execute(sql, params=None): + nonlocal configured_timeout_ms + remaining_ms = int((deadline - monotonic()) * 1000) + if remaining_ms <= 0: + raise _CleanupDeadlineExceeded + timeout_ms = min(statement_timeout_ms, remaining_ms) + if timeout_ms != configured_timeout_ms: + statement_limit = f"{timeout_ms}ms" + lock_limit = f"{lock_timeout_ms}ms" + cursor.execute( + _CONFIGURE_TRANSACTION_SQL, + [statement_limit, statement_limit, lock_limit, lock_limit], + ) + configured_timeout_ms = timeout_ms + if monotonic() >= deadline: + raise _CleanupDeadlineExceeded + cursor.execute(sql, params) + if monotonic() >= deadline: + raise _CleanupDeadlineExceeded + + return execute + + +def _summarize_candidates(candidates): + """Keep the newest two different socket exclusions for each exact audience. + + Any client can exclude at most one socket. Its latest relevant expired event + is therefore either the newest event or the newest with a different socket. + """ + + audiences = {} + for event_id, key, channel, payload, targets, all_users, socket in candidates: + key = bytes(key).hex() + audience = (channel, payload, targets, all_users) + if key not in audiences: + audiences[key] = (audience, {}) + original, sockets = audiences[key] + if original != audience: + raise RuntimeError("Realtime history route hash collision") + sockets[socket] = max(event_id, sockets.get(socket, 0)) + + summaries = [] + for key, ((channel, payload, targets, all_users), sockets) in audiences.items(): + latest, *rest = sorted(sockets.items(), key=lambda pair: pair[1], reverse=True) + previous_socket, previous_id = rest[0] if rest else (None, None) + summaries.append( + { + "route_key": key, + "channel_group": channel, + "payload_text": payload, + "target_user_ids": targets, + "all_users": all_users, + "latest_event_id": latest[1], + "latest_socket_id": latest[0], + "previous_event_id": previous_id, + "previous_socket_id": previous_socket, + } + ) + return summaries + + +def compact_events_batch( + cutoff, deadline, *, batch_size, statement_timeout_ms, lock_timeout_ms +): + """Publish expired-event evidence and delete full payloads in one commit.""" + + try: + with transaction.atomic(durable=True), connection.cursor() as cursor: + execute = _make_executor( + cursor, deadline, statement_timeout_ms, lock_timeout_ms + ) + execute("SELECT ws_initialize_realtime_history()") + execute(_SELECT_EXPIRED_SQL, [cutoff, batch_size]) + candidates = cursor.fetchall() + if not candidates: + return 0 + summaries = _summarize_candidates(candidates) + execute(_UPSERT_HISTORY_SQL, [json.dumps(summaries)]) + expected = {entry["route_key"]: entry for entry in summaries} + for key, channel, payload, targets, all_users in cursor.fetchall(): + entry = expected[bytes(key).hex()] + if (channel, payload, targets, all_users) != ( + entry["channel_group"], + entry["payload_text"], + entry["target_user_ids"], + entry["all_users"], + ): + # ON CONFLICT serializes concurrent summaries. A hash collision + # must roll back that merge as well as leave originals intact. + raise RuntimeError("Realtime history route hash collision") + ids = [row[0] for row in candidates] + execute(_DELETE_EXPIRED_SQL, [ids]) + deleted = cursor.rowcount + execute(_ADVANCE_HIGH_WATER_SQL, [max(ids)]) + if cursor.rowcount != 1: + raise RuntimeError("Realtime history state is missing") + return deleted + except _CleanupDeadlineExceeded: + # Roll back the summary and deletion together before reporting no progress. + return 0 diff --git a/backend/src/baserow/ws/migrations/0003_realtime_event_history.py b/backend/src/baserow/ws/migrations/0003_realtime_event_history.py new file mode 100644 index 0000000000..381d7c0cad --- /dev/null +++ b/backend/src/baserow/ws/migrations/0003_realtime_event_history.py @@ -0,0 +1,168 @@ +# Store expired delivery evidence separately so cleanup can delete full payloads. +# The PostgreSQL helper initializes a safe replay boundary on deployment and after +# an UNLOGGED reset. Existing event data and indexes remain unchanged; drain old +# cleanup and update readers before enabling the new cleanup. + +from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex +from django.db import migrations, models, transaction + +INITIALIZE_HISTORY = """ +CREATE OR REPLACE FUNCTION ws_initialize_realtime_history() RETURNS void +LANGUAGE plpgsql AS $function$ +DECLARE + event_sequence regclass; + sequence_value bigint; + sequence_called boolean; + history_floor bigint; +BEGIN + -- Called by migration, replay and cleanup when history state is missing. + -- UNLOGGED history can disappear after a crash; cursors below its new floor + -- must refresh because neither full events nor summaries prove their gap. + IF EXISTS (SELECT 1 FROM ws_realtime_event_history_state WHERE id = 1) THEN + RETURN; + END IF; + -- Wait for pending INSERTs before trusting their allocated IDs as a boundary. + LOCK TABLE ws_realtime_events IN SHARE MODE; + IF EXISTS (SELECT 1 FROM ws_realtime_event_history_state WHERE id = 1) THEN + RETURN; + END IF; + event_sequence := pg_get_serial_sequence('ws_realtime_events', 'id')::regclass; + -- ws.0002 keeps this sequence LOGGED. Cached IDs could otherwise be inserted + -- below the new floor after this transaction releases its table lock. + IF NOT EXISTS ( + SELECT 1 FROM pg_sequence WHERE seqrelid = event_sequence AND seqcache = 1 + ) THEN + RAISE EXCEPTION 'Realtime history requires CACHE 1 for the event sequence'; + END IF; + EXECUTE format('SELECT last_value, is_called FROM %s', event_sequence) + INTO sequence_value, sequence_called; + history_floor := CASE WHEN sequence_called THEN sequence_value ELSE 0 END; + INSERT INTO ws_realtime_event_history_state (id, floor, compacted_event_id) + VALUES (1, history_floor, history_floor) + ON CONFLICT (id) DO NOTHING; +END; +$function$; +""" + + +def _set_timeouts(cursor): + for setting, timeout in [("lock_timeout", "1s"), ("statement_timeout", "3s")]: + cursor.execute( + "SELECT set_config(%s, %s, true) WHERE " + "current_setting(%s)::interval = interval '0' OR " + "current_setting(%s)::interval > %s::interval", + [setting, timeout, setting, setting, timeout], + ) + + +def forwards(apps, schema_editor): + """Install history initialization; the scheduled task performs compaction.""" + + db = schema_editor.connection + with transaction.atomic(using=db.alias), db.cursor() as cursor: + _set_timeouts(cursor) + cursor.execute("ALTER TABLE ws_realtime_event_history_summary SET UNLOGGED") + cursor.execute("ALTER TABLE ws_realtime_event_history_state SET UNLOGGED") + cursor.execute(INITIALIZE_HISTORY) + cursor.execute("SELECT ws_initialize_realtime_history()") + + +def backwards(apps, schema_editor): + # Django drops only the two new tables afterward. Remaining full events and + # the durable sequence are preserved; rollback cannot restore deleted events. + # Disable replay and re-establish client baselines before restoring old readers. + db = schema_editor.connection + with transaction.atomic(using=db.alias), db.cursor() as cursor: + _set_timeouts(cursor) + cursor.execute("DROP FUNCTION IF EXISTS ws_initialize_realtime_history()") + + +class Migration(migrations.Migration): + dependencies = [("ws", "0002_realtime_event_indexes")] + + operations = [ + migrations.CreateModel( + name="RealtimeEventHistorySummary", + fields=[ + ("route_key", models.BinaryField(primary_key=True, serialize=False)), + ("channel_group", models.TextField()), + ("payload", models.JSONField()), + ( + "target_user_ids", + ArrayField( + models.IntegerField(), + default=list, + db_default=[], + editable=False, + ), + ), + ( + "all_users", + models.BooleanField( + default=False, db_default=False, editable=False + ), + ), + ("latest_event_id", models.BigIntegerField()), + ("latest_socket_id", models.TextField(null=True)), + ("previous_event_id", models.BigIntegerField(null=True)), + ("previous_socket_id", models.TextField(null=True)), + ], + options={ + "db_table": "ws_realtime_event_history_summary", + "indexes": [ + models.Index( + fields=["channel_group"], + name="ws_history_summary_channel_idx", + ), + GinIndex( + fields=["target_user_ids"], + condition=models.Q(channel_group="users"), + name="ws_history_summary_targets_idx", + ), + models.Index( + fields=["all_users"], + condition=models.Q(channel_group="users", all_users=True), + name="ws_history_summary_all_idx", + ), + ], + "constraints": [ + models.CheckConstraint( + condition=models.Q( + previous_event_id__isnull=True, + previous_socket_id__isnull=True, + ) + | ( + models.Q(previous_event_id__isnull=False) + & models.Q( + previous_event_id__lt=models.F("latest_event_id") + ) + ), + name="ws_history_previous_pair", + ) + ], + }, + ), + migrations.CreateModel( + name="RealtimeEventHistoryState", + fields=[ + ( + "id", + models.PositiveSmallIntegerField( + primary_key=True, serialize=False, db_default=1 + ), + ), + ("floor", models.BigIntegerField(db_default=0)), + ("compacted_event_id", models.BigIntegerField(db_default=0)), + ], + options={ + "db_table": "ws_realtime_event_history_state", + "constraints": [ + models.CheckConstraint( + condition=models.Q(id=1), name="ws_history_state_singleton" + ) + ], + }, + ), + migrations.RunPython(forwards, backwards), + ] diff --git a/backend/src/baserow/ws/models.py b/backend/src/baserow/ws/models.py index 71968a8ca8..155dc641b2 100644 --- a/backend/src/baserow/ws/models.py +++ b/backend/src/baserow/ws/models.py @@ -10,11 +10,8 @@ class RealtimeEvent(models.Model): channel_group = models.TextField() payload = models.JSONField() created_at = models.DateTimeField(auto_now_add=True) - # PostgreSQL trigger ws_realtime_event_targets_before_write populates these - # from payload on INSERT or updates to payload/channel_group. It calls - # ws_set_realtime_event_targets() from migrations/0002_realtime_event_indexes.py, - # including for older workers during a rolling deployment, so replay can use - # recipient indexes instead of scanning JSON. + # The ws_set_realtime_event_targets() pg-trigger populates these from payload on + # insert/update target_user_ids = ArrayField( models.IntegerField(), default=list, db_default=[], editable=False ) @@ -44,3 +41,74 @@ class Meta: name="ws_realtime_created_id_idx", ), ] + + +class RealtimeEventHistorySummary(models.Model): + """Expired event maxima for an exact audience, without business payloads.""" + + UNLOGGED = True + + route_key = models.BinaryField(primary_key=True) + channel_group = models.TextField() + # Routing fields only; the two socket values are stored separately so new + # browser sessions do not create new summary rows for the same audience. + payload = models.JSONField() + target_user_ids = ArrayField( + models.IntegerField(), default=list, db_default=[], editable=False + ) + all_users = models.BooleanField(default=False, db_default=False, editable=False) + latest_event_id = models.BigIntegerField() + latest_socket_id = models.TextField(null=True) + # The latest event with a different ignored socket. A null socket means the + # event ignores nobody; only a null event ID means this second pair is absent. + previous_event_id = models.BigIntegerField(null=True) + previous_socket_id = models.TextField(null=True) + + class Meta: + db_table = "ws_realtime_event_history_summary" + # Keep changing maxima out of indexes so summary updates can use HOT. + indexes = [ + models.Index( + fields=["channel_group"], name="ws_history_summary_channel_idx" + ), + GinIndex( + fields=["target_user_ids"], + condition=models.Q(channel_group="users"), + name="ws_history_summary_targets_idx", + ), + models.Index( + fields=["all_users"], + condition=models.Q(channel_group="users", all_users=True), + name="ws_history_summary_all_idx", + ), + ] + constraints = [ + models.CheckConstraint( + condition=models.Q( + previous_event_id__isnull=True, previous_socket_id__isnull=True + ) + | ( + models.Q(previous_event_id__isnull=False) + & models.Q(previous_event_id__lt=models.F("latest_event_id")) + ), + name="ws_history_previous_pair", + ) + ] + + +class RealtimeEventHistoryState(models.Model): + """Known history boundary and the high-water mark after full rows are removed.""" + + UNLOGGED = True + + id = models.PositiveSmallIntegerField(primary_key=True, db_default=1) + floor = models.BigIntegerField(db_default=0) + compacted_event_id = models.BigIntegerField(db_default=0) + + class Meta: + db_table = "ws_realtime_event_history_state" + constraints = [ + models.CheckConstraint( + condition=models.Q(id=1), name="ws_history_state_singleton" + ) + ] diff --git a/backend/src/baserow/ws/realtime_events.py b/backend/src/baserow/ws/realtime_events.py index 5f3110737f..d6134e4916 100644 --- a/backend/src/baserow/ws/realtime_events.py +++ b/backend/src/baserow/ws/realtime_events.py @@ -1,14 +1,14 @@ from __future__ import annotations +import json from dataclasses import dataclass from datetime import timedelta from time import monotonic from typing import TYPE_CHECKING, Any, Optional from django.conf import settings -from django.db import connection, transaction -from django.db.models import Max, Q -from django.db.models.functions import Coalesce +from django.db import connection +from django.db.models import Case, F, Q, When from django.db.models.query import QuerySet from django.utils import timezone @@ -25,13 +25,13 @@ from baserow.ws.consumers import SubscribedPages from baserow.ws.models import RealtimeEvent -REALTIME_EVENTS_RETENTION = timedelta(hours=24) -REALTIME_EVENTS_CLEANUP_INTERVAL_MINUTES = 1 +REALTIME_EVENTS_CLEANUP_INTERVAL_MINUTES = 10 REALTIME_EVENTS_CLEANUP_BATCH_SIZE = 5000 -REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS = 30 +# Leave headroom below Celery's default five-minute soft task limit. +REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS = 4 * 60 REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS = 3000 REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS = 250 -REALTIME_EVENTS_CLEANUP_LOCK_SECONDS = 120 +REALTIME_EVENTS_CLEANUP_LOCK_SECONDS = REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS + 90 # ``replay_events`` cursor sentinels. Must match the constants in # web-frontend/modules/core/plugins/realtimeProtocol.js. @@ -54,6 +54,7 @@ class ReplayEventsResult: # Transient infrastructure failure, rather than an unrecoverable replay gap. # Older clients still receive the force-refresh fallback. retry_after_ms: int | None = None + refresh_reason: str | None = None class RealtimeEventHandler: @@ -65,6 +66,12 @@ def is_recording_enabled() -> bool: return settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS > 0 + @staticmethod + def get_replay_retention() -> timedelta: + """Maximum age of full events accepted by replay and retained by cleanup.""" + + return timedelta(hours=settings.REALTIME_REPLAY_RETENTION_HOURS) + @staticmethod def record_events( events_data: list[tuple[str, dict[str, Any]]], @@ -188,46 +195,17 @@ def cleanup_old_realtime_events( @staticmethod def _delete_realtime_events_batch(cutoff, deadline) -> int: - """Delete one bounded oldest-first batch and commit before returning.""" - - # A caller must not accidentally turn many batches into one transaction. - # RealtimeEvent is UNLOGGED, so its storage is always on the primary DB. - with transaction.atomic(durable=True), connection.cursor() as cursor: - remaining_ms = int((deadline - monotonic()) * 1000) - if remaining_ms <= 0: - return 0 - statement_timeout = ( - f"{min(REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS, remaining_ms)}ms" - ) - lock_timeout = f"{REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS}ms" - # Both limits are transaction-local and must preserve stricter - # database/operator settings. These expressions only read settings. - cursor.execute( - "SELECT " - "set_config('statement_timeout', CASE WHEN " - "current_setting('statement_timeout')::interval = interval '0' OR " - "current_setting('statement_timeout')::interval > %s::interval " - "THEN %s ELSE current_setting('statement_timeout') END, true), " - "set_config('lock_timeout', CASE WHEN " - "current_setting('lock_timeout')::interval = interval '0' OR " - "current_setting('lock_timeout')::interval > %s::interval " - "THEN %s ELSE current_setting('lock_timeout') END, true)", - [statement_timeout, statement_timeout, lock_timeout, lock_timeout], - ) - if monotonic() >= deadline: - return 0 - # The (created_at, id) index finds the oldest bounded candidate set. - # This ephemeral log has no model deletion hooks or relationships; - # delete directly without loading payloads or collecting model rows. - cursor.execute( - "WITH expired AS MATERIALIZED (" - "SELECT id FROM ws_realtime_events WHERE created_at < %s " - "ORDER BY created_at, id LIMIT %s FOR UPDATE SKIP LOCKED" - ") DELETE FROM ws_realtime_events AS event " - "USING expired WHERE event.id = expired.id", - [cutoff, REALTIME_EVENTS_CLEANUP_BATCH_SIZE], - ) - return cursor.rowcount + """Save expired delivery evidence and delete one independently committed batch.""" + + from baserow.ws.history import compact_events_batch + + return compact_events_batch( + cutoff, + deadline, + batch_size=REALTIME_EVENTS_CLEANUP_BATCH_SIZE, + statement_timeout_ms=REALTIME_EVENTS_CLEANUP_STATEMENT_TIMEOUT_MS, + lock_timeout_ms=REALTIME_EVENTS_CLEANUP_LOCK_TIMEOUT_MS, + ) @staticmethod def get_page_group_names(pages: "SubscribedPages") -> list[str]: @@ -258,91 +236,154 @@ def get_replay_events_result( last_seen_id: int, web_socket_id: Optional[str], ) -> ReplayEventsResult: - """ - Decide how a realtime client should catch up after connecting. Only - called when replay recording is enabled — well-behaved clients skip - ``replay_events`` when the authentication handshake says it is off, - and ``_handle_replay_events`` drops the message otherwise. + """Distinguish unchanged, replayable and expired history in one snapshot.""" - :param user_id: The id of the reconnecting user. - :param page_group_names: Page channel group names the user is - subscribed to. Must not include ``"users"`` — that channel is - added unconditionally by - ``get_users_channel_live_delivery_filter``. - :param last_seen_id: ``FIRST_CONNECT_CURSOR`` for a fresh connection, - ``NO_REPLAY_AVAILABLE`` for a reconnect with no usable high-water - mark, or a positive event id the client last saw. - :param web_socket_id: The client's persistent web socket id, used to - exclude events the client itself originated. - :returns: A result containing replay events or a force-refresh instruction. - """ + from baserow.ws.models import RealtimeEvent - if last_seen_id == FIRST_CONNECT_CURSOR: - # Connecting for the first time - clients only need the latest event id to - # know where to start for future reconnects. + def refresh(reason): return ReplayEventsResult( - force_refresh=False, - latest_event_id=RealtimeEventHandler.get_latest_event_id(), - replay_events=[], + True, NO_REPLAY_AVAILABLE, [], refresh_reason=reason ) if last_seen_id == NO_REPLAY_AVAILABLE: - # Reconnect without a high-water mark — we can't prove what was - # missed, so the client has to refresh. + return refresh("missing_cursor") + if last_seen_id == FIRST_CONNECT_CURSOR: return ReplayEventsResult( - force_refresh=True, - latest_event_id=NO_REPLAY_AVAILABLE, - replay_events=[], + False, RealtimeEventHandler.get_latest_event_id(), [] ) - replay_window_events = list( - RealtimeEventHandler.get_replay_window( - user_id, page_group_names, last_seen_id, web_socket_id - ) + rows = RealtimeEventHandler._get_replay_snapshot( + user_id, + page_group_names, + last_seen_id, + web_socket_id, ) - - if replay_window_events: - # The first event must be the last seen event, and we must not exceed the - # replay limit with the remaining events, for a successful replay. - baseline = replay_window_events[0] - latest_event_id = replay_window_events[-1].id - replay_events = replay_window_events[1:] - max_events = settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS - can_replay = ( - baseline.id == last_seen_id - # Cleanup can skip a locked expired baseline while deleting - # later expired events. That surviving row cannot prove that - # the replay window is complete. - and baseline.created_at >= timezone.now() - REALTIME_EVENTS_RETENTION - and len(replay_events) <= max_events - ) - if can_replay: - return ReplayEventsResult( - force_refresh=False, - latest_event_id=latest_event_id, - replay_events=replay_events, + if not rows or last_seen_id < rows[0][0]: + return refresh("unknown_history") + latest_event_id = rows[0][1] + if last_seen_id > latest_event_id: + return refresh("cursor_ahead") + + if rows[0][2]: + return refresh("expired_payload") + + cutoff = timezone.now() - RealtimeEventHandler.get_replay_retention() + events = [] + for _, _, _, event_id, channel_group, payload, created_at in rows: + if event_id is None: + continue + if created_at < cutoff: + return refresh("expired_payload") + events.append( + RealtimeEvent( + id=event_id, + channel_group=channel_group, + payload=json.loads(payload) + if isinstance(payload, str) + else payload, + created_at=created_at, ) - - # Empty window or unable to anchor against ``last_seen_id`` — the - # cursor has expired, the client missed too many events, or the - # filter excluded the baseline. Force a refresh. + ) + if len(events) > settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS: + return refresh("event_limit") + # Do not acknowledge unrelated higher IDs: a lower relevant INSERT may + # still be uncommitted. Retain the original cursor when nothing replays. return ReplayEventsResult( - force_refresh=True, - latest_event_id=NO_REPLAY_AVAILABLE, - replay_events=[], + False, events[-1].id if events else last_seen_id, events ) + @staticmethod + def _initialize_realtime_history(): + # Usually a cheap existence check. After an UNLOGGED reset the function + # waits for in-flight inserts before recording a conservative loss floor. + with connection.cursor() as cursor: + cursor.execute("SELECT ws_initialize_realtime_history()") + @staticmethod def get_latest_event_id() -> int: - """ - Return the latest persisted realtime event id. + """Return the high-water mark, including evicted history.""" + + with connection.cursor() as cursor: + sql = ( + "SELECT GREATEST(floor, compacted_event_id, " + "COALESCE((SELECT id FROM ws_realtime_events " + "ORDER BY id DESC LIMIT 1), 0)) " + "FROM ws_realtime_event_history_state WHERE id = 1" + ) + cursor.execute(sql) + row = cursor.fetchone() + if row is None: + RealtimeEventHandler._initialize_realtime_history() + cursor.execute(sql) + row = cursor.fetchone() + return row[0] - :return: The highest event id, or ``0`` when no events exist. - """ + @staticmethod + def _get_replay_snapshot(user_id, page_group_names, last_seen_id, web_socket_id): + """Read expired evidence, payloads and loss state together across cleanup.""" - from baserow.ws.models import RealtimeEvent + events = RealtimeEventHandler.get_replay_window( + user_id, page_group_names, last_seen_id, web_socket_id + ) + history = RealtimeEventHandler.get_expired_history( + user_id, page_group_names, last_seen_id, web_socket_id + ) + events_sql, events_params = events.values_list( + "id", "channel_group", "payload", "created_at" + ).query.sql_with_params() + history_sql, history_params = history.values( + "route_key" + ).query.sql_with_params() + sql = ( + "WITH history AS MATERIALIZED (" # noqa: S608 + "SELECT state.floor, GREATEST(state.floor, state.compacted_event_id, " + "COALESCE((SELECT id FROM ws_realtime_events " + "ORDER BY id DESC LIMIT 1), 0)) AS latest_event_id, " + f"CASE WHEN state.floor <= %s THEN EXISTS({history_sql}) " + "ELSE false END AS expired " + "FROM ws_realtime_event_history_state AS state WHERE state.id = 1) " + "SELECT history.*, replay.* FROM history " + "LEFT JOIN LATERAL (" + "SELECT events.id, events.channel_group, " + "CASE WHEN events.created_at < %s THEN NULL ELSE events.payload END, " + f"events.created_at FROM ({events_sql}) AS events " + "WHERE history.floor <= %s AND NOT history.expired" + ") AS replay ON true ORDER BY replay.id" + ) + params = [ + last_seen_id, + *history_params, + timezone.now() - RealtimeEventHandler.get_replay_retention(), + *events_params, + last_seen_id, + ] + with connection.cursor() as cursor: + cursor.execute(sql, params) + rows = cursor.fetchall() + if not rows: + RealtimeEventHandler._initialize_realtime_history() + cursor.execute(sql, params) + rows = cursor.fetchall() + return rows - return RealtimeEvent.objects.aggregate(latest=Coalesce(Max("id"), 0))["latest"] + @staticmethod + def get_expired_history(user_id, page_group_names, last_seen_id, web_socket_id): + """Match compacted audiences, accounting for the client's socket exclusion.""" + + from baserow.ws.models import RealtimeEventHistorySummary + + latest = F("latest_event_id") + if web_socket_id is not None: + latest = Case( + When(latest_socket_id=web_socket_id, then=F("previous_event_id")), + default=latest, + ) + return RealtimeEventHistorySummary.objects.alias( + relevant_event_id=latest + ).filter( + RealtimeEventHandler.get_relevant_events_filter(user_id, page_group_names), + relevant_event_id__gt=last_seen_id, + ) @staticmethod def get_replay_window( @@ -351,37 +392,21 @@ def get_replay_window( last_seen_id: int, web_socket_id: Optional[str], ) -> QuerySet[RealtimeEvent]: - """ - Return the baseline event followed by replayable events. + """Return relevant payloads after the cursor, capped at limit plus one. - :param user_id: The id of the reconnecting user. - :param page_group_names: Page channel group names the user is - subscribed to. Must not include ``"users"``. - :param last_seen_id: Highest event id the client has already processed. - :param web_socket_id: The client's persistent web socket id, used to - exclude events the client itself originated. - :return: An ordered queryset containing ``last_seen_id`` when it still - exists, plus relevant events after it. The queryset is capped at - baseline plus one more than the configured replay limit so the caller - can detect that the client must refresh. + Keep the ID bound outside audience OR predicates so PostgreSQL's ordered + index scan can start at the cursor instead of visiting retained prehistory. """ from baserow.ws.models import RealtimeEvent - replay_filter = ( - Q(id__gt=last_seen_id) - & RealtimeEventHandler.get_not_own_event_filter(web_socket_id) - & RealtimeEventHandler.get_relevant_events_filter(user_id, page_group_names) - ) - - replay_filter |= Q(id=last_seen_id) - - # Keep the cursor bound outside the OR so PostgreSQL can start an ordered - # primary-key scan at the cursor. Otherwise LIMIT can select a plan that - # scans the entire retained history before reaching the baseline. return RealtimeEvent.objects.filter( - replay_filter, id__gte=last_seen_id - ).order_by("id")[: settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS + 2] + RealtimeEventHandler.get_not_own_event_filter(web_socket_id) + & RealtimeEventHandler.get_relevant_events_filter( + user_id, page_group_names + ), + id__gt=last_seen_id, + ).order_by("id")[: settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS + 1] @staticmethod def get_relevant_events_filter( @@ -425,7 +450,10 @@ def get_not_own_event_filter(web_socket_id: Optional[str]) -> Q: """ return ( - ~Q(payload__ignore_web_socket_id=web_socket_id) + ( + ~Q(payload__ignore_web_socket_id=web_socket_id) + | ~Q(payload__has_key="ignore_web_socket_id") + ) if web_socket_id is not None else Q() ) diff --git a/backend/src/baserow/ws/replay.py b/backend/src/baserow/ws/replay.py index a1daca152b..9c760ef2c1 100644 --- a/backend/src/baserow/ws/replay.py +++ b/backend/src/baserow/ws/replay.py @@ -234,7 +234,9 @@ def _get_executor(): def _force_refresh(): - return ReplayEventsResult(True, NO_REPLAY_AVAILABLE, []) + return ReplayEventsResult( + True, NO_REPLAY_AVAILABLE, [], refresh_reason="missing_cursor" + ) def _retry_later(): @@ -315,6 +317,7 @@ async def get_replay_events_result( started_at = monotonic() outcome = "error" + reason = "none" try: if last_seen_id == NO_REPLAY_AVAILABLE: result = _force_refresh() @@ -341,6 +344,7 @@ async def get_replay_events_result( result = await asyncio.shield(task) if result.force_refresh: outcome = "refresh" + reason = result.refresh_reason or "unknown" elif last_seen_id == FIRST_CONNECT_CURSOR: outcome = "baseline" else: @@ -362,6 +366,6 @@ async def get_replay_events_result( outcome = "cancelled" raise finally: - attributes = {"process.pid": os.getpid(), "outcome": outcome} + attributes = {"process.pid": os.getpid(), "outcome": outcome, "reason": reason} websocket_replay_requests.add(1, attributes) websocket_replay_duration.record((monotonic() - started_at) * 1000, attributes) diff --git a/backend/src/baserow/ws/tasks.py b/backend/src/baserow/ws/tasks.py index c13b3a57b1..d8af063a28 100644 --- a/backend/src/baserow/ws/tasks.py +++ b/backend/src/baserow/ws/tasks.py @@ -861,7 +861,6 @@ def cleanup_old_realtime_events(self): from baserow.ws.realtime_events import ( REALTIME_EVENTS_CLEANUP_BUDGET_SECONDS, REALTIME_EVENTS_CLEANUP_LOCK_SECONDS, - REALTIME_EVENTS_RETENTION, RealtimeEventHandler, ) from baserow.ws.telemetry import record_realtime_cleanup_skipped @@ -882,7 +881,7 @@ def cleanup_old_realtime_events(self): return try: return RealtimeEventHandler.cleanup_old_realtime_events( - REALTIME_EVENTS_RETENTION, deadline=deadline + RealtimeEventHandler.get_replay_retention(), deadline=deadline ) finally: try: diff --git a/backend/tests/baserow/config/test_settings_validation.py b/backend/tests/baserow/config/test_settings_validation.py new file mode 100644 index 0000000000..c9ac70389c --- /dev/null +++ b/backend/tests/baserow/config/test_settings_validation.py @@ -0,0 +1,39 @@ +import os +import subprocess +import sys + +import pytest + + +def _import_base_settings(retention_hours: str) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join(sys.path), + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS": retention_hours, + } + return subprocess.run( # noqa: S603 - only the current interpreter runs fixed code. + [sys.executable, "-c", "import baserow.config.settings.base"], + check=False, + capture_output=True, + env=env, + text=True, + ) + + +@pytest.mark.parametrize("retention_hours", ["0", "-1"]) +def test_non_positive_realtime_replay_retention_is_rejected(retention_hours: str): + """ + A non-positive retention disables compaction instead of failing, so the guard + in ``settings.base`` is the only thing stopping it. + + The settings module validates at import time, so the check can only be + exercised from a fresh interpreter; the ``settings`` fixture never re-runs it. + """ + + result = _import_base_settings(retention_hours) + + assert result.returncode != 0 + assert ( + "BASEROW_REALTIME_REPLAY_RETENTION_HOURS must be a positive integer" + in result.stderr + ) diff --git a/backend/tests/baserow/ws/conftest.py b/backend/tests/baserow/ws/conftest.py index 461c5b0767..7630efb6d2 100644 --- a/backend/tests/baserow/ws/conftest.py +++ b/backend/tests/baserow/ws/conftest.py @@ -26,10 +26,21 @@ def _install_realtime_targets(django_db_setup, django_db_blocker): migration.forwards(None, editor) +@pytest.fixture(scope="session") +def _install_realtime_history(_install_realtime_targets, django_db_blocker): + migration = import_module("baserow.ws.migrations.0003_realtime_event_history") + with django_db_blocker.unblock(), connection.schema_editor(atomic=False) as editor: + migration.forwards(None, editor) + + @pytest.fixture -def _django_db_helper(_install_realtime_targets, _django_db_helper): - # Wrapping pytest-django's helper does not request a DB for non-DB tests. - return _django_db_helper +def _django_db_helper(_install_realtime_history, _django_db_helper): + # Initialize known history inside pytest-django's per-test transaction. + from baserow.ws.models import RealtimeEventHistoryState + + RealtimeEventHistoryState.objects.update_or_create( + pk=1, defaults={"floor": 0, "compacted_event_id": 0} + ) @pytest.fixture(autouse=True) diff --git a/backend/tests/baserow/ws/test_ws_history_migration.py b/backend/tests/baserow/ws/test_ws_history_migration.py new file mode 100644 index 0000000000..cb5877ada5 --- /dev/null +++ b/backend/tests/baserow/ws/test_ws_history_migration.py @@ -0,0 +1,262 @@ +from contextlib import closing +from datetime import timedelta +from importlib import import_module +from time import monotonic + +from django.db import DatabaseError, OperationalError, connection, transaction +from django.db.migrations.loader import MigrationLoader +from django.utils import timezone + +import pytest + +from baserow.ws import history +from baserow.ws.models import ( + RealtimeEvent, + RealtimeEventHistoryState, + RealtimeEventHistorySummary, +) +from baserow.ws.realtime_events import RealtimeEventHandler + +pytestmark = [pytest.mark.django_db(transaction=True), pytest.mark.websockets] + + +def record(channel="table-1", socket=None): + return RealtimeEventHandler.record_events( + [ + ( + channel, + { + "type": "broadcast_to_group", + "ignore_web_socket_id": socket, + "payload": {"type": "rows_updated", "private_data": "discard"}, + }, + ) + ] + )[0] + + +def compact(): + return history.compact_events_batch( + timezone.now(), + monotonic() + 10, + batch_size=5000, + statement_timeout_ms=3000, + lock_timeout_ms=250, + ) + + +def test_history_catalog_keeps_new_tables_unlogged_and_sequence_logged(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT relname, relpersistence FROM pg_class WHERE oid IN (" + "'ws_realtime_event_history_summary'::regclass, " + "'ws_realtime_event_history_state'::regclass, " + "pg_get_serial_sequence('ws_realtime_events', 'id')::regclass)" + ) + persistence = dict(cursor.fetchall()) + assert persistence.pop("ws_realtime_event_history_summary") == "u" + assert persistence.pop("ws_realtime_event_history_state") == "u" + assert list(persistence.values()) == ["p"] + cursor.execute( + "SELECT seqcache FROM pg_sequence WHERE seqrelid = " + "pg_get_serial_sequence('ws_realtime_events', 'id')::regclass" + ) + assert cursor.fetchone() == (1,) + cursor.execute( + "SELECT indisvalid, pg_get_expr(indpred, indrelid) FROM pg_index " + "WHERE indexrelid = 'ws_realtime_created_id_idx'::regclass" + ) + assert cursor.fetchone() == (True, None) + cursor.execute( + "SELECT count(*) FROM pg_trigger " + "WHERE tgrelid = 'ws_realtime_events'::regclass " + "AND NOT tgisinternal AND (tgtype & 8) <> 0" + ) + assert cursor.fetchone() == (0,) + + +def test_history_migration_roundtrip_preserves_event_data_and_age_index(): + loader = MigrationLoader(None) + schema_migration = loader.get_migration("ws", "0003_realtime_event_history") + previous_state = loader.project_state([("ws", "0002_realtime_event_indexes")]) + + def filenodes(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT pg_relation_filenode('ws_realtime_events'), " + "pg_relation_filenode('ws_realtime_created_id_idx')" + ) + return cursor.fetchone() + + at_current_schema = True + try: + # Exercise all CreateModel/RunPython operations in both directions, + # without reapplying ws.0002's deliberate replay-buffer reset. + with connection.schema_editor() as editor: + schema_migration.unapply(previous_state.clone(), editor) + at_current_schema = False + old_id, recent_id = record(), record() + RealtimeEvent.objects.filter(pk=old_id).update( + created_at=timezone.now() - timedelta(days=2) + ) + originals = list(RealtimeEvent.objects.order_by("id").values()) + original_filenodes = filenodes() + + with connection.schema_editor() as editor: + schema_migration.apply(previous_state.clone(), editor) + at_current_schema = True + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert filenodes() == original_filenodes + assert not RealtimeEventHistorySummary.objects.exists() + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == state.compacted_event_id == recent_id + + with connection.schema_editor() as editor: + schema_migration.unapply(previous_state.clone(), editor) + at_current_schema = False + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert filenodes() == original_filenodes + with connection.cursor() as cursor: + cursor.execute( + "SELECT to_regclass('ws_realtime_event_history_summary'), " + "to_regclass('ws_realtime_event_history_state'), " + "to_regprocedure('ws_initialize_realtime_history()')" + ) + assert cursor.fetchone() == (None, None, None) + # Older writers still use the unchanged table and durable sequence. + assert record() > recent_id + finally: + if not at_current_schema: + with connection.schema_editor() as editor: + schema_migration.apply(previous_state.clone(), editor) + + +def test_history_initialization_is_idempotent_with_existing_summary_and_floor(): + expired_id = record() + assert compact() == 1 + retained_id = record() + original_summary = RealtimeEventHistorySummary.objects.values().get() + original_state = RealtimeEventHistoryState.objects.values().get() + migration = import_module("baserow.ws.migrations.0003_realtime_event_history") + + for _ in range(2): + with connection.schema_editor() as editor: + migration.forwards(None, editor) + assert RealtimeEventHistorySummary.objects.values().get() == original_summary + assert RealtimeEventHistoryState.objects.values().get() == original_state + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [retained_id] + assert original_state["floor"] == 0 + assert original_state["compacted_event_id"] == expired_id + + +def test_history_reset_uses_allocated_sequence_ids_as_conservative_floor(settings): + settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = 100 + committed_id = record() + assert compact() == 1 + with pytest.raises(RuntimeError, match="rollback"): + with transaction.atomic(): + allocated_id = record() + raise RuntimeError("rollback") + assert allocated_id > committed_id + + # Simulate loss of every UNLOGGED history relation while retaining the + # logged sequence, including IDs allocated by a rolled-back transaction. + with connection.cursor() as cursor: + cursor.execute( + "TRUNCATE ws_realtime_events, ws_realtime_event_history_summary, " + "ws_realtime_event_history_state" + ) + assert RealtimeEventHandler.get_latest_event_id() >= allocated_id + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == state.compacted_event_id >= allocated_id + assert not RealtimeEventHistorySummary.objects.exists() + result = RealtimeEventHandler.get_replay_events_result( + 42, ["table-1"], committed_id, None + ) + assert result.force_refresh + assert result.refresh_reason == "unknown_history" + assert record() > state.floor + + +def test_history_initialization_waits_for_inflight_inserts(): + RealtimeEventHistoryState.objects.all().delete() + with closing( + connection.Database.connect(**connection.get_connection_params()) + ) as writer: + with writer.cursor() as cursor: + cursor.execute( + "INSERT INTO ws_realtime_events (channel_group, payload, created_at) " + "VALUES ('other', '{}', now()) RETURNING id" + ) + inserted_id = cursor.fetchone()[0] + + # Initialization must wait for the invisible writer's RowExclusive lock + # before treating its allocated ID as covered by the loss floor. + with pytest.raises(OperationalError) as error: + with transaction.atomic(), connection.cursor() as cursor: + cursor.execute("SET LOCAL lock_timeout = '25ms'") + cursor.execute("SELECT ws_initialize_realtime_history()") + cause = error.value.__cause__ + assert ( + getattr(cause, "pgcode", None) or getattr(cause, "sqlstate", None) + ) == "55P03" + assert not RealtimeEventHistoryState.objects.exists() + writer.commit() + + with connection.cursor() as cursor: + cursor.execute("SELECT ws_initialize_realtime_history()") + assert RealtimeEventHistoryState.objects.get(pk=1).floor >= inserted_id + + +def test_missing_history_with_cached_sequence_ids_fails_closed(): + RealtimeEventHistoryState.objects.all().delete() + with connection.cursor() as cursor: + cursor.execute( + "DO $block$ BEGIN EXECUTE format('ALTER SEQUENCE %s CACHE 100', " + "pg_get_serial_sequence('ws_realtime_events', 'id')); END $block$" + ) + try: + with ( + connection.cursor() as cursor, + pytest.raises(DatabaseError, match="CACHE 1"), + ): + cursor.execute("SELECT ws_initialize_realtime_history()") + assert not RealtimeEventHistoryState.objects.exists() + finally: + with connection.cursor() as cursor: + cursor.execute( + "DO $block$ BEGIN EXECUTE format('ALTER SEQUENCE %s CACHE 1', " + "pg_get_serial_sequence('ws_realtime_events', 'id')); END $block$" + ) + cursor.execute("SELECT ws_initialize_realtime_history()") + + +@pytest.mark.parametrize("existing_summary", [False, True]) +def test_route_hash_collision_rolls_back_summary_and_keeps_originals( + monkeypatch, existing_summary +): + record("first-audience", socket="first") + if existing_summary: + assert compact() == 1 + collision_key = bytes(RealtimeEventHistorySummary.objects.get().route_key) + else: + collision_key = b"x" * 32 + record("second-audience", socket="second") + original_events = list(RealtimeEvent.objects.order_by("id").values()) + original_summaries = list(RealtimeEventHistorySummary.objects.values()) + original_state = RealtimeEventHistoryState.objects.values().get() + summarize = history._summarize_candidates + + def force_hash_collision(candidates): + # Inject a digest collision after real SQL routing extraction. The + # existing-summary case still executes the actual ON CONFLICT update. + return summarize( + [(event_id, collision_key, *route) for event_id, _, *route in candidates] + ) + + monkeypatch.setattr(history, "_summarize_candidates", force_hash_collision) + with pytest.raises(RuntimeError, match="route hash collision"): + compact() + assert list(RealtimeEvent.objects.order_by("id").values()) == original_events + assert list(RealtimeEventHistorySummary.objects.values()) == original_summaries + assert RealtimeEventHistoryState.objects.values().get() == original_state diff --git a/backend/tests/baserow/ws/test_ws_index_migration.py b/backend/tests/baserow/ws/test_ws_index_migration.py index 17c8ed1967..e8aebcfbbf 100644 --- a/backend/tests/baserow/ws/test_ws_index_migration.py +++ b/backend/tests/baserow/ws/test_ws_index_migration.py @@ -6,7 +6,7 @@ import pytest -from baserow.ws.models import RealtimeEvent +from baserow.ws.models import RealtimeEvent, RealtimeEventHistoryState from baserow.ws.realtime_events import RealtimeEventHandler @@ -85,20 +85,28 @@ def test_replay_reset_upgrade_and_rollback_keep_ids_and_old_writer_compatibility _apply("backwards") try: old_cursor = _legacy_insert() + lost_id = _legacy_insert() old_filenode = _table_storage()[0] _apply("forwards") assert RealtimeEvent.objects.count() == 0 assert _table_storage()[0] != old_filenode assert {"target_user_ids", "all_users"} <= _columns() + # Activate history after the reset, as ws.0003 does in deployment. The + # sequence records the missed event even though its payload is now gone. + RealtimeEventHistoryState.objects.all().delete() + RealtimeEventHandler._initialize_realtime_history() + assert RealtimeEventHistoryState.objects.get(pk=1).floor == lost_id new_id = _legacy_insert() - assert new_id > old_cursor + assert new_id > lost_id > old_cursor event = RealtimeEvent.objects.get(id=new_id) assert event.target_user_ids == [42] assert event.all_users is False - assert RealtimeEventHandler.get_replay_events_result( + result = RealtimeEventHandler.get_replay_events_result( 42, [], old_cursor, "socket" - ).force_refresh + ) + assert result.force_refresh is True + assert result.refresh_reason == "unknown_history" indexes = _indexes() assert migration.OLD_INDEX not in indexes assert migration.USERS_INDEX not in indexes diff --git a/backend/tests/baserow/ws/test_ws_realtime_cleanup.py b/backend/tests/baserow/ws/test_ws_realtime_cleanup.py index 0adca409c0..a5b00ee524 100644 --- a/backend/tests/baserow/ws/test_ws_realtime_cleanup.py +++ b/backend/tests/baserow/ws/test_ws_realtime_cleanup.py @@ -1,31 +1,42 @@ from contextlib import closing from datetime import timedelta -from unittest.mock import patch +from unittest.mock import Mock, patch from django.db import OperationalError, connection, transaction from django.utils import timezone import pytest -from baserow.ws import realtime_events, tasks +from baserow.ws import history, realtime_events, tasks from baserow.ws.models import RealtimeEvent from baserow.ws.realtime_events import RealtimeEventHandler from baserow.ws.tasks import cleanup_old_realtime_events @pytest.mark.parametrize("max_events", [0, 5]) +@pytest.mark.parametrize("retention_hours", [1, 24, 48, 240]) def test_cleanup_task_uses_independent_retention_even_when_recording_disabled( - settings, max_events + settings, max_events, retention_hours ): settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = max_events + settings.REALTIME_REPLAY_RETENTION_HOURS = retention_hours settings.SIMPLE_JWT = {"REFRESH_TOKEN_LIFETIME": timedelta(days=7)} with ( patch.object(RealtimeEventHandler, "cleanup_old_realtime_events") as cleanup, - patch("django.core.cache.cache.lock"), + patch("django.core.cache.cache.lock") as make_lock, patch.object(tasks, "monotonic", return_value=100), ): cleanup_old_realtime_events() - cleanup.assert_called_once_with(timedelta(hours=24), deadline=130) + cleanup.assert_called_once_with(timedelta(hours=retention_hours), deadline=340) + assert make_lock.call_args.kwargs["timeout"] == 330 + + +def test_cleanup_runs_every_ten_minutes(): + sender = Mock() + tasks.setup_periodic_ws_realtime_events_cleanup(sender) + sender.add_periodic_task.assert_called_once_with( + timedelta(minutes=10), cleanup_old_realtime_events.s() + ) def test_cleanup_task_skips_an_overlapping_run(): @@ -141,7 +152,9 @@ def fail_second_batch(*args): @pytest.mark.django_db(transaction=True) -def test_retention_boundary_preserves_fresh_replay_and_expires_old_cursor(settings): +def test_retention_boundary_preserves_replay_after_an_acknowledged_expired_event( + settings, +): settings.SIMPLE_JWT = {"REFRESH_TOKEN_LIFETIME": timedelta(days=7)} now = timezone.now() with patch.object(realtime_events.timezone, "now", return_value=now): @@ -155,12 +168,11 @@ def test_retention_boundary_preserves_fresh_replay_and_expires_old_cursor(settin assert list( RealtimeEvent.objects.order_by("id").values_list("id", flat=True) ) == [boundary, fresh, latest] - expired_result = RealtimeEventHandler.get_replay_events_result( - 1, ["table-1"], expired, None - ) - assert expired_result.force_refresh is True - assert expired_result.replay_events == [] - for cursor, expected in ((boundary, [fresh, latest]), (fresh, [latest])): + for cursor, expected in ( + (expired, [boundary, fresh, latest]), + (boundary, [fresh, latest]), + (fresh, [latest]), + ): result = RealtimeEventHandler.get_replay_events_result( 1, ["table-1"], cursor, None ) @@ -175,6 +187,7 @@ def test_cleanup_stops_at_its_work_budget(monkeypatch): create_events(timedelta(days=2), 5) now = 0 monkeypatch.setattr(realtime_events, "monotonic", lambda: now) + monkeypatch.setattr(history, "monotonic", lambda: now) original = RealtimeEventHandler._delete_realtime_events_batch def consume_budget(*args): @@ -211,7 +224,7 @@ def test_cleanup_timeouts_are_local_and_preserve_stricter_settings( observed = [] def check_timeouts(execute, sql, params, many, context): - if sql.startswith("WITH expired"): + if sql.startswith("WITH candidates"): with connection.cursor() as cursor: cursor.execute("SHOW statement_timeout") statement = cursor.fetchone()[0] @@ -242,6 +255,81 @@ def check_timeouts(execute, sql, params, many, context): connection.close() +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "statement_timeout,lock_timeout,expected_statements,expected_lock", + [ + ("0", "0", ["3s", "3s", "2s", "1s", "1s"], "250ms"), + ( + "1500ms", + "50ms", + ["1500ms", "1500ms", "1500ms", "1s", "1s"], + "50ms", + ), + ], +) +def test_cleanup_tightens_timeouts_as_the_batch_uses_its_remaining_budget( + monkeypatch, statement_timeout, lock_timeout, expected_statements, expected_lock +): + expired = create_events(timedelta(days=2), 2) + now = 0 + monkeypatch.setattr(history, "monotonic", lambda: now) + observed = [] + timeout_updates = 0 + statement_times = { + "SELECT ws_initialize_realtime_history()": 5, + "WITH candidates": 8, + "INSERT INTO ws_realtime_event_history_summary": 9, + "DELETE FROM ws_realtime_events": 9, + "UPDATE ws_realtime_event_history_state": 9, + } + + def observe_settings_and_advance_clock(execute, sql, params, many, context): + nonlocal now, timeout_updates + if "set_config('statement_timeout'" in sql: + timeout_updates += 1 + for prefix, finished_at in statement_times.items(): + if sql.startswith(prefix): + with connection.cursor() as cursor: + cursor.execute("SHOW statement_timeout") + statement = cursor.fetchone()[0] + cursor.execute("SHOW lock_timeout") + lock = cursor.fetchone()[0] + observed.append((prefix, statement, lock)) + result = execute(sql, params, many, context) + now = finished_at + return result + return execute(sql, params, many, context) + + with connection.cursor() as cursor: + cursor.execute( + "SELECT set_config('statement_timeout', %s, false)", [statement_timeout] + ) + cursor.execute("SELECT set_config('lock_timeout', %s, false)", [lock_timeout]) + try: + with connection.execute_wrapper(observe_settings_and_advance_clock): + assert ( + RealtimeEventHandler._delete_realtime_events_batch( + timezone.now() - timedelta(days=1), deadline=10 + ) + == 2 + ) + assert observed == [ + (prefix, statement, expected_lock) + for prefix, statement in zip(statement_times, expected_statements) + ] + # Reuse the cap until the remaining budget falls below three seconds. + assert timeout_updates == 3 + assert not RealtimeEvent.objects.filter(id__in=expired).exists() + with connection.cursor() as cursor: + cursor.execute("SHOW statement_timeout") + assert cursor.fetchone()[0] == statement_timeout + cursor.execute("SHOW lock_timeout") + assert cursor.fetchone()[0] == lock_timeout + finally: + connection.close() + + @pytest.mark.django_db(transaction=True) def test_cleanup_statement_timeout_stops_slow_database_work(monkeypatch): monkeypatch.setattr( @@ -250,7 +338,7 @@ def test_cleanup_statement_timeout_stops_slow_database_work(monkeypatch): expired = create_events(timedelta(days=2), 1) def slow_delete(execute, sql, params, many, context): - if sql.startswith("WITH expired"): + if sql.startswith("WITH candidates"): return execute("SELECT pg_sleep(1)", [], many, context) return execute(sql, params, many, context) diff --git a/backend/tests/baserow/ws/test_ws_realtime_events.py b/backend/tests/baserow/ws/test_ws_realtime_events.py index 2fbadd0d6a..b6bab75d43 100644 --- a/backend/tests/baserow/ws/test_ws_realtime_events.py +++ b/backend/tests/baserow/ws/test_ws_realtime_events.py @@ -5,6 +5,7 @@ from django.conf import settings from django.db import connection from django.test import override_settings +from django.utils import timezone import pytest from asgiref.sync import sync_to_async @@ -12,7 +13,7 @@ from loguru import logger from baserow.config.asgi import application -from baserow.ws.models import RealtimeEvent +from baserow.ws.models import RealtimeEvent, RealtimeEventHistoryState from baserow.ws.realtime_events import ( FIRST_CONNECT_CURSOR, NO_REPLAY_AVAILABLE, @@ -594,6 +595,7 @@ def test_replay_raises_when_over_threshold(): web_socket_id=None, ) assert result.force_refresh is True + assert result.refresh_reason == "event_limit" assert result.replay_events == [] @@ -1192,7 +1194,6 @@ def test_replay_window_ordered_scan_does_not_visit_events_before_cursor(): nodes.extend(node.get("Plans", [])) assert [event.id for event in window] == [ - baseline, events[900].id, events[905].id, events[910].id, @@ -1237,7 +1238,7 @@ def test_stale_users_replay_does_not_filter_unrelated_individual_payloads(): filtered_rows += node.get("Rows Removed by Index Recheck", 0) nodes.extend(node.get("Plans", [])) - assert [event.id for event in window] == [baseline, targeted, everyone] + assert [event.id for event in window] == [targeted, everyone] # The shared event type must not make replay inspect every other user's # payload. Check work performed by PostgreSQL, rather than machine timing. assert filtered_rows < 50 @@ -1297,30 +1298,34 @@ def test_replay_events_result_future_last_seen_uses_one_query( ) assert result.force_refresh is True + assert result.refresh_reason == "cursor_ahead" assert result.latest_event_id == NO_REPLAY_AVAILABLE assert result.replay_events == [] -@pytest.mark.django_db +@pytest.mark.django_db(transaction=True) @pytest.mark.websockets -def test_replay_events_result_missing_last_seen_uses_one_query( +def test_replay_events_result_compacted_last_seen_uses_one_query( django_assert_num_queries, ): - missing_id = _record_user_broadcast(1, {"type": "missing"}) - RealtimeEvent.objects.filter(id=missing_id).delete() - _record_user_broadcast(1, {"type": "latest"}) + compacted_id = _record_user_broadcast(1, {"type": "acknowledged"}) + RealtimeEvent.objects.filter(id=compacted_id).update( + created_at=timezone.now() - timedelta(days=2) + ) + latest_id = _record_user_broadcast(1, {"type": "latest"}) + assert RealtimeEventHandler.cleanup_old_realtime_events(timedelta(days=1)) == 1 with django_assert_num_queries(1): result = _replay_events_result( user_id=1, page_group_names=[], - last_seen_id=missing_id, + last_seen_id=compacted_id, web_socket_id=None, ) - assert result.force_refresh is True - assert result.latest_event_id == NO_REPLAY_AVAILABLE - assert result.replay_events == [] + assert result.force_refresh is False + assert result.latest_event_id == latest_id + assert [event.id for event in result.replay_events] == [latest_id] @pytest.mark.django_db @@ -1758,13 +1763,12 @@ async def test_replay_events_replays_individual_payloads_event(data_fixture): @pytest.mark.asyncio @pytest.mark.django_db(transaction=True) @pytest.mark.websockets -async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): +async def test_replay_events_cant_replay_below_the_known_history_floor(data_fixture): user, token = await sync_to_async(data_fixture.create_user_and_token)() await sync_to_async(data_fixture.create_workspace)(user=user) - # Last-seen event cleaned by retention while a newer one survives: replay can't - # anchor, so it must force a refresh. Capture the real id — the sequence isn't - # reset between transactional tests. + # A cursor below the known-history floor cannot prove recovery, even when + # newer full events survive. Use real IDs because sequences are not reset. stale_last_seen_id = await sync_to_async(_record_event)( "users", { @@ -1775,8 +1779,7 @@ async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): "ignore_web_socket_id": "ws-other", }, ) - await sync_to_async(RealtimeEvent.objects.filter(id=stale_last_seen_id).delete)() - await sync_to_async(_record_event)( + latest_id = await sync_to_async(_record_event)( "users", { "type": "broadcast_to_users", @@ -1786,6 +1789,9 @@ async def test_replay_events_cant_replay_when_last_seen_expired(data_fixture): "ignore_web_socket_id": "ws-other", }, ) + await sync_to_async(RealtimeEventHistoryState.objects.filter(pk=1).update)( + floor=latest_id + ) communicator = WebsocketCommunicator( application, diff --git a/backend/tests/baserow/ws/test_ws_realtime_history.py b/backend/tests/baserow/ws/test_ws_realtime_history.py new file mode 100644 index 0000000000..6e949c56b2 --- /dev/null +++ b/backend/tests/baserow/ws/test_ws_realtime_history.py @@ -0,0 +1,474 @@ +import json +import re +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from threading import Barrier +from time import monotonic + +from django.db import connection, connections +from django.utils import timezone + +import pytest + +from baserow.ws import realtime_events +from baserow.ws.models import ( + RealtimeEvent, + RealtimeEventHistoryState, + RealtimeEventHistorySummary, +) +from baserow.ws.realtime_events import FIRST_CONNECT_CURSOR, RealtimeEventHandler + +pytestmark = [pytest.mark.django_db(transaction=True), pytest.mark.websockets] + + +@pytest.fixture(autouse=True) +def replay_settings(settings): + settings.BASEROW_REALTIME_REPLAY_MAX_EVENTS = 100 + settings.REALTIME_REPLAY_RETENTION_HOURS = 24 + + +def group_event(*, event_type="rows_updated", socket=None, excluded=None, **data): + event = { + "type": "broadcast_to_group", + "payload": {"type": event_type, **data}, + "ignore_web_socket_id": socket, + } + if excluded is not None: + event["exclude_user_ids"] = excluded + return event + + +def users_event(*, recipients=None, everyone=False, socket=None): + return { + "type": "broadcast_to_users", + "payload": {"type": "workspace_updated", "private_data": "omit-me"}, + "user_ids": [42] if recipients is None else recipients, + "send_to_all_users": everyone, + "ignore_web_socket_id": socket, + } + + +def individual_event(*, recipients=None, socket=None): + return { + "type": "broadcast_to_users_individual_payloads", + "payload_map": { + str(user_id): {"type": event_type, "private_data": "omit-me"} + for user_id, event_type in ( + {42: "workspace_updated"} if recipients is None else recipients + ).items() + }, + "ignore_web_socket_id": socket, + } + + +def record(group="table-1", payload=None, *, age=timedelta()): + event_id = RealtimeEventHandler.record_events( + [(group, group_event() if payload is None else payload)] + )[0] + RealtimeEvent.objects.filter(pk=event_id).update(created_at=timezone.now() - age) + return event_id + + +def replay(cursor, *, user=42, groups=None, socket="own"): + return RealtimeEventHandler.get_replay_events_result( + user, ["table-1"] if groups is None else groups, cursor, socket + ) + + +def cleanup(): + return RealtimeEventHandler.cleanup_old_realtime_events(timedelta(days=1)) + + +@pytest.mark.parametrize("missed_old", [False, True]) +@pytest.mark.parametrize("recent", [False, True]) +def test_missing_baseline_preserves_no_change_replay_and_refresh(missed_old, recent): + baseline = record("other-page", age=timedelta(days=3)) + record("other-page", age=timedelta(days=2)) + if missed_old: + record(age=timedelta(days=2)) + fresh_id = record(age=timedelta(hours=1)) if recent else None + + # An expired relevant gap requires refresh even while its payload still + # exists, and a newer full event must not hide that gap after compaction. + assert replay(baseline).force_refresh is missed_old + assert cleanup() == 2 + missed_old + assert not RealtimeEvent.objects.filter(pk=baseline).exists() + + result = replay(baseline) + assert result.force_refresh is missed_old + assert result.refresh_reason == ("expired_payload" if missed_old else None) + assert [event.id for event in result.replay_events] == ( + [fresh_id] if recent and not missed_old else [] + ) + if not missed_old: + assert result.latest_event_id == (fresh_id if recent else baseline) + state = RealtimeEventHistoryState.objects.get(pk=1) + assert state.floor == 0 + assert state.compacted_event_id > baseline + + +@pytest.mark.parametrize( + "group,payload,socket,relevant", + [ + ("table-1", group_event(), "own", True), + ("other-page", group_event(), "own", False), + ("table-1", group_event(socket="own"), "own", False), + ("table-1", group_event(socket="other"), "own", True), + ("table-1", group_event(socket="own"), None, True), + ("table-1", group_event(excluded=[42]), "own", False), + ("table-1", group_event(excluded=[7]), "own", True), + ("users", users_event(), "own", True), + ("users", users_event(recipients=[7]), "own", False), + ("users", users_event(recipients=[], everyone=True), "own", True), + ("users", users_event(socket="own"), "own", False), + ("users", users_event(socket="own"), None, True), + ("users", individual_event(), "own", True), + ( + "users", + individual_event(recipients={7: "workspace_updated"}), + "own", + False, + ), + ("users", individual_event(socket="own"), "own", False), + ("users", individual_event(socket="own"), None, True), + ], + ids=[ + "page", + "other-page", + "own-page", + "other-socket-page", + "page-without-client-socket", + "excluded-page", + "other-excluded-page", + "target-user", + "other-user", + "everyone", + "own-user", + "user-without-client-socket", + "individual", + "other-individual", + "own-individual", + "individual-without-client-socket", + ], +) +def test_summary_and_expired_payload_have_identical_relevance( + group, payload, socket, relevant +): + baseline = record("baseline", age=timedelta(days=3)) + expired = record(group, payload, age=timedelta(days=2)) + + assert replay(baseline, socket=socket).force_refresh is relevant + assert cleanup() == 2 + assert not RealtimeEvent.objects.filter(pk=expired).exists() + result = replay(baseline, socket=socket) + assert result.force_refresh is relevant + assert result.replay_events == [] + + +@pytest.mark.parametrize("kind", ["group", "users", "individual"]) +def test_missing_and_null_socket_share_one_summary_without_losing_changes(kind): + group = "table-1" if kind == "group" else "users" + make_payload = { + "group": group_event, + "users": users_event, + "individual": individual_event, + }[kind] + payload = make_payload() + del payload["ignore_web_socket_id"] + missing = record(group, payload, age=timedelta(days=2)) + null = record(group, make_payload(), age=timedelta(days=2)) + own = record(group, make_payload(socket="own"), age=timedelta(days=2)) + + assert cleanup() == 3 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (own, "own") + assert (summary.previous_event_id, summary.previous_socket_id) == (null, None) + assert replay(missing).force_refresh is True + assert replay(null).force_refresh is False + assert replay(null, socket=None).force_refresh is True + + +@pytest.mark.parametrize( + "ignored_socket,client_socket", + [(42, "42"), (True, "true"), (["own"], "own"), ({"id": "own"}, "own")], + ids=["number", "boolean", "array", "object"], +) +def test_non_string_socket_values_do_not_exclude_a_client( + ignored_socket, client_socket +): + baseline = record("baseline", age=timedelta(days=3)) + record(payload=group_event(socket=ignored_socket), age=timedelta(days=2)) + assert replay(baseline, socket=client_socket).force_refresh is True + + assert cleanup() == 2 + summary = RealtimeEventHistorySummary.objects.get(channel_group="table-1") + assert summary.latest_socket_id is None + assert replay(baseline, socket=client_socket).force_refresh is True + + +def test_many_originating_sockets_keep_only_two_distinct_socket_watermarks(): + ids = [ + record(payload=group_event(socket=f"socket-{index}"), age=timedelta(days=2)) + for index in range(20) + ] + newest = record(payload=group_event(socket="socket-19"), age=timedelta(days=2)) + + assert cleanup() == 21 + assert not RealtimeEvent.objects.exists() + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (newest, "socket-19") + assert (summary.previous_event_id, summary.previous_socket_id) == ( + ids[-2], + "socket-18", + ) + assert replay(ids[-3], socket="socket-19").force_refresh is True + assert replay(ids[-2], socket="socket-19").force_refresh is False + assert replay(ids[-2], socket="socket-18").force_refresh is True + assert replay(ids[-2], socket=None).force_refresh is True + + +def test_summary_merges_batches_in_id_order_despite_different_timestamp_order( + monkeypatch, +): + monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 2) + ids = [ + record(payload=group_event(socket=socket), age=timedelta(days=days)) + for socket, days in [("a", 2), ("b", 5), ("a", 4), ("c", 3)] + ] + + assert cleanup() == 4 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (ids[-1], "c") + assert (summary.previous_event_id, summary.previous_socket_id) == (ids[-2], "a") + assert RealtimeEventHistoryState.objects.get(pk=1).compacted_event_id == ids[-1] + assert replay(ids[-2], socket="c").force_refresh is False + assert replay(ids[1], socket="c").force_refresh is True + + +def test_later_cleanup_of_lower_ids_does_not_regress_summary_or_high_water(): + lower = record(payload=group_event(socket="a"), age=timedelta(hours=1)) + other = record(payload=group_event(socket="b"), age=timedelta(days=2)) + latest = record(payload=group_event(socket="a"), age=timedelta(days=2)) + assert cleanup() == 2 + original = RealtimeEventHistorySummary.objects.values().get() + + RealtimeEvent.objects.filter(pk=lower).update( + created_at=timezone.now() - timedelta(days=3) + ) + assert cleanup() == 1 + assert RealtimeEventHistorySummary.objects.values().get() == original + assert RealtimeEventHandler.get_latest_event_id() == latest + + repeated = record(payload=group_event(socket="a"), age=timedelta(days=2)) + assert cleanup() == 1 + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == (repeated, "a") + assert (summary.previous_event_id, summary.previous_socket_id) == (other, "b") + + newest = record(payload=group_event(socket="c"), age=timedelta(days=2)) + assert cleanup() == 1 + summary.refresh_from_db() + assert (summary.latest_event_id, summary.latest_socket_id) == (newest, "c") + assert (summary.previous_event_id, summary.previous_socket_id) == (repeated, "a") + + +@pytest.mark.parametrize("individual", [False, True], ids=["group", "individual"]) +def test_summary_retains_recovery_types_without_business_payload(individual): + for event_type in ("rows_updated", "rows_deleted"): + payload = ( + individual_event(recipients={42: event_type, 7: "workspace_updated"}) + if individual + else group_event(event_type=event_type, secret="omit-me", rows=[1, 2, 3]) + ) + payload["request_debug_data"] = "omit-me" + record("users" if individual else "table-1", payload, age=timedelta(days=2)) + + assert cleanup() == 2 + summaries = list(RealtimeEventHistorySummary.objects.order_by("latest_event_id")) + assert len(summaries) == 2 + for summary, event_type in zip(summaries, ("rows_updated", "rows_deleted")): + assert "ignore_web_socket_id" not in summary.payload + assert "request_debug_data" not in summary.payload + assert "omit-me" not in json.dumps(summary.payload) + if individual: + assert summary.payload["payload_map"] == { + "7": {"type": "workspace_updated"}, + "42": {"type": event_type}, + } + assert summary.target_user_ids == [7, 42] + assert summary.all_users is False + else: + assert summary.payload["payload"] == {"type": event_type} + + +@pytest.mark.parametrize("kind", ["recipients", "exclusions"]) +def test_reordered_and_duplicate_audience_ids_share_one_summary(kind): + for audience in ([42, 7, 42], [7, 42]): + payload = ( + users_event(recipients=audience) + if kind == "recipients" + else group_event(excluded=audience) + ) + latest = record( + "users" if kind == "recipients" else "table-1", + payload, + age=timedelta(days=2), + ) + + assert cleanup() == 2 + summary = RealtimeEventHistorySummary.objects.get() + key = "user_ids" if kind == "recipients" else "exclude_user_ids" + assert summary.payload[key] == [7, 42] + assert summary.latest_event_id == latest + + +def test_retention_increase_cannot_make_summarized_payload_replayable(settings): + baseline = record("baseline", age=timedelta(days=3)) + expired = record(age=timedelta(days=2)) + assert cleanup() == 2 + + settings.REALTIME_REPLAY_RETENTION_HOURS = 240 + result = replay(baseline) + assert result.force_refresh is True + assert result.replay_events == [] + recent = record(age=timedelta(hours=1)) + result = replay(expired) + assert result.force_refresh is False + assert [event.id for event in result.replay_events] == [recent] + + +def test_empty_full_buffer_keeps_summarized_high_water_without_raising_loss_floor(): + record(age=timedelta(days=3)) + latest = record(age=timedelta(days=2)) + + assert cleanup() == 2 + assert not RealtimeEvent.objects.exists() + assert RealtimeEventHandler.get_latest_event_id() == latest + state = RealtimeEventHistoryState.objects.get(pk=1) + assert (state.floor, state.compacted_event_id) == (0, latest) + baseline = replay(FIRST_CONNECT_CURSOR) + assert baseline.force_refresh is False + assert baseline.latest_event_id == latest + assert baseline.replay_events == [] + + fresh = record() + assert fresh > latest + result = replay(latest) + assert result.force_refresh is False + assert [event.id for event in result.replay_events] == [fresh] + + +def is_event_delete(sql): + return re.search(r'\bDELETE\s+FROM\s+"?ws_realtime_events\b', sql, re.IGNORECASE) + + +def test_failure_after_deletion_rolls_back_payloads_summary_and_high_water(): + record(payload=group_event(value="first"), age=timedelta(days=2)) + record(payload=group_event(value="second"), age=timedelta(days=2)) + record(age=timedelta(hours=1)) + originals = list(RealtimeEvent.objects.order_by("id").values()) + state = RealtimeEventHistoryState.objects.values().get() + + def fail_after_delete(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if is_event_delete(sql): + raise RuntimeError("failed after deleting expired payloads") + return result + + with ( + connection.execute_wrapper(fail_after_delete), + pytest.raises(RuntimeError, match="failed after deleting expired payloads"), + ): + cleanup() + + assert list(RealtimeEvent.objects.order_by("id").values()) == originals + assert not RealtimeEventHistorySummary.objects.exists() + assert RealtimeEventHistoryState.objects.values().get() == state + + +def test_replay_on_another_connection_remains_correct_during_compaction_delete(): + baseline = record("baseline", age=timedelta(days=3)) + record(age=timedelta(days=2)) + fresh = record(age=timedelta(hours=1)) + reads = [] + + def read_history(): + try: + result = replay(baseline) + return RealtimeEventHandler.get_latest_event_id(), result + finally: + connections["default"].close() + + with ThreadPoolExecutor(max_workers=1) as executor: + + def read_while_delete_is_uncommitted(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if is_event_delete(sql): + assert connection.in_atomic_block + # The reader has its own connection and must finish before the + # deleting transaction commits, with the expired gap still visible. + reads.append(executor.submit(read_history).result(timeout=5)) + return result + + with connection.execute_wrapper(read_while_delete_is_uncommitted): + assert cleanup() == 2 + + assert len(reads) == 1 + latest, result = reads[0] + assert latest == fresh + assert result.force_refresh is True + assert result.replay_events == [] + assert replay(baseline).force_refresh is True + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [fresh] + + +@pytest.mark.parametrize( + "latest_socket", ["b", None], ids=["two-sockets", "null-socket"] +) +def test_concurrent_batches_merge_the_same_route_without_losing_either_socket( + monkeypatch, latest_socket +): + monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 1) + baseline = record("baseline") + first = record(payload=group_event(socket="a"), age=timedelta(days=2)) + latest = record(payload=group_event(socket=latest_socket), age=timedelta(days=2)) + candidates_locked = Barrier(2) + + def compact_one(): + db = connections["default"] + + def synchronize_candidates(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if "FOR UPDATE SKIP LOCKED" in sql: + # Both transactions lock separate full rows before either inserts + # the shared route. The upsert must merge their evidence safely. + candidates_locked.wait(timeout=5) + return result + + try: + with db.execute_wrapper(synchronize_candidates): + return RealtimeEventHandler._delete_realtime_events_batch( + timezone.now() - timedelta(days=1), monotonic() + 10 + ) + finally: + db.close() + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(compact_one) for _ in range(2)] + try: + assert [future.result(timeout=10) for future in futures] == [1, 1] + finally: + candidates_locked.abort() + + summary = RealtimeEventHistorySummary.objects.get() + assert (summary.latest_event_id, summary.latest_socket_id) == ( + latest, + latest_socket, + ) + assert (summary.previous_event_id, summary.previous_socket_id) == (first, "a") + assert list(RealtimeEvent.objects.values_list("id", flat=True)) == [baseline] + state = RealtimeEventHistoryState.objects.get(pk=1) + assert (state.floor, state.compacted_event_id) == (0, latest) + assert replay(baseline, socket="b").force_refresh is True + assert replay(first, socket="b").force_refresh is (latest_socket is None) + assert replay(first, socket="a").force_refresh is True diff --git a/backend/tests/baserow/ws/test_ws_replay_executor.py b/backend/tests/baserow/ws/test_ws_replay_executor.py index c5cfbc4915..51edd812ab 100644 --- a/backend/tests/baserow/ws/test_ws_replay_executor.py +++ b/backend/tests/baserow/ws/test_ws_replay_executor.py @@ -324,7 +324,9 @@ async def test_replay_executor_releases_capacity_if_cancelled_before_start( async def test_missing_replay_cursor_does_not_need_a_database_or_pool(): with patch.object(replay, "_get_executor") as executor: result = await replay.get_replay_events_result(1, [], NO_REPLAY_AVAILABLE, None) - assert result == ReplayEventsResult(True, NO_REPLAY_AVAILABLE, []) + assert result == ReplayEventsResult( + True, NO_REPLAY_AVAILABLE, [], refresh_reason="missing_cursor" + ) executor.assert_not_called() diff --git a/backend/tests/baserow/ws/test_ws_storage_telemetry.py b/backend/tests/baserow/ws/test_ws_storage_telemetry.py index 1ab8c8946b..2d12b6f566 100644 --- a/backend/tests/baserow/ws/test_ws_storage_telemetry.py +++ b/backend/tests/baserow/ws/test_ws_storage_telemetry.py @@ -8,7 +8,7 @@ import pytest from redis.exceptions import ConnectionError as RedisConnectionError -from baserow.ws import realtime_events, telemetry +from baserow.ws import history, realtime_events, telemetry from baserow.ws.models import RealtimeEvent from baserow.ws.realtime_events import RealtimeEventHandler from baserow.ws.tasks import cleanup_old_realtime_events @@ -248,6 +248,7 @@ def test_cleanup_budget_reports_progress_without_claiming_completion( monkeypatch.setattr(realtime_events, "REALTIME_EVENTS_CLEANUP_BATCH_SIZE", 2) now = [0.0] monkeypatch.setattr(realtime_events, "monotonic", lambda: now[0]) + monkeypatch.setattr(history, "monotonic", lambda: now[0]) delete_batch = RealtimeEventHandler._delete_realtime_events_batch def slow_batch(cutoff, deadline): diff --git a/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json b/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json new file mode 100644 index 0000000000..488c0a8e6b --- /dev/null +++ b/changelog/entries/unreleased/bug/avoids_unnecessary_refresh_prompts_when_reconnecting_after_o.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Avoids unnecessary refresh prompts when reconnecting after old real-time history expired", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-09-14" +} diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml index 253bad35cd..1b5f41091b 100644 --- a/docker-compose.no-caddy.yml +++ b/docker-compose.no-caddy.yml @@ -92,6 +92,7 @@ x-backend-variables: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_MAX_FIELD_TEXT_LENGTH: BASEROW_REALTIME_REPLAY_MAX_EVENTS: + BASEROW_REALTIME_REPLAY_RETENTION_HOURS: BASEROW_AUTOMATION_HISTORY_PAGE_SIZE_LIMIT: BASEROW_AUTOMATION_WORKFLOW_RATE_LIMIT_MAX_RUNS: diff --git a/docker-compose.yml b/docker-compose.yml index 54cf9bdc7d..aa28525d9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ x-backend-variables: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_MAX_FIELD_TEXT_LENGTH: BASEROW_REALTIME_REPLAY_MAX_EVENTS: + BASEROW_REALTIME_REPLAY_RETENTION_HOURS: BASEROW_AUTOMATION_HISTORY_PAGE_SIZE_LIMIT: BASEROW_AUTOMATION_WORKFLOW_RATE_LIMIT_MAX_RUNS: diff --git a/docs/installation/configuration.md b/docs/installation/configuration.md index 9bfdaa3be8..be97cc2381 100644 --- a/docs/installation/configuration.md +++ b/docs/installation/configuration.md @@ -235,7 +235,8 @@ Baserow can throttle the number of concurrent requests a single user (or, option | BASEROW\_ENTERPRISE\_MAX\_PERIODIC\_DATA\_SYNC\_CONSECUTIVE\_ERRORS | The maximum number of consecutive periodic data sync error before it's disabled. | 4 | | BASEROW\_DEADLOCK\_INITIAL\_BACKOFF | The initial backoff time for database deadlock retries. | 2 | | BASEROW\_DEADLOCK\_MAX\_RETRIES | The maximum number of database deadlock retries. | 1 | -| BASEROW\_REALTIME\_REPLAY\_MAX\_EVENTS | Maximum number of missed events replayed to a reconnecting WebSocket client. When the count exceeds this limit, the client is asked to refresh instead of receiving individual events. Set to `0` to disable event recording and replay. Events are retained for 24 hours independently of JWT lifetime, and retention cleanup continues when recording is disabled. | 200 | +| BASEROW\_REALTIME\_REPLAY\_MAX\_EVENTS | Maximum number of missed events replayed to a reconnecting WebSocket client. When the count exceeds this limit, the client is asked to refresh instead of receiving individual events. Set to `0` to disable event recording and replay. Full events are replayable for the configured retention window, independently of JWT lifetime. Expired events become routing summaries without business payloads; relevant missed expired history requires a refresh. Cleanup continues when recording is disabled. | 200 | +| BASEROW\_REALTIME\_REPLAY\_RETENTION\_HOURS | Maximum age in hours of full realtime events available to replay after a WebSocket reconnect. Must be a positive integer. Increasing this window retains more full payloads and requires more database storage, but cannot restore events already compacted. Independent of JWT lifetime. | 24 | | BASEROW\_DEPENDANT\_ROWS\_REALTIME\_UPDATE\_LIMIT | Per table, the maximum number of rows changed by a dependency cascade (formulas, lookups, link row display values in other tables) that are broadcast as exact realtime row updates. Above the limit, subscribers of the affected table receive a whole-table refresh event instead. Set to `0` to disable realtime events for dependant rows entirely. Higher values increase the background serialization work and websocket traffic per edit. | 200 | | BASEROW\_PERIODIC\_FIELD\_UPDATE\_BATCH\_COUNT | The number of parallel tasks the periodic field update (e.g. `now()` formulas) is split into each cycle. Raise it to spread the work across more workers when a single task can't refresh every workspace before the next cycle starts. If cycles are still being skipped, also give each cycle more time with `BASEROW_PERIODIC_FIELD_UPDATE_CRONTAB`. | 1 | diff --git a/docs/installation/monitoring.md b/docs/installation/monitoring.md index 286e63bb51..9ee652e7d7 100644 --- a/docs/installation/monitoring.md +++ b/docs/installation/monitoring.md @@ -160,7 +160,7 @@ for the execution and recovery model. | `baserow.websocket_sync_queue_duration` / `baserow.websocket_sync_execution_duration` | Milliseconds before the executor starts work versus milliseconds on its thread, including database connection cleanup. | | `baserow.websocket_sync_pending` / `baserow.websocket_sync_executing` | Awaiting callers, including queued and running calls, versus work actually executing. Executing work can outlive a cancelled caller. | | `baserow.websocket_event_loop_lag` | Scheduling delay in milliseconds, sampled once per second while WebSocket applications are active. | -| `baserow.websocket_replay_requests` | Decisions by `baseline`, `replayed`, `refresh`, `overloaded`, `deadline_exceeded`, `query_timeout`, `database_error`, `cancelled`, or `error`. | +| `baserow.websocket_replay_requests` | Decisions by `baseline`, `replayed`, `refresh`, `overloaded`, `deadline_exceeded`, `query_timeout`, `database_error`, `cancelled`, or `error`. Refresh `reason` distinguishes `missing_cursor`, `unknown_history`, `cursor_ahead`, `expired_payload`, and `event_limit`; other outcomes use `none`. | | `baserow.websocket_replay_duration` / `baserow.websocket_replay_events` | Caller wait in milliseconds and number of events returned by a completed decision. | | `baserow.websocket_replay_inflight` / `baserow.websocket_replay_capacity` | Occupied slots and initialized replay pool capacity, including work still running after cancellation or a deadline. | | `baserow.websocket_replay_queued` / `baserow.websocket_replay_queue_capacity` / `baserow.websocket_replay_queue_duration` | Waiting requests, admission queue capacity, and wait in milliseconds, including cancelled/expired waits. Admission precedes the separate synchronous executor queue measurement. | @@ -168,7 +168,7 @@ for the execution and recovery model. | `baserow.realtime_recording_events` | Attempted recording envelopes by `destination=users/page` and handler `outcome=success/error`. A successful handler does not guarantee an enclosing transaction committed. | | `baserow.realtime_recording_batch_size` / `baserow.realtime_recording_duration` | Envelopes per attempted batch and handler duration in milliseconds, including adaptation and database work. | | `baserow.realtime_cleanup_deleted` / `baserow.realtime_cleanup_batch_size` | Deleted events and rows per successfully committed cleanup batch. | -| `baserow.realtime_cleanup_batch_duration` | Batch duration in milliseconds, including commit. Failed batches have `outcome=error` and contribute no deleted rows. | +| `baserow.realtime_cleanup_batch_duration` | Batch duration in milliseconds, including expired-history summary updates and commit. Failed batches have `outcome=error` and contribute no deleted rows. | | `baserow.realtime_cleanup_run_deleted` / `baserow.realtime_cleanup_run_duration` | Committed progress and run duration in milliseconds, by outcome. Earlier commits still count if a later batch fails. | | `baserow.realtime_cleanup_skipped` | Scheduled attempts skipped for `reason=overlap` (another task owns the lease) or `reason=lock_error` (lease acquisition failed). | @@ -217,14 +217,25 @@ has exhausted memory. Compare recording rate with committed cleanup deletions over time. A cleanup run ending with `budget` retained its earlier commits but exhausted its time allowance; `success` can still leave locked rows for the next run. Repeated overlap or lock -errors explain runs that never reached the database. Track oldest event age, +errors explain runs that never reached the database. Track oldest full-event age, +event-table size, history-summary count and size, `pg_stat_user_tables` live/dead tuple estimates and vacuum/analyze timestamps, and database I/O alongside these metrics. Deletion and vacuum make space reusable; they do not normally reduce allocated table files. Replay refresh fallbacks also create HTTP reads, so include that traffic when assessing capacity. +Cleanup deletes expired business payloads while preserving routing summaries for +exact audiences. Summaries have no fixed size bound: distinct audiences can +accumulate, although new ignored sockets alone do not create additional rows. +Measure summary growth alongside recent event volume and cleanup progress. + For users-channel replay, compare rows and heap blocks visited with events actually returned. Recipient selection should use `target_user_ids` and `all_users`, with `ws_realtime_targets_idx` and `ws_realtime_all_users_idx` available to the planner. There is no full-payload GIN index. Include recipient-trigger work in recording measurements; smaller indexes do not by themselves guarantee faster inserts. + +Refresh reasons separate storage coverage from event volume: `expired_payload` +means relevant missed history was compacted or falls outside the replay window; +`unknown_history` means the cursor predates the known history boundary after +activation or reset; `event_limit` means too many replayable changes remain. diff --git a/docs/technical/websockets.md b/docs/technical/websockets.md index ae1e70adc1..1d989d39aa 100644 --- a/docs/technical/websockets.md +++ b/docs/technical/websockets.md @@ -204,6 +204,16 @@ resets the buffer before restoring its old indexes; neither direction restores discarded history. These resets affect only realtime replay, not the underlying user data. +Migration `ws.0003` adds separate unlogged history-summary and history-state tables; +it preserves the existing event table and indexes. Before migration, pause and +drain the old cleanup task. Update all replay readers before enabling the new +cleanup, because older readers cannot use the summaries. Initial activation +establishes a conservative history floor from the logged event sequence. Cursors +below that floor refresh. After an unlogged-table reset, initialization takes the +same boundary while briefly blocking inserts, preserving the sequence's `CACHE 1` +requirement. Before rolling back, disable replay and establish fresh client +baselines; removing the summaries cannot restore compacted payloads. + ### Last Seen Event ID During normal delivery, the frontend advances its cursor to the highest `_event_id` @@ -218,11 +228,11 @@ sends a `replay_events` message carrying its last seen event ID as `last_seen_id The server uses that cursor and those subscriptions to decide whether recovery is possible, with these completed outcomes: -1. **Nothing missed** — The replay window contains only the client's `last_seen_id`. The client is already up to date for the channel groups being restored. +1. **Nothing missed** — No relevant event follows the client's `last_seen_id`, in either full events or expired-history summaries. The client is already up to date for the channel groups being restored, even if the acknowledged event itself has been removed. 2. **Events replayed** — The server fetches the missed events for the client's page channel groups and implicit `users` group, filters out the client's own broadcasts (via its web socket id) and any events not relevant to that user, and re-invokes them through the consumer's handlers in order — exactly as if they had arrived live. The client catches up without a page reload. -3. **Can't replay** — Either too many events were missed (more than `BASEROW_REALTIME_REPLAY_MAX_EVENTS`), the client's `last_seen_id` has already been cleaned up by retention, or the server finds a persisted event it cannot safely re-deliver through a websocket broadcast handler. The server responds with `force_refresh=true` and the client shows a "workspace data is outdated" toast with a refresh action. +3. **Can't replay** — A relevant missed event has expired, the cursor predates known history or is ahead of the recorded high-water mark, too many events were missed (more than `BASEROW_REALTIME_REPLAY_MAX_EVENTS`), or the server finds a persisted event it cannot safely re-deliver through a websocket broadcast handler. The server responds with `force_refresh=true` and the client shows a "workspace data is outdated" toast with a refresh action. -Every `replay_events_result` with `force_refresh=false` includes `latest_event_id`, the latest event ID the server can safely acknowledge for that replay decision. If a client connects without a `last_seen_id` (a fresh page load), the server returns the latest persisted event ID as the new baseline because there is nothing to replay. When replay succeeds, `latest_event_id` is the last event in the replay window and might be lower than the global latest persisted ID if newer events were irrelevant to that client. If the server responds with `force_refresh=true`, `latest_event_id` is not meaningful and the client should refresh instead. +Every `replay_events_result` with `force_refresh=false` includes `latest_event_id`, the latest event ID the server can safely acknowledge for that replay decision. A fresh page load receives the recorded high-water mark, including compacted history, as its baseline. When replay succeeds, `latest_event_id` is the last replayed event, or the original cursor when nothing relevant changed. It can be lower than the global high-water mark when newer events were irrelevant. Full events, expired summaries and the known-history floor are checked in one database snapshot, so cleanup cannot expose a gap between deleting an event and recording its expired history. If `force_refresh=true`, `latest_event_id` is not meaningful and the client should refresh instead. ### Replay resource limits and retries @@ -243,7 +253,7 @@ connection attempt. For overload, timeouts, and database failures, clients advertising `supports_retry=true` receive `replay_events_retry` and retry on the same socket with backoff and jitter. Older clients receive the existing refresh fallback. -An expired cursor, excessive event gap, or disabled recording still requires a +Expired relevant history, an excessive event gap, or disabled recording requires a refresh when recovering missed updates. The frontend keeps one replay request or retry timer active and holds the original @@ -256,19 +266,35 @@ longer be verified. An unrecoverable gap stays marked outdated across reconnects ### Event Cleanup -A periodic Celery task removes events older than 24 hours, independently of JWT refresh-token lifetime. It runs every minute, including when recording is disabled, with a 30-second work budget and at most 5,000 events per committed batch. Each deletion statement has a three-second timeout and a 250 ms lock timeout, preserving stricter database settings. Locked rows are left for a later run. Clients whose baseline has expired use the existing refresh fallback. - -Each batch commits separately, so earlier deletions survive a later failure. A -scheduled run skips cleanup while another task owns the nonblocking lease. The -retention target is not a hard maximum row age: locked rows or a sustained cleanup -backlog can remain until a later run. - -A surviving baseline older than the retention window cannot prove complete -history: cleanup may have skipped its lock while deleting newer expired events. -Replay checks the baseline's age as well as its existence before acknowledging it. - -The `(created_at, id)` index supports bounded expiration scans. Recipient indexes -are restricted to the shared `users` channel; page events retain the +A periodic Celery task compacts events older than `BASEROW_REALTIME_REPLAY_RETENTION_HOURS` (24 hours by default), independently of JWT refresh-token lifetime. It runs every ten minutes, including when recording is disabled, with a 240-second work budget and at most 5,000 events per committed batch. Each statement has a three-second timeout and a 250 ms lock timeout, tightened to the remaining budget without relaxing stricter database settings. Locked rows are left for a later run. + +Each exact audience gets a separate summary containing routing metadata and the +newest expired event IDs, without full business payloads. The summary keeps the +newest event and the newest event with a different ignored socket: any client +excludes at most one socket, so these two entries preserve its latest relevant +expired event. New browser sessions therefore do not create new summary rows for +an otherwise identical audience. Distinct audiences can still accumulate; summary +storage has no fixed size bound or expiry. Row-scoped channel groups such as +`table--row-` give one summary per edited row rather than per table, so +size this against row volume. + +Summary updates, deletion of all selected full events, and advancement of the +compacted high-water mark commit together. Each batch commits separately, so +earlier progress survives a later failure. Ordinary replay reads remain available +while a batch is uncommitted. A scheduled run skips cleanup while another task owns +the nonblocking 330-second lease. The retention target is not a hard maximum row +age: locked rows or a sustained cleanup backlog can remain until a later run. + +Replay also checks the age of relevant full events left behind by cleanup, so +locked rows or cleanup lag cannot make expired payloads replayable. Increasing the +retention window cannot restore already compacted payloads. Deleting events or +summaries outside this cleanup is unsupported without explicitly invalidating +existing replay cursors. + +The existing `(created_at, id)` index supports bounded expiration scans; no second +age index is added. Since expired originals are removed, retained summaries do not +accumulate in that scan. Recipient indexes are restricted to the shared `users` +channel; page events retain the `(channel_group, id)` index. Cleanup makes storage reusable through PostgreSQL vacuum; it does not normally shrink the table's allocated files. Monitor recording rate, committed cleanup progress, and database vacuum activity together; see @@ -287,5 +313,6 @@ an immediate `ANALYZE`. | Setting | Default | Purpose | |---|---|---| | `BASEROW_REALTIME_REPLAY_MAX_EVENTS` | 200 | Maximum number of missed events the server will replay. Beyond this, the client is told to refresh. Set to `0` to disable event recording and replay; retention cleanup continues. Clients learn replay availability during authentication and use refresh when missed events cannot be recovered. | +| `BASEROW_REALTIME_REPLAY_RETENTION_HOURS` | 24 | Maximum age in hours of full events available for replay. Must be a positive integer. Older relevant missed history requires a refresh; increasing the window cannot restore compacted payloads. | See [configuration.md](../installation/configuration.md) for the full settings reference. From 6dae44dfeaac86ec9e07601270fa06560b442aa2 Mon Sep 17 00:00:00 2001 From: Davide Silvestri <75379892+silvestrid@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:30:40 +0200 Subject: [PATCH 2/4] fix: don't serialize invalid field rules as null on database export (#6096) FieldRuleHandler.export_rule() returns None for rules that aren't both active and valid, but DatabaseApplicationType.export_serialized() appended that sentinel to the payload, so snapshots, duplications and file exports of a table with an active-but-invalid rule wrote a null entry that crashed the subsequent import with AttributeError: 'NoneType' object has no attribute 'pop'. Fixes #6095 --- .../contrib/database/application_types.py | 7 +- .../contrib/database/field_rules/handlers.py | 6 +- .../test_field_rules_import_export.py | 70 +++++++++++++++++++ ...ication_crash_for_tables_with_an_inva.json | 9 +++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py create mode 100644 changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json diff --git a/backend/src/baserow/contrib/database/application_types.py b/backend/src/baserow/contrib/database/application_types.py index 52df49d192..5d8a5bb953 100755 --- a/backend/src/baserow/contrib/database/application_types.py +++ b/backend/src/baserow/contrib/database/application_types.py @@ -234,7 +234,8 @@ def export_tables_serialized( rule_type, ) in field_rules_handler.applicable_rules_with_types: exported_field_rule = field_rules_handler.export_rule(rule) - serialized_field_rules.append(exported_field_rule) + if exported_field_rule is not None: + serialized_field_rules.append(exported_field_rule) structure = DatabaseExportSerializedStructure.table( id=table.id, @@ -770,6 +771,10 @@ def _import_field_rules(self, serialized_tables, id_mapping, import_export_confi field_rules_handler = FieldRuleHandler(table) serialized_rules = serialized_table["field_rules"] for serialized_rule in serialized_rules: + # exports made before #6095 may contain null entries for rules + # that were active but invalid at export time. + if not serialized_rule: + continue field_rules_handler.import_rule( serialized_rule, id_mapping["database_fields"] ) diff --git a/backend/src/baserow/contrib/database/field_rules/handlers.py b/backend/src/baserow/contrib/database/field_rules/handlers.py index cb0e24bba9..36b21c61ef 100644 --- a/backend/src/baserow/contrib/database/field_rules/handlers.py +++ b/backend/src/baserow/contrib/database/field_rules/handlers.py @@ -596,9 +596,13 @@ def validate_rows_for_rule( return rule_type.validate_rows(self.table, rule, queryset=queryset) - def export_rule(self, rule: FieldRule): + def export_rule(self, rule: FieldRule) -> dict | None: """ Exports a rule. + + :param rule: the rule to export. + :return: the serialized rule, or None if the rule is not exportable because + it is disabled or invalid. Callers must not add None to the export payload. """ exportable = rule.is_active and rule.is_valid diff --git a/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py b/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py new file mode 100644 index 0000000000..60c3ca4e3e --- /dev/null +++ b/backend/tests/baserow/contrib/database/field_rules/test_field_rules_import_export.py @@ -0,0 +1,70 @@ +import pytest + +from baserow.contrib.database.field_rules.handlers import FieldRuleHandler +from baserow.contrib.database.field_rules.models import FieldRule +from baserow.core.registries import ImportExportConfig, application_type_registry + + +def _create_table_with_rule(data_fixture, is_valid: bool): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + database = data_fixture.create_database_application(workspace=workspace) + table = data_fixture.create_database_table(database=database) + data_fixture.create_text_field(table=table, name="text") + + rule = FieldRuleHandler(table, user).create_rule("dummy", {}) + FieldRule.objects.filter(id=rule.id).update(is_valid=is_valid) + return user, database, table + + +@pytest.mark.django_db +def test_export_serialized_skips_invalid_field_rules( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=False) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + assert serialized["tables"][0]["field_rules"] == [] + + +@pytest.mark.django_db +def test_import_serialized_with_invalid_field_rule( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=False) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + imported_workspace = data_fixture.create_workspace(user=user) + imported_database = database_type.import_serialized( + imported_workspace, serialized, config, {}, None, None + ) + + imported_table = imported_database.table_set.get() + assert not FieldRule.objects.filter(table=imported_table).exists() + + +@pytest.mark.django_db +def test_import_serialized_with_valid_field_rule( + data_fixture, fake_field_rule_registry +): + user, database, table = _create_table_with_rule(data_fixture, is_valid=True) + + database_type = application_type_registry.get("database") + config = ImportExportConfig(include_permission_data=True) + serialized = database_type.export_serialized(database, config) + + assert [r["type"] for r in serialized["tables"][0]["field_rules"]] == ["dummy"] + + imported_workspace = data_fixture.create_workspace(user=user) + imported_database = database_type.import_serialized( + imported_workspace, serialized, config, {}, None, None + ) + + imported_table = imported_database.table_set.get() + assert FieldRule.objects.filter(table=imported_table).count() == 1 diff --git a/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json b/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json new file mode 100644 index 0000000000..2190af9314 --- /dev/null +++ b/changelog/entries/unreleased/bug/6095_fixed_snapshot_and_duplication_crash_for_tables_with_an_inva.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fixed a crash when creating a snapshot, duplicating or exporting a database containing an active but invalid field rule.", + "issue_origin": "github", + "issue_number": 6095, + "domain": "database", + "bullet_points": [], + "created_at": "2026-09-15" +} From 22cd0eda09365800fece02dc2f41405365155893 Mon Sep 17 00:00:00 2001 From: Davide Silvestri <75379892+silvestrid@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:31:08 +0200 Subject: [PATCH 3/4] feat: warn about AI model usage instead of silently breaking consumers (#6075) * feat: report AI provider model usage per feature type Feature types now count the rows they store referencing a provider model, so the admin can be warned before disabling, deleting or narrowing one instead of being blocked. Counting is by provider type and model identifier because consumers do not store the provider row id, which makes the result an upper bound: fit to warn, never to block. Default-model features are excluded, their RESTRICT foreign key already blocks the change. * feat: count AI field and AI Agent references to a provider model Both features store a provider type and model identifier per row rather than a reference to the model, so nothing told an administrator what a model change would break. They now count their own rows, skipping every consumer whose owner is trashed: a field's table, database or workspace, and an agent service's automation node, builder workflow action, integration or application. * feat: expose AI provider model usage over the API GET /ai-providers/models//usage/ reports the per-consumer counts to warn about and, separately, the default-model features that refuse the change outright. Kuma holds a restricted foreign key, so naming it up front lets the client say which selection to repoint instead of letting the admin discover it through a generic error. Permissions and scoping mirror the sibling model endpoints, so a workspace-scoped caller cannot read an instance model's usage. * feat: warn before an AI model change breaks its consumers Disabling, deleting, renaming or narrowing a model now names what depends on it and proceeds on confirmation. Disabling was entirely unguarded before, and a rename orphans consumers just as thoroughly as a delete because they persist the identifier string. The counts are an upper bound: an instance model and a workspace model sharing an identifier are indistinguishable to a consumer, so the copy says a feature uses the model rather than claiming it will break. A failed lookup degrades to the plain confirmation instead of trapping the administrator. * fix: keep an unavailable AI model visible in the AI field form The form dropped a saved provider or model as soon as it stopped being available, leaving empty dropdowns that hid what the field had been configured with. It now retains the saved selection, disabled and marked unavailable, the way the AI Agent node form already does, while the validator keeps reading the genuine list so the field cannot be saved until a real model is picked. * docs: describe the AI provider model usage warning * refactor: drop redundant blocking feature types from model usage The usage endpoint pre-announced the features whose selection refuses a change, but ERROR_AI_PROVIDER_MODEL_IN_USE already names them and the frontend already renders that detail. Worse, the modal applied the list only on rename, the one path that is never refused: Kuma holds a foreign key to the model row, so the selection follows the rename instead of breaking. The same claim was in the installation docs; both are corrected to say what actually happens. Also drop the `trashed=False` filters from both model reference counts. AIAgentService.objects and AIField.objects are no-trash managers, so the generated SQL already excludes trashed rows. Rename the endpoint's operation id to get_ai_provider_model_usage: the documented response is an object, not a list. * fix: address Qodo review of the model usage warning Count a service only while a live owner still reaches it. The previous exclusion dropped the whole service as soon as any related builder action was trashed, so a service shared by a live and a trashed action vanished from the warning. Both owner managers already encode which ancestors count as trashed, so the check reuses them through EXISTS instead of restating the rules. Treat a failed usage lookup as unknown rather than zero. Returning empty counts made every caller skip its confirmation, letting a rename orphan consumers with no warning at all; the dialog now says the check failed and still lets the admin proceed. Cover fetchModelUsage directly: payload forms, workspace scoping and the snake_case to camelCase mapping had no store test. --- .../baserow/api/ai_provider/serializers.py | 9 + backend/src/baserow/api/ai_provider/urls.py | 6 + backend/src/baserow/api/ai_provider/views.py | 28 ++ .../ai/ai_provider_feature_types.py | 47 ++++ .../src/baserow/core/ai_provider/handler.py | 21 ++ .../baserow/core/ai_provider/registries.py | 24 ++ .../src/baserow/core/ai_provider/service.py | 23 ++ .../api/ai_provider/test_ai_provider_views.py | 150 ++++++++++- .../ai/test_ai_provider_feature_types.py | 220 ++++++++++++++++ .../baserow/core/ai_provider/test_handler.py | 100 ++++++++ ...warned_which_ai_fields_and_ai_agent_a.json | 9 + docs/installation/ai-assistant.md | 14 + .../fields/ai_provider_feature_types.py | 32 +++ .../test_ai_field_provider_feature_types.py | 91 +++++++ .../components/field/fieldAISubForm.spec.js | 15 +- .../ai/AIProviderModelFormModal.vue | 96 ++++++- .../core/components/ai/SelectAIModelForm.vue | 133 ++++++++-- .../workspace/AIProviderWorkspaceSettings.vue | 35 ++- web-frontend/modules/core/locales/en.json | 9 + .../core/mixins/aiProviderModelUsage.js | 45 ++++ .../modules/core/pages/admin/aiProviders.vue | 33 ++- .../modules/core/services/aiProvider.js | 3 + web-frontend/modules/core/store/aiProvider.js | 15 ++ web-frontend/modules/core/utils/aiProvider.js | 48 ++++ .../aiProviderModelFormModal.spec.js | 239 ++++++++++++++++++ .../aiProviderWorkspaceSettings.spec.js | 183 ++++++++++++++ .../core/components/selectAIModelForm.spec.js | 204 +++++++++++++++ .../unit/core/pages/admin/aiProviders.spec.js | 94 +++++++ .../test/unit/core/store/aiProvider.spec.js | 45 ++++ 29 files changed, 1919 insertions(+), 52 deletions(-) create mode 100644 changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json create mode 100644 premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py create mode 100644 web-frontend/modules/core/mixins/aiProviderModelUsage.js create mode 100644 web-frontend/test/unit/core/components/selectAIModelForm.spec.js diff --git a/backend/src/baserow/api/ai_provider/serializers.py b/backend/src/baserow/api/ai_provider/serializers.py index 26a2f3d0f8..96a4172d47 100644 --- a/backend/src/baserow/api/ai_provider/serializers.py +++ b/backend/src/baserow/api/ai_provider/serializers.py @@ -100,6 +100,15 @@ class AIProviderModelUpdateSerializer(serializers.Serializer): ) +class AIProviderModelUsageEntrySerializer(serializers.Serializer): + feature_type = serializers.CharField() + count = serializers.IntegerField() + + +class AIProviderModelUsageSerializer(serializers.Serializer): + usage = AIProviderModelUsageEntrySerializer(many=True) + + class AIProviderFeatureModelSerializer(serializers.Serializer): id = serializers.IntegerField() model_identifier = serializers.CharField() diff --git a/backend/src/baserow/api/ai_provider/urls.py b/backend/src/baserow/api/ai_provider/urls.py index c9180c1795..692ffe14f2 100644 --- a/backend/src/baserow/api/ai_provider/urls.py +++ b/backend/src/baserow/api/ai_provider/urls.py @@ -6,6 +6,7 @@ AIProviderModelDiscoveryView, AIProviderModelsTestView, AIProviderModelsView, + AIProviderModelUsageView, AIProviderModelView, AIProvidersView, AIProviderTypesView, @@ -36,4 +37,9 @@ ), path("models/test/", AIProviderModelsTestView.as_view(), name="test_models"), path("models//", AIProviderModelView.as_view(), name="model_item"), + path( + "models//usage/", + AIProviderModelUsageView.as_view(), + name="model_usage", + ), ] diff --git a/backend/src/baserow/api/ai_provider/views.py b/backend/src/baserow/api/ai_provider/views.py index 92c6d568c1..0d968c7dd5 100644 --- a/backend/src/baserow/api/ai_provider/views.py +++ b/backend/src/baserow/api/ai_provider/views.py @@ -63,6 +63,7 @@ AIProviderModelsTestRequestSerializer, AIProviderModelsTestResponseSerializer, AIProviderModelUpdateSerializer, + AIProviderModelUsageSerializer, AIProviderModelWriteSerializer, AIProviderScopeRequestSerializer, AIProviderTypeSerializer, @@ -379,6 +380,33 @@ def delete(self, request, model_id): return Response(status=HTTP_204_NO_CONTENT) +class AIProviderModelUsageView(APIView): + permission_classes = (IsAuthenticated,) + + @extend_schema( + tags=["AI providers"], + operation_id="get_ai_provider_model_usage", + parameters=[AIProviderScopeRequestSerializer], + responses={200: AIProviderModelUsageSerializer}, + ) + @map_exceptions(EXCEPTION_MAP) + def get(self, request, model_id): + _ensure_feature_enabled() + usage = AIProviderService.get_model_usage( + request.user, model_id, workspace_id=_get_workspace_id(request) + ) + return Response( + AIProviderModelUsageSerializer( + { + "usage": [ + {"feature_type": feature_type, "count": count} + for feature_type, count in usage.items() + ] + } + ).data + ) + + class AIProviderModelsTestView(APIView): permission_classes = (IsAuthenticated,) diff --git a/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py index e704a22cf1..af2e6da528 100644 --- a/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py +++ b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py @@ -1,3 +1,8 @@ +from django.db.models import Exists, OuterRef + +from baserow.contrib.automation.nodes.models import AutomationNode +from baserow.contrib.builder.workflow_actions.models import AIAgentWorkflowAction +from baserow.contrib.integrations.ai.models import AIAgentService from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT from baserow.core.ai_provider.registries import AIProviderModelFeatureType from baserow.core.ai_provider.resolution import ScopedAIProviderState @@ -8,6 +13,48 @@ class AIAgentAIProviderModelFeatureType(AIProviderModelFeatureType): type = AI_PROVIDER_FEATURE_AI_AGENT + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the AI Agent services selecting one provider model. + + One service is owned by an automation node or by builder workflow + actions, and trashing an owner leaves the service row untouched, so a + service counts only while at least one live owner still reaches it. The + owner managers already encode which ancestors count as trashed. A + service whose integration or application is gone belongs to no + workspace, so the joins drop it from both scopes. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier the services persist. + :param workspace: The workspace to narrow to, or None for the instance + scope, which counts every workspace. + :return: The number of services referencing the model. + """ + + live_automation_owner = Exists( + AutomationNode.objects.filter(service_id=OuterRef("pk")) + ) + live_builder_owner = Exists( + AIAgentWorkflowAction.objects.filter( + service_id=OuterRef("pk"), page__trashed=False + ) + ) + queryset = AIAgentService.objects.filter( + ai_generative_ai_type=provider_type, + ai_generative_ai_model=model_identifier, + integration__trashed=False, + integration__application__trashed=False, + integration__application__workspace__trashed=False, + ).filter(live_automation_owner | live_builder_owner) + if workspace is not None: + queryset = queryset.filter(integration__application__workspace=workspace) + return queryset.count() + def get_workspace_availability( self, workspace: Workspace | None, diff --git a/backend/src/baserow/core/ai_provider/handler.py b/backend/src/baserow/core/ai_provider/handler.py index 4c12a0e1dc..44c8123656 100644 --- a/backend/src/baserow/core/ai_provider/handler.py +++ b/backend/src/baserow/core/ai_provider/handler.py @@ -550,6 +550,27 @@ def delete_model(model: AIProviderModel) -> None: model.delete() clear_ai_provider_state_cache() + @staticmethod + def get_model_usage(model: AIProviderModel) -> dict[str, int]: + """ + Count the consumers referencing a model, per feature. + + Default-model features are skipped: their RESTRICT foreign key already + blocks the change instead of warning about it. + + :param model: The model about to be disabled, deleted or narrowed. + :return: The reference count of every per-consumer feature. + """ + + config = model.provider_config + return { + feature_type.type: feature_type.count_model_references( + config.provider_type, model.model_identifier, config.workspace + ) + for feature_type in ai_provider_model_feature_type_registry.get_all() + if not feature_type.supports_default_model + } + @staticmethod def _registered_default_model_feature_types() -> set[str]: """ diff --git a/backend/src/baserow/core/ai_provider/registries.py b/backend/src/baserow/core/ai_provider/registries.py index 20237fbcd3..2017c7e17b 100644 --- a/backend/src/baserow/core/ai_provider/registries.py +++ b/backend/src/baserow/core/ai_provider/registries.py @@ -1,3 +1,4 @@ +from baserow.core.models import Workspace from baserow.core.registry import Instance, Registry from .constants import ( @@ -24,6 +25,29 @@ class AIProviderModelFeatureType(Instance): supports_default_model = False required_model_capabilities = (AI_PROVIDER_MODEL_CAPABILITY_TEXT,) + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the rows this feature stores referencing one provider model. + + Consumers persist a provider type and model identifier instead of the + provider row id, so an instance model and a workspace override of the + same type share a count. The result is an upper bound: it may warn an + administrator, but it must never block a change. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier consumers persist. + :param workspace: The workspace owning the provider, or None for the + instance scope, which counts every workspace. + :return: The number of references this feature holds. + """ + + return 0 + def get_workspace_availability(self, workspace, state=None) -> dict: """Return the client-facing effective availability for this feature. diff --git a/backend/src/baserow/core/ai_provider/service.py b/backend/src/baserow/core/ai_provider/service.py index 4324e7f1ea..f5dfbe81fb 100644 --- a/backend/src/baserow/core/ai_provider/service.py +++ b/backend/src/baserow/core/ai_provider/service.py @@ -239,6 +239,29 @@ def delete_model( AIProviderHandler.delete_model(model) cls._send_updated(user, workspace, True, provider_type, {model_identifier}) + @classmethod + def get_model_usage( + cls, + user: AbstractUser, + model_id: int, + workspace_id: int | None = None, + ) -> dict[str, int]: + """ + Report what still depends on a model before an admin changes it. + + Default-model features are not reported: their selection already refuses + the change with ERROR_AI_PROVIDER_MODEL_IN_USE, which names them. + + :param user: The user asking for the counts. + :param model_id: The model about to be disabled, deleted or narrowed. + :param workspace_id: The workspace scope, or None for the instance scope. + :return: The per-consumer-feature counts to warn about. + """ + + workspace = cls._check_permissions(user, workspace_id) + model = AIProviderHandler.get_model(model_id, workspace=workspace) + return AIProviderHandler.get_model_usage(model) + @classmethod def test_models( cls, diff --git a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py index 74288c7c59..0a0b6781aa 100644 --- a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py +++ b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py @@ -15,6 +15,7 @@ ) from baserow.core.ai_provider.constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_TEST_MAX_TOKENS, AI_PROVIDER_TEST_TIMEOUT_SECONDS, @@ -39,6 +40,7 @@ ("post", "test_models"), ("patch", "model_item"), ("delete", "model_item"), + ("get", "model_usage"), ) @@ -83,7 +85,7 @@ def _request_ai_provider_api(api_client, case, headers): data = {"provider_type": "openai"} elif url_name == "test_models": data = {"model_ids": [model.id]} - elif url_name == "model_item": + elif url_name in ("model_item", "model_usage"): kwargs = {"model_id": model.id} if method == "patch": data = {"is_enabled": False} @@ -881,6 +883,152 @@ def test_models_can_be_created_updated_disabled_and_deleted( assert not AIProviderModel.objects.exists() +@pytest.mark.django_db +def test_model_usage_reports_every_per_consumer_feature( + api_client, staff_headers, enabled_ai_providers +): + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}), + **staff_headers, + ) + + assert response.status_code == HTTP_200_OK + assert response.json() == { + "usage": [ + {"feature_type": AI_PROVIDER_FEATURE_AI_AGENT, "count": 0}, + {"feature_type": AI_PROVIDER_FEATURE_AI_FIELDS, "count": 0}, + ] + } + + +@pytest.mark.django_db +def test_model_usage_omits_default_model_features_which_the_error_reports( + api_client, staff_headers, enabled_ai_providers +): + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5", feature_types=["kuma"] + ) + response = api_client.put( + reverse("api:ai_provider:feature_item", kwargs={"feature_type": "kuma"}), + {"mode": "model", "model_id": model.id}, + format="json", + **staff_headers, + ) + assert response.status_code == HTTP_200_OK + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}), + **staff_headers, + ) + + assert response.status_code == HTTP_200_OK + usage = response.json()["usage"] + assert "kuma" not in [entry["feature_type"] for entry in usage] + assert all(entry["count"] == 0 for entry in usage) + + response = api_client.delete( + reverse("api:ai_provider:model_item", kwargs={"model_id": model.id}), + **staff_headers, + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_IN_USE" + assert "kuma" in response.json()["detail"] + + +@pytest.mark.django_db +def test_model_usage_is_scoped_like_the_other_model_endpoints( + api_client, data_fixture, staff_headers, enabled_ai_providers +): + user, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=user) + headers = {"HTTP_AUTHORIZATION": f"JWT {token}"} + workspace_query = f"?workspace_id={workspace.id}" + instance_provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="instance-secret" + ) + instance_model = AIProviderModel.objects.create( + provider_config=instance_provider, model_identifier="gpt-5" + ) + workspace_provider = AIProviderConfig.objects.create( + provider_type="mistral", api_key="workspace-secret", workspace=workspace + ) + workspace_model = AIProviderModel.objects.create( + provider_config=workspace_provider, model_identifier="mistral-large" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": workspace_model.id}) + + workspace_query, + **headers, + ) + assert response.status_code == HTTP_200_OK + assert [entry["feature_type"] for entry in response.json()["usage"]] == [ + AI_PROVIDER_FEATURE_AI_AGENT, + AI_PROVIDER_FEATURE_AI_FIELDS, + ] + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": instance_model.id}) + + workspace_query, + **headers, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": workspace_model.id}), + **staff_headers, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + +@pytest.mark.django_db +def test_model_usage_requires_workspace_admin_permission( + api_client, data_fixture, enabled_ai_providers +): + owner = data_fixture.create_user() + member, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=owner) + data_fixture.create_user_workspace( + user=member, workspace=workspace, permissions="MEMBER" + ) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="secret", workspace=workspace + ) + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5" + ) + + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": model.id}) + + f"?workspace_id={workspace.id}", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_USER_INVALID_GROUP_PERMISSIONS" + + +@pytest.mark.django_db +def test_model_usage_of_an_unknown_model_returns_not_found( + api_client, staff_headers, enabled_ai_providers +): + response = api_client.get( + reverse("api:ai_provider:model_usage", kwargs={"model_id": 999999}), + **staff_headers, + ) + + assert response.status_code == HTTP_404_NOT_FOUND + assert response.json()["error"] == "ERROR_AI_PROVIDER_MODEL_DOES_NOT_EXIST" + + @pytest.mark.django_db def test_saved_models_are_tested_in_one_request_and_results_are_persisted( api_client, staff_headers, enabled_ai_providers diff --git a/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py index 5c7ea6e7da..3d8c79d9a8 100644 --- a/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py +++ b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py @@ -1,11 +1,14 @@ import pytest +from baserow.contrib.builder.workflow_actions.models import AIAgentWorkflowAction +from baserow.contrib.integrations.ai.models import AIIntegration from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT from baserow.core.ai_provider.handler import AIProviderHandler from baserow.core.ai_provider.models import AIProviderConfig, AIProviderModel from baserow.core.ai_provider.registries import ( ai_provider_model_feature_type_registry, ) +from baserow.core.trash.handler import TrashHandler def test_ai_agent_feature_type_is_registered(): @@ -56,3 +59,220 @@ def test_agent_selection_does_not_block_model_deletion(data_fixture, settings): AIProviderHandler.delete_model(model) assert not AIProviderModel.objects.filter(id=model.id).exists() + + +def _attach_builder_owner(data_fixture, service, page, trashed=False): + action = data_fixture.create_workflow_action( + AIAgentWorkflowAction, + page=page, + element=data_fixture.create_builder_button_element(page=page), + service=service, + ) + if trashed: + action.trashed = True + action.save() + return action + + +@pytest.mark.django_db +def test_ai_agent_count_model_references(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + workspace = page.builder.workspace + integration = data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ) + + def add_service(**kwargs): + service = data_fixture.create_ai_agent_service( + integration=integration, **kwargs + ) + _attach_builder_owner(data_fixture, service, page) + return service + + add_service(ai_generative_ai_type="openai", ai_generative_ai_model="agent-model") + add_service(ai_generative_ai_type="openai", ai_generative_ai_model="another-model") + add_service(ai_generative_ai_type="anthropic", ai_generative_ai_model="agent-model") + add_service( + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + trashed=True, + ) + other_page = data_fixture.create_builder_page(user=user) + other_workspace = other_page.builder.workspace + other_service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=other_page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, other_service, other_page) + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + assert ( + feature_type.count_model_references("openai", "agent-model", other_workspace) + == 1 + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 2 + assert feature_type.count_model_references("openai", "unknown-model", None) == 0 + assert feature_type.count_model_references("mistral", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_ancestors(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, service, page) + application = service.integration.application + workspace = application.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + + application.trashed = True + application.save() + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + application.trashed = False + application.save() + workspace.trashed = True + workspace.save() + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_skips_services_without_integration( + data_fixture, +): + data_fixture.create_ai_agent_service( + integration=None, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert ( + feature_type.count_model_references( + "openai", "agent-model", data_fixture.create_workspace() + ) + == 0 + ) + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_automation_owners( + data_fixture, +): + user = data_fixture.create_user() + workflow = data_fixture.create_automation_workflow(user=user) + integration = data_fixture.create_integration( + AIIntegration, application=workflow.automation, user=user + ) + node = data_fixture.create_automation_node( + user=user, + workflow=workflow, + type="ai_agent", + service=data_fixture.create_ai_agent_service( + integration=integration, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ), + ) + workspace = workflow.automation.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + + TrashHandler.trash(user, workspace, workflow.automation, node) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + TrashHandler.restore_item(user, "automation_node", node.id) + + assert feature_type.count_model_references("openai", "agent-model", None) == 1 + + TrashHandler.trash(user, workspace, workflow.automation, workflow) + + assert feature_type.count_model_references("openai", "agent-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_ignores_trashed_builder_owners(data_fixture): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + element = data_fixture.create_builder_button_element(page=page) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + action = data_fixture.create_workflow_action( + AIAgentWorkflowAction, page=page, element=element, service=service + ) + workspace = page.builder.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 + + action.trashed = True + action.save() + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + action.trashed = False + action.save() + TrashHandler.trash(user, workspace, page.builder, element) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + TrashHandler.restore_item(user, "builder_element", element.id) + TrashHandler.trash(user, workspace, page.builder, page) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 0 + + +@pytest.mark.django_db +def test_ai_agent_count_model_references_keeps_services_with_one_live_owner( + data_fixture, +): + user = data_fixture.create_user() + page = data_fixture.create_builder_page(user=user) + service = data_fixture.create_ai_agent_service( + integration=data_fixture.create_integration( + AIIntegration, application=page.builder, user=user + ), + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ) + _attach_builder_owner(data_fixture, service, page) + _attach_builder_owner(data_fixture, service, page, trashed=True) + workspace = page.builder.workspace + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + + assert feature_type.count_model_references("openai", "agent-model", workspace) == 1 diff --git a/backend/tests/baserow/core/ai_provider/test_handler.py b/backend/tests/baserow/core/ai_provider/test_handler.py index 5961ddc899..7b67d3534c 100644 --- a/backend/tests/baserow/core/ai_provider/test_handler.py +++ b/backend/tests/baserow/core/ai_provider/test_handler.py @@ -904,3 +904,103 @@ class SecondFeatureType(AIProviderModelFeatureType): assert exc_info.value.model_identifier == "first-model" assert exc_info.value.feature_types == ["first_feature"] assert second_model.id + + +@pytest.mark.django_db +def test_get_model_usage_counts_per_consumer_features_only(monkeypatch): + class CountingFeatureType(AIProviderModelFeatureType): + type = "counting_feature" + + def count_model_references( + self, provider_type, model_identifier, workspace=None + ): + return 3 + + class PlainFeatureType(AIProviderModelFeatureType): + type = "plain_feature" + + class DefaultModelFeatureType(AIProviderModelFeatureType): + type = "default_model_feature" + supports_default_model = True + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + { + "counting_feature": CountingFeatureType(), + "plain_feature": PlainFeatureType(), + "default_model_feature": DefaultModelFeatureType(), + }, + ) + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-4o" + ) + + assert AIProviderHandler.get_model_usage(model) == { + "counting_feature": 3, + "plain_feature": 0, + } + + +@pytest.mark.django_db +def test_get_model_usage_passes_the_provider_scope(monkeypatch, data_fixture): + calls = [] + + class RecordingFeatureType(AIProviderModelFeatureType): + type = "recording_feature" + + def count_model_references( + self, provider_type, model_identifier, workspace=None + ): + calls.append((provider_type, model_identifier, workspace)) + return 0 + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + {"recording_feature": RecordingFeatureType()}, + ) + workspace = data_fixture.create_workspace() + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="secret", workspace=workspace + ) + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-4o" + ) + + assert AIProviderHandler.get_model_usage(model) == {"recording_feature": 0} + assert calls == [("openai", "gpt-4o", workspace)] + + +@pytest.mark.django_db +def test_get_model_usage_excludes_selected_default_model_feature_types(monkeypatch): + class ConsumerFeatureType(AIProviderModelFeatureType): + type = "consumer_feature" + + class DefaultModelFeatureType(AIProviderModelFeatureType): + type = "default_model_feature" + supports_default_model = True + + monkeypatch.setattr( + ai_provider_model_feature_type_registry, + "registry", + { + "consumer_feature": ConsumerFeatureType(), + "default_model_feature": DefaultModelFeatureType(), + }, + ) + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + model = AIProviderModel.objects.create( + provider_config=provider, + model_identifier="gpt-4o", + feature_types=["consumer_feature", "default_model_feature"], + ) + + assert AIProviderHandler.get_model_usage(model) == {"consumer_feature": 0} + + AIProviderHandler.update_feature_setting( + "default_model_feature", AI_PROVIDER_FEATURE_MODE_MODEL, model=model + ) + + assert AIProviderHandler.get_model_usage(model) == {"consumer_feature": 0} diff --git a/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json b/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json new file mode 100644 index 0000000000..ade253f91a --- /dev/null +++ b/changelog/entries/unreleased/feature/administrators_are_now_warned_which_ai_fields_and_ai_agent_a.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Administrators are now warned which AI fields and AI Agent actions use an AI provider model before it is disabled or deleted.", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-09-11" +} diff --git a/docs/installation/ai-assistant.md b/docs/installation/ai-assistant.md index f0ff4c83f2..f313be200a 100644 --- a/docs/installation/ai-assistant.md +++ b/docs/installation/ai-assistant.md @@ -29,6 +29,20 @@ a model; it does not force AI Fields or AI Agent actions to use Kuma's model. Us **Test model** to check every selected feature. AI Fields and AI Agent actions check for a text response, while Kuma also checks tool calling. +Changing a model that is already in use is confirmed, not applied silently. Disabling +or deleting a model, unchecking one of its features, or renaming its identifier first +reports how many AI Fields and AI Agent actions use it, and applies the change once +confirmed: those consumers store the provider type and the model identifier rather +than a reference to the model, so they keep the old selection and stop working until +they are repointed. Counts are per provider type and identifier, so an instance model +and a workspace model sharing an identifier report the same consumers. The Kuma +selection is a real reference instead, so disabling or deleting the model, or +unchecking Kuma, is refused while it is selected as the Kuma model: repoint that +selection first, in the **AI features** section of the scope that holds it. Renaming +is allowed and the Kuma selection follows the model to its new identifier. A consumer left on a model that is disabled or +gone keeps showing its saved provider and model, marked unavailable, so it can be +found and repointed. + For an existing installation, see the [AI provider upgrade and import instructions](../development/feature-flags.md#preparing-the-ai-providers-feature). Schema migrations run during the normal upgrade. Provider imports and republishing diff --git a/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py b/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py index 83e9b132ae..ae8a1e4fa8 100644 --- a/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py +++ b/premium/backend/src/baserow_premium/fields/ai_provider_feature_types.py @@ -1,11 +1,43 @@ from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_FIELDS from baserow.core.ai_provider.registries import AIProviderModelFeatureType from baserow.core.generative_ai.registries import generative_ai_model_type_registry +from baserow.core.models import Workspace +from baserow_premium.fields.models import AIField class AIFieldsAIProviderModelFeatureType(AIProviderModelFeatureType): type = AI_PROVIDER_FEATURE_AI_FIELDS + def count_model_references( + self, + provider_type: str, + model_identifier: str, + workspace: Workspace | None = None, + ) -> int: + """ + Count the AI fields selecting one provider model. + + A field is skipped as soon as any ancestor is trashed, because the whole + subtree disappears with it. + + :param provider_type: The provider type owning the model. + :param model_identifier: The identifier the fields persist. + :param workspace: The workspace to narrow to, or None for the instance + scope, which counts every workspace. + :return: The number of fields referencing the model. + """ + + queryset = AIField.objects.filter( + ai_generative_ai_type=provider_type, + ai_generative_ai_model=model_identifier, + table__trashed=False, + table__database__trashed=False, + table__database__workspace__trashed=False, + ) + if workspace is not None: + queryset = queryset.filter(table__database__workspace=workspace) + return queryset.count() + def get_workspace_availability(self, workspace, state=None) -> dict: models = generative_ai_model_type_registry.get_enabled_models_per_type( workspace, feature_type=self.type, state=state diff --git a/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py b/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py new file mode 100644 index 0000000000..f18ea5a036 --- /dev/null +++ b/premium/backend/tests/baserow_premium_tests/fields/test_ai_field_provider_feature_types.py @@ -0,0 +1,91 @@ +import pytest + +from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_FIELDS +from baserow.core.ai_provider.registries import ( + ai_provider_model_feature_type_registry, +) + + +def get_feature_type(): + return ai_provider_model_feature_type_registry.get(AI_PROVIDER_FEATURE_AI_FIELDS) + + +@pytest.mark.django_db +def test_ai_fields_count_model_references(premium_data_fixture): + table = premium_data_fixture.create_database_table() + workspace = table.database.workspace + other_table = premium_data_fixture.create_database_table() + other_workspace = other_table.database.workspace + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="another-model", + ) + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="anthropic", + ai_generative_ai_model="field-model", + ) + trashed_field = premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + trashed_field.trashed = True + trashed_field.save() + premium_data_fixture.create_ai_field( + table=other_table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + feature_type = get_feature_type() + + assert feature_type.count_model_references("openai", "field-model", workspace) == 1 + assert ( + feature_type.count_model_references("openai", "field-model", other_workspace) + == 1 + ) + assert feature_type.count_model_references("openai", "field-model", None) == 2 + assert feature_type.count_model_references("openai", "unknown-model", None) == 0 + assert feature_type.count_model_references("mistral", "field-model", None) == 0 + + +@pytest.mark.django_db +def test_ai_fields_count_model_references_ignores_trashed_ancestors( + premium_data_fixture, +): + table = premium_data_fixture.create_database_table() + database = table.database + workspace = database.workspace + premium_data_fixture.create_ai_field( + table=table, + ai_generative_ai_type="openai", + ai_generative_ai_model="field-model", + ) + feature_type = get_feature_type() + assert feature_type.count_model_references("openai", "field-model", None) == 1 + + table.trashed = True + table.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 + assert feature_type.count_model_references("openai", "field-model", workspace) == 0 + + table.trashed = False + table.save() + database.trashed = True + database.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 + + database.trashed = False + database.save() + workspace.trashed = True + workspace.save() + + assert feature_type.count_model_references("openai", "field-model", None) == 0 diff --git a/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js b/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js index 1abf254651..e7f070314a 100644 --- a/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js +++ b/premium/web-frontend/test/unit/premium/components/field/fieldAISubForm.spec.js @@ -87,7 +87,7 @@ describe('FieldAISubForm component', () => { expect(wrapper.vm.isFormValid()).toBe(true) }) - test('a disabled model is removed before it can be selected', async () => { + test('a disabled model stays visible but cannot be selected', async () => { const wrapper = await mountComponent( { formula: "'hello'", @@ -100,10 +100,15 @@ describe('FieldAISubForm component', () => { } ) - const optionNames = wrapper - .findAll('.select__item-name-text') - .map((item) => item.text()) - expect(optionNames).not.toContain('disabled-model') + const disabledOption = wrapper + .findAll('.select__item') + .find( + (option) => + option.find('.select__item-name-text').text() === 'disabled-model' + ) + expect(disabledOption).toBeTruthy() + expect(disabledOption.classes()).toContain('disabled') + wrapper.vm.submit() await wrapper.vm.$nextTick() expect(wrapper.emitted('submitted')).toBeUndefined() diff --git a/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue b/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue index 264ff2bb14..299337838b 100644 --- a/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue +++ b/web-frontend/modules/core/components/ai/AIProviderModelFormModal.vue @@ -1,5 +1,10 @@ diff --git a/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue b/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue new file mode 100644 index 0000000000..1c70a6346e --- /dev/null +++ b/web-frontend/modules/database/components/view/grid/GridViewGroupByColumns.vue @@ -0,0 +1,76 @@ + + + diff --git a/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue b/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue index a1a6c0cb81..7140ba4521 100644 --- a/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue +++ b/web-frontend/modules/database/components/view/grid/GridViewGroupByRows.vue @@ -19,10 +19,10 @@
@@ -130,6 +130,11 @@ export default { required: false, default: () => false, }, + groupByWidths: { + type: Array, + required: false, + default: () => [], + }, includeAddField: { type: Boolean, required: false, @@ -198,6 +203,9 @@ export default { const field = allFieldsInTable.find((f) => f.id === groupBy.field) return field }, + getGroupByWidth(groupBy, index) { + return this.groupByWidths[index] ?? groupBy.width + }, }, } diff --git a/web-frontend/modules/database/components/view/grid/GridViewSection.vue b/web-frontend/modules/database/components/view/grid/GridViewSection.vue index 1f33eab688..9725d0a739 100644 --- a/web-frontend/modules/database/components/view/grid/GridViewSection.vue +++ b/web-frontend/modules/database/components/view/grid/GridViewSection.vue @@ -18,16 +18,19 @@ class="grid-view__group-by-divider" :style="{ left: left + 'px' }" >
- +
+
+
{{ $t('gridView.rowCount', { count }) }}
@@ -198,6 +214,7 @@ import GridViewHead from '@baserow/modules/database/components/view/grid/GridVie import GridViewPlaceholder from '@baserow/modules/database/components/view/grid/GridViewPlaceholder' import GridViewRows from '@baserow/modules/database/components/view/grid/GridViewRows' import GridViewGroupByRows from '@baserow/modules/database/components/view/grid/GridViewGroupByRows' +import GridViewGroupByColumns from '@baserow/modules/database/components/view/grid/GridViewGroupByColumns' import GridViewRowAdd from '@baserow/modules/database/components/view/grid/GridViewRowAdd' import gridViewHelpers from '@baserow/modules/database/mixins/gridViewHelpers' import GridViewFieldFooter from '@baserow/modules/database/components/view/grid/GridViewFieldFooter' @@ -211,6 +228,7 @@ export default { GridViewPlaceholder, GridViewRows, GridViewGroupByRows, + GridViewGroupByColumns, GridViewRowAdd, GridViewFieldFooter, }, @@ -254,6 +272,11 @@ export default { required: false, default: () => false, }, + groupByWidths: { + type: Array, + required: false, + default: () => [], + }, includeAddField: { type: Boolean, required: false, @@ -321,6 +344,9 @@ export default { fieldsLeftOffset: 0, resizeObserver: null, horizontalScrollEvent: null, + // Keep the active handle mounted until mouseup can persist its width, even + // when dragging past the available space activates responsive fitting. + resizingGroupWidth: false, } }, computed: { @@ -338,7 +364,7 @@ export default { width += this.gridViewRowDetailsWidth } if (this.includeGroupBy) { - width += this.activeGroupByWidth + width += this.groupColumnsWidth } // The add button has a width of 100 and we reserve 100 at the right side. @@ -348,20 +374,38 @@ export default { return width }, + renderedGroupByWidths() { + return this.activeGroupBys.map( + (groupBy, index) => this.groupByWidths[index] ?? groupBy.width + ) + }, + groupByWidthsAreResponsivelyFitted() { + const configuredWidth = this.activeGroupBys.reduce( + (total, groupBy) => + total + Math.max(groupBy.width, this.GRID_VIEW_MIN_FIELD_WIDTH), + 0 + ) + return this.groupColumnsWidth < configuredWidth - 0.01 + }, + groupColumnsWidth() { + if (!this.includeGroupBy) { + return 0 + } + return this.renderedGroupByWidths.reduce( + (total, width) => total + width, + 0 + ) + }, groupByDividers() { if (!this.includeGroupBy) { return [] } let last = 0 - const dividers = this.activeGroupBys - .filter((groupBy, index) => index < this.activeGroupBys.length - 1) - .map((groupBy) => { - last += groupBy.width - return { groupBy, left: last } - }) - - return dividers + return this.activeGroupBys.map((groupBy, index) => { + last += this.renderedGroupByWidths[index] + return { groupBy, left: last } + }) }, useGroupByRows() { return this.activeGroupBys.length > 0 diff --git a/web-frontend/modules/database/locales/en.json b/web-frontend/modules/database/locales/en.json index e8314e98f5..b66816cbc1 100644 --- a/web-frontend/modules/database/locales/en.json +++ b/web-frontend/modules/database/locales/en.json @@ -601,6 +601,7 @@ "viewSettingsRowHeightSize": "row height", "viewSettingsFrozenColumnCount": "frozen columns", "viewSettingsRowIdentifierType": "row identifier", + "viewSettingsGroupByLayout": "group layout", "filters": "Filters", "sorts": "Sorts", "groupBys": "Groups", @@ -643,6 +644,9 @@ "maxGroupBysReached": "You can group by up to {count} fields.", "collapseAllGroups": "Collapse all", "expandAllGroups": "Expand all", + "layout": "Group layout", + "layoutSection": "Sections", + "layoutColumn": "Columns", "hiddenFieldWarning": "One or more group bys reference hidden fields that won't be visible to editors and lower roles." }, "viewGroupBy": { diff --git a/web-frontend/modules/database/mixins/gridViewGroupByValue.js b/web-frontend/modules/database/mixins/gridViewGroupByValue.js new file mode 100644 index 0000000000..a008db7f10 --- /dev/null +++ b/web-frontend/modules/database/mixins/gridViewGroupByValue.js @@ -0,0 +1,83 @@ +// Host components provide the `groupByField`, `groupPath` and `groupDisplay` computeds. +export default { + computed: { + fieldType() { + if (!this.groupByField) { + return null + } + return this.$registry.get('field', this.groupByField.type) + }, + groupValue() { + const field = this.groupByField + if (!field) { + return null + } + return this.groupPath[`field_${field.id}`] + }, + displayValue() { + const field = this.groupByField + const display = this.groupDisplay + if (!field || !display) { + return undefined + } + const key = `field_${field.id}` + return key in display ? display[key] : undefined + }, + rowValueForGroup() { + const field = this.groupByField + if (!field || !this.fieldType) { + return null + } + // Reference fields (selects, links, collaborators) only render from `display`. + if (this.displayValue !== undefined) { + return this.displayValue + } + return this.fieldType.getRowValueFromGroupValue(field, this.groupValue) + }, + isEmptyValue() { + const value = this.rowValueForGroup + if (Array.isArray(value)) { + return value.length === 0 + } + return value === null || value === undefined || value === '' + }, + groupByComponent() { + if (this.isEmptyValue || !this.groupByField || !this.fieldType) { + return null + } + if (typeof this.fieldType.getGroupByComponent !== 'function') { + return null + } + return this.fieldType.getGroupByComponent(this.groupByField) + }, + fallbackValueText() { + const value = this.rowValueForGroup + if (value === null || value === undefined) { + return '' + } + if (typeof value === 'boolean') { + return value ? 'true' : 'false' + } + if (this.fieldType?.toHumanReadableString) { + try { + const text = this.fieldType.toHumanReadableString( + this.groupByField, + value + ) + if (typeof text === 'string') { + return text + } + } catch (_) { + // Fall back to the generic object/string rendering below. + } + } + if (typeof value === 'object') { + if ('value' in value) { + return value.value + } + return JSON.stringify(value) + } + return String(value) + }, + }, +} diff --git a/web-frontend/modules/database/store/view/grid.js b/web-frontend/modules/database/store/view/grid.js index 5615d14be0..075d84dc03 100644 --- a/web-frontend/modules/database/store/view/grid.js +++ b/web-frontend/modules/database/store/view/grid.js @@ -44,6 +44,8 @@ import { renderViewport, visibleGroupPagesInViewport, visibleSectionsInViewport, + GROUP_BY_LAYOUT_SECTION, + GROUP_BY_LAYOUT_COLUMN, } from '@baserow/modules/database/utils/gridGroupByRender' import { getGroupByCollapseAllState, @@ -92,6 +94,11 @@ const REFRESH_ROW_DELAY_MS = 1000 // geometry is unaffected. High enough that a viewport's working set never evicts, so // collapse then re-expand never refetches. const GROUP_BY_MAX_RETAINED_SECTIONS = 200 +// Loading a sparse group page can change the estimated height of earlier unloaded +// pages, moving the same viewport onto another page. Refine until the geometry +// converges, but cap the number of sequential network round trips for corrupt or +// adversarial page data. The per-request seen sets below also prevent duplicate work. +const GROUP_BY_MAX_VIEWPORT_REFINEMENT_PASSES = 100 const DEFAULT_FLAT_VIEW = { group_bys: [], sortings: [], @@ -240,6 +247,8 @@ function getGroupByLayoutFromState(state) { state.rowHeight, state.bufferRequestSize, state.activeGroupBys, + state.groupByLayout, + state.count, ] const cached = groupByLayoutCacheByState.get(state) if ( @@ -258,6 +267,8 @@ function getGroupByLayoutFromState(state) { fields: getGroupByFieldRefsFromState(state), rowHeight: state.rowHeight, pageSize: state.bufferRequestSize, + layout: state.groupByLayout, + rootRowCount: state.count, }) groupByLayoutCacheByState.set(state, { key: cacheKey, value: layout }) @@ -535,6 +546,8 @@ function getGroupBySnapshotLayout(snapshot, activeGroupBys, fields, state) { fields: groupByFields, rowHeight: state.rowHeight, pageSize: state.bufferRequestSize, + layout: state.groupByLayout, + rootRowCount: state.count, }) } @@ -965,6 +978,7 @@ export const state = () => ({ count: 0, // The height of a single row. rowHeight: 33, + groupByLayout: GROUP_BY_LAYOUT_SECTION, // The distance to the top in pixels the visible rows should have. rowsTop: 0, // The amount of rows that must be visible above and under the middle row. @@ -1718,6 +1732,12 @@ export const mutations = { UPDATE_ROW_HEIGHT(state, value) { state.rowHeight = value }, + SET_GROUP_BY_LAYOUT(state, value) { + state.groupByLayout = + value === GROUP_BY_LAYOUT_COLUMN + ? GROUP_BY_LAYOUT_COLUMN + : GROUP_BY_LAYOUT_SECTION + }, ADD_CHECKBOX_SELECTED_ROW(state, rowId) { if (!state.checkboxSelectedRows.includes(rowId)) { state.checkboxSelectedRows.push(rowId) @@ -2167,7 +2187,11 @@ export const actions = { fields ) const viewport = getGroupByViewport(getters, scrollTop) - const useDepthPages = shouldUseGroupByDepthPages(state.groupBy) + // Columns always render every level expanded, even when the saved Section + // collapse state is collapse-all. Its fetch strategy must match that layout. + const forceExpandedLayout = getters.isGroupByColumnLayout + const useDepthPages = + !forceExpandedLayout && shouldUseGroupByDepthPages(state.groupBy) let layout = getters.getGroupByLayout if ( @@ -2179,7 +2203,8 @@ export const actions = { view, fields, adhocFiltering: getters.getAdhocFiltering, - includeDescendants: state.groupBy.collapse?.mode === 'expand', + includeDescendants: + forceExpandedLayout || state.groupBy.collapse?.mode === 'expand', descendantLimit: getters.getBufferRequestSize, descendantRowBudget: getGroupByDescendantRowBudget(getters, scrollTop), signal, @@ -2192,7 +2217,11 @@ export const actions = { const seenPageRequests = new Set() const seenDepthRequests = new Set() - for (let depth = 0; depth < groupByFields.length; depth += 1) { + for ( + let pass = 0; + pass < GROUP_BY_MAX_VIEWPORT_REFINEMENT_PASSES; + pass += 1 + ) { if (useDepthPages) { const pageToFetch = getVisibleGroupDepthPageToFetch({ layout, @@ -2214,6 +2243,9 @@ export const actions = { adhocFiltering: getters.getAdhocFiltering, signal, }) + if (isGroupByRequestStale(getters, state, groupByGeneration, signal)) { + return [] + } layout = getters.getGroupByLayout continue } @@ -2347,6 +2379,9 @@ export const actions = { { commit, dispatch, getters }, { path, view, fields } ) { + if (getters.isGroupByColumnLayout) { + return + } const groupByFields = getGroupByFieldsFromActiveGroupBys( getters.getActiveGroupBys, fields @@ -2368,6 +2403,9 @@ export const actions = { { commit, dispatch, getters }, { view, fields, collapse } ) { + if (getters.isGroupByColumnLayout) { + return + } commit('SET_GROUP_BY_COLLAPSE', getGroupByCollapseAllState(collapse)) commit('CLEAR_AREA_SELECTION') const scrollTop = getClampedGroupByScrollTop(getters) @@ -3080,6 +3118,13 @@ export const actions = { groupByCollapse = getGroupByCollapseAllState(false) } + // Keep the persisted collapse state in the snapshot so switching back to + // Sections restores it. Only the fetch decisions are forced to expand for Columns. + const forceExpandedLayout = getters.isGroupByColumnLayout + const effectiveExpandAll = + forceExpandedLayout || + (groupByCollapse.mode === 'expand' && groupByCollapse.paths.length === 0) + commit('INVALIDATE_GROUP_BY_REQUESTS') const groupByGeneration = state.groupBy.generation || 0 const snapshot = { @@ -3093,9 +3138,7 @@ export const actions = { // first page is independent of the new tree and can be fetched in parallel with the // skeleton. Mixed/collapsed states tie their offsets to the tree, so they can't. const parallelExpandRows = - scrollTop === 0 && - groupByCollapse.mode === 'expand' && - groupByCollapse.paths.length === 0 + scrollTop === 0 && effectiveExpandAll ? GridService($client).fetchRows({ gridId, offset: 0, @@ -3124,7 +3167,8 @@ export const actions = { filters: getFilters(view, getters.getAdhocFiltering), offset: 0, limit: getters.getBufferRequestSize, - includeDescendants: groupByCollapse.mode === 'expand', + includeDescendants: + forceExpandedLayout || groupByCollapse.mode === 'expand', descendantLimit: getters.getBufferRequestSize, descendantRowBudget: getGroupByDescendantRowBudget(getters), groupBy: getGroupBy(rootGetters, gridId, getters.getAdhocGrouping), @@ -3949,7 +3993,7 @@ export const actions = { const startRowIndex = getters.getMultiSelectStartRowIndex const startFieldIndex = getters.getMultiSelectStartFieldIndex - const maxRowIndex = getters.getRowsLength + getters.getBufferStartIndex - 1 + const maxRowIndex = getters.getSelectionMaxRowIndex const maxFieldIndex = getters.getNumberOfVisibleFields - 1 if (headRowIndex > maxRowIndex || headFieldIndex > maxFieldIndex) { @@ -5284,7 +5328,7 @@ export const actions = { } if ( - rowIndex > getters.getRowsLength + getters.getBufferStartIndex - 1 || + rowIndex > getters.getSelectionMaxRowIndex || fieldIndex > getters.getNumberOfVisibleFields - 1 ) { return @@ -6419,6 +6463,9 @@ export const actions = { setRowHeight({ commit, dispatch, getters }, value) { commit('UPDATE_ROW_HEIGHT', value) }, + setGroupByLayout({ commit }, value) { + commit('SET_GROUP_BY_LAYOUT', value) + }, toggleCheckboxRowSelection({ commit, dispatch, state, getters }, { row }) { const { $registry, $client, $i18n, $config } = this const rowId = row.id @@ -6497,6 +6544,14 @@ export const getters = { } return state.rows.length }, + getSelectionMaxRowIndex(state, getters) { + // Columns use absolute row offsets, including unloaded group pages. Sections + // number only the expanded sections, while flat grids use their row buffer. + if (getters.isGroupByMode && getters.isGroupByColumnLayout) { + return state.count - 1 + } + return getters.getRowsLength + getters.getBufferStartIndex - 1 + }, getPlaceholderHeight(state) { return state.count * state.rowHeight }, @@ -6780,6 +6835,12 @@ export const getters = { getGroupByCollapse(state) { return state.groupBy.collapse }, + getGroupByLayoutMode(state) { + return state.groupByLayout + }, + isGroupByColumnLayout(state) { + return state.groupByLayout === GROUP_BY_LAYOUT_COLUMN + }, // Takes no field arg so the memoization in getGroupByLayoutFromState holds: a // per-call fields array would be a new reference every time and defeat the cache. getGroupByLayout(state) { diff --git a/web-frontend/modules/database/utils/gridGroupByRender.js b/web-frontend/modules/database/utils/gridGroupByRender.js index 929b9f11eb..57e866b084 100644 --- a/web-frontend/modules/database/utils/gridGroupByRender.js +++ b/web-frontend/modules/database/utils/gridGroupByRender.js @@ -5,6 +5,102 @@ export const GROUP_GAP = 8 // matching the fixed-height ungrouped `GridViewRowAdd`. export const ADD_ROW_HEIGHT = ROW_HEIGHT +export const GROUP_BY_LAYOUT_SECTION = 'section' +export const GROUP_BY_LAYOUT_COLUMN = 'column' + +const SECTION_GEOMETRY = Object.freeze({ + layout: GROUP_BY_LAYOUT_SECTION, + headerHeight: HEADER_HEIGHT, + groupGap: GROUP_GAP, + addRowHeight: ADD_ROW_HEIGHT, + addRowPerGroup: true, + unloadedGroupHeight: HEADER_HEIGHT, +}) + +const EXPAND_ALL = Object.freeze({ mode: 'expand', paths: [] }) + +export function getLayoutGeometry(layout, rowHeight = ROW_HEIGHT) { + if (layout !== GROUP_BY_LAYOUT_COLUMN) { + return SECTION_GEOMETRY + } + return { + layout: GROUP_BY_LAYOUT_COLUMN, + headerHeight: 0, + groupGap: 0, + addRowHeight: ADD_ROW_HEIGHT, + addRowPerGroup: false, + // A row per unloaded group keeps the placeholder tall enough to reach the viewport. + unloadedGroupHeight: rowHeight, + } +} + +function layoutGeometryOf(layout) { + return layout?.geometry || SECTION_GEOMETRY +} + +function pushGroupHeader(items, geometry, rowHeight, node) { + if (geometry.headerHeight === 0) { + items.push({ + type: 'groupSpan', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + y: node.y, + height: node.rowCount * rowHeight, + }) + return node.y + } + items.push({ + type: 'header', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + aggregationRowCount: node.aggregationRowCount, + y: node.y, + height: geometry.headerHeight, + collapsed: node.collapsed, + gapAbove: node.gapAbove, + aggregations: node.aggregations, + }) + return node.y + geometry.headerHeight +} + +function pushGroupAddRow(items, geometry, node) { + if (!geometry.addRowPerGroup) { + return node.y + } + items.push({ + type: 'addRow', + depth: node.depth, + path: node.path, + display: node.display, + rowCount: node.rowCount, + y: node.y, + height: geometry.addRowHeight, + }) + return node.y + geometry.addRowHeight +} + +function pushTrailingAddRow(items, geometry, y, rootPageLoaded) { + // Without per-group add-row lines the grid must always end with one. + const needed = geometry.addRowPerGroup + ? rootPageLoaded && items.length === 0 + : rootPageLoaded || items.length > 0 + if (!needed) { + return y + } + items.push({ + type: 'addRow', + depth: 0, + path: {}, + y, + height: geometry.addRowHeight, + }) + return y + geometry.addRowHeight +} + const GROUP_BANNER_DEPTH_INDENT_PX = 24 const GROUP_BANNER_BASE_GUTTER = 12 const GROUP_BANNER_CHEVRON_WIDTH = 24 @@ -104,9 +200,23 @@ export function buildLayout({ fields, rowHeight = ROW_HEIGHT, pageSize = GROUP_PAGE_SIZE, + layout = GROUP_BY_LAYOUT_SECTION, + rootRowCount = null, }) { + const geometry = getLayoutGeometry(layout, rowHeight) + const effectiveCollapse = + geometry.layout === GROUP_BY_LAYOUT_COLUMN ? EXPAND_ALL : collapse + if (pages !== null) { - return buildPagedLayout({ pages, collapse, fields, rowHeight, pageSize }) + return buildPagedLayout({ + pages, + collapse: effectiveCollapse, + fields, + rowHeight, + pageSize, + geometry, + rootRowCount, + }) } const items = [] @@ -114,11 +224,11 @@ export function buildLayout({ let visibleRowCount = 0 if (!nodes || nodes.length === 0 || !fields || fields.length === 0) { - return { items, totalHeight: 0, totalRowCount: 0 } + return { items, totalHeight: 0, totalRowCount: 0, geometry } } const maxDepth = fields.length - 1 - const exceptionKeys = collapseExceptionKeys(collapse, fields) + const exceptionKeys = collapseExceptionKeys(effectiveCollapse, fields) let skipDescendantsAtDepth = -1 let lastPlacedDepth = null @@ -139,13 +249,13 @@ export function buildLayout({ // its parent's banner (a first child) sits flush against it. const gapAbove = lastPlacedDepth !== null && lastPlacedDepth >= node.depth if (gapAbove) { - y += GROUP_GAP + y += geometry.groupGap } lastPlacedDepth = node.depth const collapsed = pathCollapsedAgainst( node.path, - collapse.mode, + effectiveCollapse.mode, exceptionKeys, fields ) @@ -153,20 +263,17 @@ export function buildLayout({ const aggregationRowCount = node.aggregationRowCount ?? node.aggregation_row_count ?? rowCount - items.push({ - type: 'header', + y = pushGroupHeader(items, geometry, rowHeight, { depth: node.depth, path: node.path, display: node.display, rowCount, aggregationRowCount, y, - height: HEADER_HEIGHT, collapsed, gapAbove, aggregations: node.aggregations ?? null, }) - y += HEADER_HEIGHT if (collapsed) { skipDescendantsAtDepth = node.depth @@ -188,20 +295,19 @@ export function buildLayout({ }) y += sectionHeight visibleRowCount += rowCount - items.push({ - type: 'addRow', + y = pushGroupAddRow(items, geometry, { depth: node.depth, path: node.path, display: node.display, rowCount, y, - height: ADD_ROW_HEIGHT, }) - y += ADD_ROW_HEIGHT } } - return { items, totalHeight: y, totalRowCount: visibleRowCount } + y = pushTrailingAddRow(items, geometry, y, true) + + return { items, totalHeight: y, totalRowCount: visibleRowCount, geometry } } function getPage(pages, parentPath, fields) { @@ -214,38 +320,41 @@ function getSortedLoadedIndexes(page) { .sort((a, b) => a - b) } -function unloadedGroupRangeHeight(startIndex, endIndex) { +function unloadedGroupRangeHeight( + startIndex, + endIndex, + geometry, + rowSlotCount = null +) { const count = Math.max(0, endIndex - startIndex) if (count === 0) { return 0 } + if (geometry.layout === GROUP_BY_LAYOUT_COLUMN && rowSlotCount !== null) { + return Math.max(0, rowSlotCount) * geometry.unloadedGroupHeight + } const gaps = count - (startIndex === 0 ? 1 : 0) - return count * HEADER_HEIGHT + gaps * GROUP_GAP + return count * geometry.unloadedGroupHeight + gaps * geometry.groupGap } -function placeholderStartIndexAtOffset(item, offset) { +function placeholderStartIndexAtOffset(item, offset, geometry) { const startIndex = item.siblingStartIndex const endIndex = item.siblingEndIndex const clampedOffset = Math.max(0, Math.min(offset, item.height)) + const slotHeight = geometry.unloadedGroupHeight + const stride = slotHeight + geometry.groupGap if (startIndex === 0) { - if (clampedOffset < HEADER_HEIGHT) { + if (clampedOffset < slotHeight) { return startIndex } return Math.min( endIndex, - startIndex + - 1 + - Math.floor( - (clampedOffset - HEADER_HEIGHT) / (HEADER_HEIGHT + GROUP_GAP) - ) + startIndex + 1 + Math.floor((clampedOffset - slotHeight) / stride) ) } - return Math.min( - endIndex, - startIndex + Math.floor(clampedOffset / (HEADER_HEIGHT + GROUP_GAP)) - ) + return Math.min(endIndex, startIndex + Math.floor(clampedOffset / stride)) } function pushUnloadedGroupPlaceholder({ @@ -256,8 +365,15 @@ function pushUnloadedGroupPlaceholder({ endIndex, globalStartIndex, y, + geometry, + rowSlotCount = null, }) { - const height = unloadedGroupRangeHeight(startIndex, endIndex) + const height = unloadedGroupRangeHeight( + startIndex, + endIndex, + geometry, + rowSlotCount + ) if (height <= 0) { return y } @@ -282,13 +398,15 @@ function buildPagedLayout({ fields, rowHeight = ROW_HEIGHT, pageSize = GROUP_PAGE_SIZE, + geometry = SECTION_GEOMETRY, + rootRowCount = null, }) { const items = [] let y = 0 let visibleRowCount = 0 if (!fields || fields.length === 0) { - return { items, totalHeight: 0, totalRowCount: 0 } + return { items, totalHeight: 0, totalRowCount: 0, geometry } } const maxDepth = fields.length - 1 @@ -307,7 +425,13 @@ function buildPagedLayout({ const advanceGlobalSiblingCount = (d, count) => globalSiblingCountByDepth.set(d, globalSiblingCount(d) + count) - const walkPage = (parentPath, depth, fallbackSiblingCount = 0) => { + const walkPage = ( + parentPath, + depth, + fallbackSiblingCount = 0, + parentRowCount = null, + parentRowOffset = null + ) => { const pageCacheKey = pathKey(parentPath, fields) if (visitedPageKeys.has(pageCacheKey)) { return @@ -318,7 +442,61 @@ function buildPagedLayout({ page?.totalSiblingCount ?? page?.total_sibling_count ?? fallbackSiblingCount + const pageY = y const loadedIndexes = getSortedLoadedIndexes(page) + const validLoadedIndexes = loadedIndexes.filter( + (loadedIndex) => loadedIndex >= 0 && loadedIndex < totalSiblingCount + ) + let remainingUnloadedSiblingCount = null + let remainingUnloadedRowCount = null + if ( + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + Number.isFinite(parentRowCount) + ) { + const loadedRowCount = validLoadedIndexes.reduce((total, loadedIndex) => { + const loadedNode = page?.nodes?.[loadedIndex] + return total + (loadedNode?.rowCount ?? loadedNode?.row_count ?? 0) + }, 0) + remainingUnloadedSiblingCount = Math.max( + 0, + totalSiblingCount - validLoadedIndexes.length + ) + // Server groups are non-empty, so reserve at least one row for each sibling + // when optimistic/stale counts momentarily disagree. Otherwise distribute the + // parent's rows not accounted for by loaded siblings across the missing ranges. + remainingUnloadedRowCount = Math.max( + remainingUnloadedSiblingCount, + parentRowCount - loadedRowCount + ) + } + + const takeUnloadedRowSlots = ( + siblingCount, + gapSiblingCount, + gapRowCount + ) => { + const anchoredGap = + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + Number.isFinite(gapRowCount) + const siblingBudget = anchoredGap + ? gapSiblingCount + : remainingUnloadedSiblingCount + const rowBudget = anchoredGap + ? Math.max(gapSiblingCount, gapRowCount) + : remainingUnloadedRowCount + if (siblingBudget === null || rowBudget === null) { + return null + } + + const extraRows = Math.max(0, rowBudget - siblingBudget) + const rowSlots = + siblingCount + Math.floor((extraRows * siblingCount) / siblingBudget) + if (remainingUnloadedSiblingCount !== null) { + remainingUnloadedSiblingCount -= siblingCount + remainingUnloadedRowCount -= rowSlots + } + return rowSlots + } let loadedPointer = 0 let index = 0 // Whether a sibling was already placed at this depth. Drives the depth-0 gap, which a @@ -332,6 +510,22 @@ function buildPagedLayout({ loadedIndex === undefined ? totalSiblingCount : loadedIndex, Math.ceil((index + 1) / pageSize) * pageSize ) + const unloadedSiblingCount = unloadedEnd - index + // A loaded node's absolute offset anchors the entire preceding gap. Only + // estimate page boundaries inside that gap: distributing rows across gaps + // on both sides of a loaded node would move it away from its actual rows. + const nextNode = page?.nodes?.[loadedIndex] + const gapEndRowOffset = + loadedIndex !== undefined + ? (nextNode?.rowOffset ?? nextNode?.row_offset) + : Number.isFinite(parentRowOffset) && + Number.isFinite(parentRowCount) + ? parentRowOffset + parentRowCount + : null + const gapRowCount = + Number.isFinite(parentRowOffset) && Number.isFinite(gapEndRowOffset) + ? gapEndRowOffset - parentRowOffset - (y - pageY) / rowHeight + : null y = pushUnloadedGroupPlaceholder({ items, parentPath, @@ -340,8 +534,14 @@ function buildPagedLayout({ endIndex: unloadedEnd, globalStartIndex: globalSiblingCount(depth), y, + geometry, + rowSlotCount: takeUnloadedRowSlots( + unloadedSiblingCount, + (loadedIndex ?? totalSiblingCount) - index, + gapRowCount + ), }) - advanceGlobalSiblingCount(depth, unloadedEnd - index) + advanceGlobalSiblingCount(depth, unloadedSiblingCount) index = unloadedEnd placedSibling = true continue @@ -365,7 +565,7 @@ function buildPagedLayout({ } const gapAbove = placedSibling if (gapAbove) { - y += GROUP_GAP + y += geometry.groupGap } placedSibling = true @@ -380,24 +580,22 @@ function buildPagedLayout({ node.aggregationRowCount ?? node.aggregation_row_count ?? rowCount const childrenCount = node.childrenCount ?? node.children_count ?? 0 - items.push({ - type: 'header', + y = pushGroupHeader(items, geometry, rowHeight, { depth: node.depth ?? depth, path: node.path, display: node.display, rowCount, aggregationRowCount, y, - height: HEADER_HEIGHT, collapsed, gapAbove, aggregations: node.aggregations ?? null, }) - y += HEADER_HEIGHT if (!collapsed) { if ((node.depth ?? depth) === maxDepth) { const sectionHeight = rowCount * rowHeight + const absoluteRowOffset = node.rowOffset ?? node.row_offset items.push({ type: 'rowSection', depth: node.depth ?? depth, @@ -406,23 +604,31 @@ function buildPagedLayout({ rowCount, y, height: sectionHeight, - firstGlobalRowOffset: visibleRowCount, - absoluteRowOffset: node.rowOffset ?? node.row_offset ?? 0, + firstGlobalRowOffset: + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + absoluteRowOffset !== undefined && + absoluteRowOffset !== null + ? absoluteRowOffset + : visibleRowCount, + absoluteRowOffset: absoluteRowOffset ?? 0, }) y += sectionHeight visibleRowCount += rowCount - items.push({ - type: 'addRow', + y = pushGroupAddRow(items, geometry, { depth: node.depth ?? depth, path: node.path, display: node.display, rowCount, y, - height: ADD_ROW_HEIGHT, }) - y += ADD_ROW_HEIGHT } else { - walkPage(node.path, (node.depth ?? depth) + 1, childrenCount) + walkPage( + node.path, + (node.depth ?? depth) + 1, + childrenCount, + rowCount, + node.rowOffset ?? node.row_offset ?? null + ) } } @@ -432,25 +638,12 @@ function buildPagedLayout({ } } - walkPage({}, 0) + walkPage({}, 0, 0, rootRowCount, 0) - // A loaded but group-less view (no rows, or every group emptied) would otherwise - // render nothing to click, so keep one top-level add-row line available. Gated on - // the loaded root page so nothing flashes while the first request is in flight, - // and on a fully empty layout so it never stacks under loading placeholders. const rootPageLoaded = getPage(pages, {}, fields) !== null - if (rootPageLoaded && items.length === 0) { - items.push({ - type: 'addRow', - depth: 0, - path: {}, - y, - height: ADD_ROW_HEIGHT, - }) - y += ADD_ROW_HEIGHT - } + y = pushTrailingAddRow(items, geometry, y, rootPageLoaded) - return { items, totalHeight: y, totalRowCount: visibleRowCount } + return { items, totalHeight: y, totalRowCount: visibleRowCount, geometry } } /** @@ -574,7 +767,8 @@ export function visibleGroupDepthPageInViewport( const visibleStart = placeholderStartIndexAtOffset( item, - Math.max(top, item.y) - item.y + Math.max(top, item.y) - item.y, + layoutGeometryOf(layout) ) // globalSiblingStartIndex maps the placeholder's per-parent start into the // depth-wide sibling space the server offsets on, so batch-fetching the right page. @@ -607,6 +801,7 @@ export function renderViewport({ const items = [] const top = viewport.scrollTop const bottom = viewport.scrollTop + viewport.clientHeight + const geometry = layoutGeometryOf(layout) for (const item of layout.items) { const itemBottom = item.y + item.height @@ -617,6 +812,19 @@ export function renderViewport({ break } + if (item.type === 'groupSpan') { + items.push({ + type: 'groupSpan', + depth: item.depth, + path: item.path, + display: item.display, + rowCount: item.rowCount, + y: item.y, + height: item.height, + }) + continue + } + if (item.type === 'header') { items.push({ type: 'header', @@ -646,16 +854,26 @@ export function renderViewport({ } if (item.type === 'groupPlaceholder') { + const rangeBottom = item.y + item.height + if (geometry.headerHeight === 0) { + const visibleTop = Math.max(top, item.y) + items.push({ + type: 'groupRangePlaceholder', + depth: item.depth, + y: visibleTop, + height: Math.min(bottom, rangeBottom) - visibleTop, + }) + continue + } // Fill the unloaded region with a staircase of skeleton headers, one per level the // group still nests through (the level count is known before the data loads), so // the structure shows instead of blank space while the descendant request resolves. - const rangeBottom = item.y + item.height const maxDepth = Math.max((fields?.length ?? 1) - 1, item.depth) const levelsBelow = maxDepth - item.depth + 1 // Sibling groups are laid out with a gap between them (see // `unloadedGroupRangeHeight`), so step by the same stride or the staircase // over-produces slots across the gapped range. - const slotStep = HEADER_HEIGHT + GROUP_GAP + const slotStep = geometry.unloadedGroupHeight + geometry.groupGap const firstSlotIndex = Math.max(0, Math.floor((top - item.y) / slotStep)) let slotIndex = firstSlotIndex for ( @@ -667,7 +885,7 @@ export function renderViewport({ type: 'groupSkeleton', depth: item.depth + (slotIndex % levelsBelow), y: slotY, - height: Math.min(HEADER_HEIGHT, rangeBottom - slotY), + height: Math.min(geometry.unloadedGroupHeight, rangeBottom - slotY), }) } continue @@ -697,6 +915,9 @@ export function renderViewport({ sectionKey, position: i, globalRowOffset: item.firstGlobalRowOffset + i, + groupEnd: + geometry.layout === GROUP_BY_LAYOUT_COLUMN && + i === item.rowCount - 1, }) } else { items.push({ @@ -735,6 +956,9 @@ export function resolveGroupByRowMoveTarget({ const sourceSectionKey = pathKey(sourcePath, fields) for (const item of layout.items) { + if (item.type === 'groupSpan') { + continue + } if (contentY < item.y || contentY >= item.y + item.height) { continue } diff --git a/web-frontend/modules/database/utils/gridGroupByWidths.js b/web-frontend/modules/database/utils/gridGroupByWidths.js new file mode 100644 index 0000000000..53e049ba28 --- /dev/null +++ b/web-frontend/modules/database/utils/gridGroupByWidths.js @@ -0,0 +1,32 @@ +/** + * Returns the widths to use when rendering group-by columns in the available space. + * The configured widths are left untouched so they can be restored exactly when the + * viewport grows again. If the columns need to shrink, only the space above the + * minimum width is scaled, which also preserves columns explicitly set to the + * minimum. + */ +export function fitGroupByWidths(groupBys, availableWidth, minimumWidth) { + const widths = groupBys.map((groupBy) => + Math.max(groupBy.width, minimumWidth) + ) + const totalWidth = widths.reduce((total, width) => total + width, 0) + + if ( + availableWidth === null || + availableWidth === undefined || + totalWidth <= availableWidth + ) { + return widths + } + + const minimumTotalWidth = minimumWidth * widths.length + if (availableWidth <= minimumTotalWidth) { + return widths.map(() => minimumWidth) + } + + const flexibleWidth = totalWidth - minimumTotalWidth + const availableFlexibleWidth = availableWidth - minimumTotalWidth + const scale = availableFlexibleWidth / flexibleWidth + + return widths.map((width) => minimumWidth + (width - minimumWidth) * scale) +} diff --git a/web-frontend/modules/database/viewTypes.js b/web-frontend/modules/database/viewTypes.js index ffa356c86a..bcd9ca93b9 100644 --- a/web-frontend/modules/database/viewTypes.js +++ b/web-frontend/modules/database/viewTypes.js @@ -614,7 +614,12 @@ export class GridViewType extends ViewType { } getCopyableViewSettings() { - return ['row_height_size', 'frozen_column_count', 'row_identifier_type'] + return [ + 'row_height_size', + 'frozen_column_count', + 'row_identifier_type', + 'group_by_layout', + ] } getName() { @@ -671,6 +676,10 @@ export class GridViewType extends ViewType { storePrefix + 'view/grid/setRowHeight', GRID_VIEW_SIZE_TO_ROW_HEIGHT_MAPPING[view.row_height_size] ) + await store.dispatch( + storePrefix + 'view/grid/setGroupByLayout', + view.group_by_layout || 'section' + ) await store.dispatch(storePrefix + 'view/grid/fetchInitial', { gridId: view.id, fields, diff --git a/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap b/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap index cd1ece5f1f..ecce9616b1 100644 --- a/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap +++ b/web-frontend/test/unit/database/__snapshots__/publicView.spec.js.snap @@ -448,13 +448,11 @@ exports[`Public View Page Tests > Can see a publicly shared grid view 1`] = ` class="grid-view__placeholder" style="width: 72px;" > -
+
+
Can see a publicly shared grid view 1`] = `
+
@@ -662,6 +661,7 @@ exports[`Public View Page Tests > Can see a publicly shared grid view 1`] = ` />
+
Can see a publicly shared grid view 1`] = ` class="grid-view__foot" > +
Adding a row to a table increases the row count
-
+
+
@@ -177,6 +178,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
gridView.rowCount - 1
@@ -253,6 +255,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -281,6 +284,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -494,8 +498,9 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
-
+
+
@@ -516,6 +521,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
gridView.rowCount - 2
@@ -592,6 +598,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
@@ -620,6 +627,7 @@ exports[`Table Component Tests > Adding a row to a table increases the row count
+
diff --git a/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap b/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap index 48fc1acfbc..192bb08db9 100644 --- a/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap +++ b/web-frontend/test/unit/database/components/view/grid/__snapshots__/gridViewDecoration.spec.js.snap @@ -63,13 +63,11 @@ exports[`GridView component with decoration > Default component with first_cell class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with first_cell
+
@@ -346,6 +345,7 @@ exports[`GridView component with decoration > Default component with first_cell />
+
Default component with first_cell class="grid-view__foot" > +
Default component with row wrapper class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with row wrapper
+
@@ -874,6 +874,7 @@ exports[`GridView component with decoration > Default component with row wrapper />
+
Default component with row wrapper class="grid-view__foot" > +
Default component with unavailable class="grid-view__placeholder" style="height: 66px; width: 72px;" > -
+
+
Default component with unavailable
+
@@ -1402,6 +1403,7 @@ exports[`GridView component with decoration > Default component with unavailable />
+
Default component with unavailable class="grid-view__foot" > +
{ const fields = [ @@ -54,6 +56,170 @@ describe('GridView component', () => { expect(sortedFields.map((field) => field.id)).toEqual([1, 3]) }) + test('frozen columns fall back when group columns would crowd the viewport', () => { + const context = { + $refs: { gridView: { clientWidth: 800 } }, + fields: [{ id: 1, name: 'Primary', primary: true }], + fieldOptions: { 1: { hidden: false, order: 0 } }, + frozenColumnCount: 1, + gridViewRowDetailsWidth: 72, + groupColumnsWidth: 0, + getFieldWidth: () => 200, + canFitFrozenColumns: true, + } + + GridView.methods.checkCanFitFrozenColumns.call(context) + expect(context.canFitFrozenColumns).toBe(true) + + context.groupColumnsWidth = 400 + GridView.methods.checkCanFitFrozenColumns.call(context) + expect(context.canFitFrozenColumns).toBe(false) + }) + + test.each([ + [1024, 130.4], + // The default 240px application sidebar leaves a 784px grid at a 1024px + // browser viewport. + [784, 82.4], + ])( + 'five group columns leave a usable data pane at a %ipx grid width', + (gridViewWidth, expectedGroupWidth) => { + const groupBys = Array.from({ length: 5 }, (_, index) => ({ + id: index, + width: 200, + })) + const groupByWidths = GridView.computed.groupByWidths.call({ + isColumnLayout: true, + activeGroupBys: groupBys, + gridViewWidth, + gridViewRowDetailsWidth: 72, + }) + const groupColumnsWidth = GridView.computed.groupColumnsWidth.call({ + groupByWidths, + }) + const leftWidth = GridView.computed.leftWidth.call({ + leftFieldsWidth: 0, + gridViewRowDetailsWidth: 72, + groupColumnsWidth, + }) + + expect(groupByWidths).toEqual(Array(5).fill(expectedGroupWidth)) + expect(gridViewWidth - leftWidth).toBeCloseTo(300) + expect(groupBys.map(({ width }) => width)).toEqual(Array(5).fill(200)) + } + ) + + test('effective group width changes recalculate grid geometry', () => { + const fieldsUpdated = vi.fn() + const nextTick = vi.fn((callback) => callback()) + const context = { fieldsUpdated, $nextTick: nextTick } + + GridView.watch.groupColumnsWidth.call(context, 240, 200) + + expect(nextTick).toHaveBeenCalledOnce() + expect(fieldsUpdated).toHaveBeenCalledOnce() + + GridView.watch.groupColumnsWidth.call(context, 240, 240) + expect(nextTick).toHaveBeenCalledOnce() + expect(fieldsUpdated).toHaveBeenCalledOnce() + + expect( + GridView.computed.groupColumnsWidth.call({ groupByWidths: [] }) + ).toBe(0) + }) + + test.each([ + [[82.4, 82.4], [200, 200], true], + [[200, 200], [200, 200], false], + [[78, 200], [40, 200], false], + ])( + 'group width resize handles responsive state for rendered widths %j', + (renderedGroupByWidths, configuredWidths, expected) => { + expect( + GridViewSection.computed.groupByWidthsAreResponsivelyFitted.call({ + activeGroupBys: configuredWidths.map((width) => ({ width })), + renderedGroupByWidths, + groupColumnsWidth: renderedGroupByWidths.reduce( + (total, width) => total + width, + 0 + ), + GRID_VIEW_MIN_FIELD_WIDTH: 78, + }) + ).toBe(expected) + } + ) + + test('row dragging starts after group columns only in column layout', async () => { + const testApp = new TestApp() + try { + const mockServer = testApp.mockServer + const store = testApp.store + const table = mockServer.createTable() + const { application } = await mockServer.createAppAndWorkspace(table) + const groupBys = [ + { + id: 10, + field: 2, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + }, + ] + const view = mockServer.createGridView(application, table, { groupBys }) + view.group_by_layout = 'column' + mockServer.createFields(application, table, [ + { + id: 1, + name: 'Name', + order: 0, + type: 'text', + primary: true, + text_default: '', + }, + { + id: 2, + name: 'Team', + order: 1, + type: 'text', + primary: false, + text_default: '', + }, + ]) + await store.dispatch('field/fetchAll', { table }) + const mountedFields = store.getters['field/getAll'] + const primary = store.getters['field/getPrimary'] + mockServer.createGridRows(view, mountedFields, []) + await store.dispatch('page/view/grid/fetchInitial', { + gridId: view.id, + fields: mountedFields, + primary, + }) + await store.dispatch('page/view/grid/updateActiveGroupBys', groupBys) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + + const wrapper = await testApp.mount(GridView, { + props: { + fields: mountedFields, + view, + table, + database: application, + readOnly: false, + storePrefix: 'page/', + row: null, + }, + }) + + expect(wrapper.findComponent(GridViewRowDragging).props('offset')).toBe( + 200 + ) + await wrapper.setProps({ view: { ...view, group_by_layout: 'section' } }) + expect(wrapper.findComponent(GridViewRowDragging).props('offset')).toBe(0) + } finally { + await testApp.afterEach() + } + }) + // The post-drag click of a multi-select lands on the rows container; that must not // cancel the selection. Regression for group-by, whose rows use their own containers. const runCancel = (targetClass) => { diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js new file mode 100644 index 0000000000..7efc824688 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByColumns.spec.js @@ -0,0 +1,112 @@ +import { TestApp } from '@baserow/test/helpers/testApp' +import GridViewGroupByColumns from '@baserow/modules/database/components/view/grid/GridViewGroupByColumns' + +describe('GridViewGroupByColumns component', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Team', type: 'text', primary: true }, + { id: 2, name: 'Role', type: 'text', primary: false }, + ] + const groupBys = [ + { id: 10, field: 1, order: 'ASC', type: 'default', width: 120 }, + { id: 11, field: 2, order: 'ASC', type: 'default', width: 90 }, + ] + + beforeEach(() => { + testApp = new TestApp() + store = testApp.store + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + const seedGroups = async (treeNodes) => { + store.commit('page/view/grid/APPLY_GROUP_BY_STATE', { + activeGroupBys: groupBys, + groupBy: { + treeNodes, + pages: {}, + absoluteRows: {}, + revision: 0, + generation: 0, + aggregationsLoading: false, + aggregationsLoadingPaths: [], + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + sectionRows: {}, + rowLocations: {}, + sectionAccessOrder: [], + addRowHoverPathKey: null, + offsetsServerConfirmed: true, + }, + }) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + } + + const mountColumns = (props = {}) => + testApp.mount(GridViewGroupByColumns, { + props: { + allFieldsInTable: fields, + workspaceId: 1, + storePrefix: 'page/', + ...props, + }, + }) + + test('renders one cell per visible span, positioned per level, with value and count', async () => { + await seedGroups([ + { path: { field_1: 'A' }, depth: 0, row_count: 3 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 2 }, + { path: { field_1: 'A', field_2: 'Y' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns() + + const spans = wrapper.findAll('.grid-view__group-span') + expect(spans).toHaveLength(3) + expect(spans[0].attributes('style')).toContain('left: 0px') + expect(spans[0].attributes('style')).toContain('width: 120px') + expect(spans[0].attributes('style')).toContain('height: 99px') + expect(spans[1].attributes('style')).toContain('left: 120px') + expect(spans[1].attributes('style')).toContain('width: 90px') + expect(spans[2].attributes('style')).toContain('top: 66px') + expect(spans[0].find('.grid-view__group-value').text()).toBe('A') + expect(spans[0].find('.grid-view__group-count').text()).toBe('3') + expect(spans[1].find('.grid-view__group-value').text()).toBe('X') + expect(spans[2].find('.grid-view__group-count').text()).toBe('1') + expect( + wrapper.find('.grid-view__group-columns').attributes('style') + ).toContain('width: 210px') + }) + + test('renders the empty label for a group without a value', async () => { + await seedGroups([ + { path: { field_1: '' }, depth: 0, row_count: 1 }, + { path: { field_1: '', field_2: '' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns() + + const empties = wrapper.findAll('.grid-view__group-value-empty') + expect(empties).toHaveLength(2) + expect(empties[0].text()).toBe('gridViewGroupByBanner.emptyValue') + }) + + test('uses responsive widths without changing the configured group widths', async () => { + await seedGroups([ + { path: { field_1: 'A' }, depth: 0, row_count: 1 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 1 }, + ]) + const wrapper = await mountColumns({ groupByWidths: [80, 100] }) + + const spans = wrapper.findAll('.grid-view__group-span') + expect(spans[0].attributes('style')).toContain('width: 80px') + expect(spans[1].attributes('style')).toContain('left: 80px') + expect(spans[1].attributes('style')).toContain('width: 100px') + expect( + wrapper.find('.grid-view__group-columns').attributes('style') + ).toContain('width: 180px') + expect(groupBys.map(({ width }) => width)).toEqual([120, 90]) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js new file mode 100644 index 0000000000..a4befb1db2 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewGroupByRows.spec.js @@ -0,0 +1,107 @@ +import GridViewGroupByRows from '@baserow/modules/database/components/view/grid/GridViewGroupByRows' +import { pathKey } from '@baserow/modules/database/utils/gridGroupByRender' +import { TestApp } from '@baserow/test/helpers/testApp' + +describe('GridViewGroupByRows component', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Team', type: 'text', primary: true, text_default: '' }, + ] + const groupBys = [ + { id: 10, field: 1, order: 'ASC', type: 'default', width: 200 }, + ] + const makeRow = (id, team) => ({ + id, + field_1: team, + _: { + loading: false, + selected: false, + hover: false, + matchFilters: true, + matchSortings: true, + matchSearch: true, + fieldSearchMatches: [], + persistentId: id, + }, + }) + + beforeEach(() => { + testApp = new TestApp() + store = testApp.store + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + test('continues column group boundaries across row cells only in Columns', async () => { + const rowsA = [makeRow(1, 'A'), makeRow(2, 'A')] + const rowsB = [makeRow(3, 'B')] + store.commit('page/view/grid/APPLY_GROUP_BY_STATE', { + activeGroupBys: groupBys, + count: 3, + groupBy: { + treeNodes: [ + { path: { field_1: 'A' }, depth: 0, row_count: 2 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + ], + pages: {}, + absoluteRows: {}, + revision: 0, + generation: 0, + aggregationsLoading: false, + aggregationsLoadingPaths: [], + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + sectionRows: { + [pathKey({ field_1: 'A' }, fields)]: rowsA, + [pathKey({ field_1: 'B' }, fields)]: rowsB, + }, + rowLocations: {}, + sectionAccessOrder: [], + addRowHoverPathKey: null, + offsetsServerConfirmed: true, + }, + }) + store.commit('page/view/grid/SET_WINDOW_HEIGHT', 1000) + await store.dispatch('page/view/grid/setGroupByLayout', 'column') + + const wrapper = await testApp.mount(GridViewGroupByRows, { + props: { + renderedFields: fields, + visibleFields: fields, + allVisibleFields: fields, + allFieldsInTable: fields, + decorationsByPlace: {}, + groupColumnsWidth: 200, + view: { + id: 1, + table: {}, + sortings: [], + row_identifier_type: 'count', + }, + includeRowDetails: true, + readOnly: false, + workspaceId: 1, + storePrefix: 'page/', + }, + global: { + stubs: { GridViewRow: true, GridViewGroupByBanner: true }, + }, + }) + + expect(wrapper.findAll('.grid-view__group-by-rows-row')).toHaveLength(3) + expect( + wrapper.findAll('.grid-view__group-by-rows-row--group-end') + ).toHaveLength(2) + + await store.dispatch('page/view/grid/setGroupByLayout', 'section') + await wrapper.vm.$nextTick() + + expect( + wrapper.findAll('.grid-view__group-by-rows-row--group-end') + ).toHaveLength(0) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js b/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js new file mode 100644 index 0000000000..06fc742e51 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/grid/gridViewSection.spec.js @@ -0,0 +1,147 @@ +import { h, nextTick, ref } from 'vue' +import flushPromises from 'flush-promises' + +import { TestApp } from '@baserow/test/helpers/testApp' +import GridViewSection from '@baserow/modules/database/components/view/grid/GridViewSection' +import { fitGroupByWidths } from '@baserow/modules/database/utils/gridGroupByWidths' +import { GRID_VIEW_MIN_FIELD_WIDTH } from '@baserow/modules/database/constants' + +describe('GridViewSection group column resizing', () => { + let testApp + + beforeEach(() => { + testApp = new TestApp() + }) + + afterEach(async () => { + window.dispatchEvent(new MouseEvent('mouseup')) + await testApp.afterEach() + }) + + const mountSection = async () => { + const availableWidth = ref(250) + const groupBy = { + id: 10, + view: 1, + field: 1, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + } + const view = { id: 1, group_bys: [groupBy], group_by_layout: 'column' } + await testApp.store.dispatch('page/view/grid/updateActiveGroupBys', [ + groupBy, + ]) + testApp.mock.onPatch('/database/views/group_by/10/').reply(200, {}) + + const wrapper = await testApp.mount( + { + render() { + return h(GridViewSection, { + view, + database: { id: 1, workspace: { id: 1 } }, + table: { id: 1 }, + visibleFields: [], + allVisibleFields: [], + allFieldsInTable: [], + decorationsByPlace: {}, + includeGroupBy: true, + readOnly: false, + storePrefix: 'page/', + groupByWidths: fitGroupByWidths( + testApp.store.getters['page/view/grid/getActiveGroupBys'], + availableWidth.value, + GRID_VIEW_MIN_FIELD_WIDTH + ), + }) + }, + }, + { + global: { + mocks: { $hasPermission: () => true }, + stubs: { + GridViewHead: true, + GridViewGroupByColumns: true, + GridViewGroupByRows: true, + }, + }, + } + ) + return { wrapper, availableWidth } + } + + const moveMouse = async (type, clientX) => { + window.dispatchEvent(new MouseEvent(type, { clientX })) + await nextTick() + } + + test.each([ + { positions: [240] }, + { positions: [270] }, + { positions: [270, 240] }, + ])( + 'persists the final width after dragging through $positions', + async ({ positions }) => { + const { wrapper, availableWidth } = await mountSection() + const finalWidth = positions.at(-1) + await wrapper + .get('.grid-view__head-group-width-handle') + .trigger('mousedown', { clientX: 200 }) + + for (const position of positions) { + await moveMouse('mousemove', position) + } + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain(`left: ${Math.min(finalWidth, 250)}px`) + expect( + wrapper.find('.grid-view__head-group-width-handle.dragging').exists() + ).toBe(true) + expect(testApp.mock.history.patch).toHaveLength(0) + + await moveMouse('mouseup', finalWidth) + await flushPromises() + + expect(testApp.mock.history.patch).toHaveLength(1) + expect(JSON.parse(testApp.mock.history.patch[0].data)).toEqual({ + width: finalWidth, + }) + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + finalWidth <= 250 + ) + + availableWidth.value = 400 + await nextTick() + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain(`left: ${finalWidth}px`) + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + true + ) + } + ) + + test('finishes a drag returned to its original width without persisting', async () => { + const { wrapper, availableWidth } = await mountSection() + await wrapper + .get('.grid-view__head-group-width-handle') + .trigger('mousedown', { clientX: 200 }) + + await moveMouse('mousemove', 270) + await moveMouse('mousemove', 200) + await moveMouse('mouseup', 200) + await flushPromises() + + expect(testApp.mock.history.patch).toHaveLength(0) + expect( + wrapper.get('.grid-view__group-by-divider').attributes('style') + ).toContain('left: 200px') + + availableWidth.value = 150 + await nextTick() + expect(wrapper.find('.grid-view__head-group-width-handle').exists()).toBe( + false + ) + }) +}) diff --git a/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js b/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js new file mode 100644 index 0000000000..178b30c020 --- /dev/null +++ b/web-frontend/test/unit/database/components/view/viewGroupByContext.spec.js @@ -0,0 +1,113 @@ +import flushPromises from 'flush-promises' +import { vi } from 'vitest' + +import { TestApp } from '@baserow/test/helpers/testApp' +import ViewGroupByContext from '@baserow/modules/database/components/view/ViewGroupByContext' + +describe('ViewGroupByContext', () => { + let testApp = null + + const database = { id: 1, workspace: { id: 1 } } + const fields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text', primary: false }, + ] + const makeView = (groupByLayout) => ({ + id: 1, + ownership_type: 'collaborative', + group_bys: [ + { + id: 10, + field: 2, + order: 'ASC', + type: 'default', + width: 200, + _: { loading: false }, + }, + ], + group_by_layout: groupByLayout, + }) + + beforeEach(() => { + testApp = new TestApp() + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + const mountContext = async (view) => { + const wrapper = await testApp.mount(ViewGroupByContext, { + props: { database, view, fields, readOnly: false, disableGroupBy: false }, + }) + await wrapper.vm.show(document.body) + await wrapper.vm.$nextTick() + return wrapper + } + + const contextElement = () => document.body.querySelector('.context.group-bys') + const segmentLabels = () => + [...contextElement().querySelectorAll('.segment-control__button')].map( + (button) => button.textContent.trim() + ) + const buttonTexts = () => + [...contextElement().querySelectorAll('.group-bys__footer-actions button')] + .map((button) => button.textContent.trim()) + .filter(Boolean) + + test('section layout shows the layout switch and the collapse buttons', async () => { + await mountContext(makeView('section')) + + expect(segmentLabels()).toEqual([ + 'viewGroupByContext.layoutSection', + 'viewGroupByContext.layoutColumn', + ]) + expect(buttonTexts()).toEqual( + expect.arrayContaining([ + 'viewGroupByContext.collapseAllGroups', + 'viewGroupByContext.expandAllGroups', + ]) + ) + }) + + test('column layout hides the collapse buttons', async () => { + await mountContext(makeView('column')) + + expect(buttonTexts()).not.toEqual( + expect.arrayContaining(['viewGroupByContext.collapseAllGroups']) + ) + expect(buttonTexts()).not.toEqual( + expect.arrayContaining(['viewGroupByContext.expandAllGroups']) + ) + const active = contextElement().querySelector( + '.segment-control__button--active' + ) + expect(active.textContent.trim()).toBe('viewGroupByContext.layoutColumn') + }) + + test.each([ + ['Sections', 'column', 'section', 0], + ['Columns', 'section', 'column', 1], + ])( + 'choosing %s updates the view setting', + async (_label, currentLayout, selectedLayout, index) => { + const view = makeView(currentLayout) + await mountContext(view) + const dispatch = vi + .spyOn(testApp.store, 'dispatch') + .mockResolvedValue(undefined) + + const selected = [ + ...contextElement().querySelectorAll('.segment-control__button'), + ][index] + selected.click() + await flushPromises() + + expect(dispatch).toHaveBeenCalledWith('view/update', { + view, + values: { group_by_layout: selectedLayout }, + readOnly: false, + }) + } + ) +}) diff --git a/web-frontend/test/unit/database/store/view/grid.spec.js b/web-frontend/test/unit/database/store/view/grid.spec.js index 1077042ada..45f7d06dcc 100644 --- a/web-frontend/test/unit/database/store/view/grid.spec.js +++ b/web-frontend/test/unit/database/store/view/grid.spec.js @@ -10332,3 +10332,491 @@ describe('Grid view store', () => { expect(rowsOrderByParam).toBe('field_1') }) }) + +describe('Grid view store group-by layout mode', () => { + let testApp = null + let store = null + + const fields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + ] + const groupBys = [ + { id: 10, field: 2, order: 'ASC', type: 'default', width: 200 }, + ] + const view = { + id: 1, + filters: [], + filter_groups: [], + filter_type: 'AND', + sortings: [], + group_bys: groupBys, + } + const seed = (targetStore, extra = {}) => { + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: groupBys, + rowHeight: 33, + windowHeight: 100, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + }, + groupBy: { + ...gridStore.state().groupBy, + treeNodes: [{ path: { field_2: 'A' }, depth: 0, row_count: 2 }], + collapse: { mode: 'expand', paths: [] }, + }, + ...extra, + }) + targetStore.replaceState({ ...targetStore.state, grid: state }) + } + + beforeEach(() => { + testApp = new TestApp() + store = testApp.createStore({ modules: { grid: gridStore } }) + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + test('setGroupByLayout switches the layout builder and forces expand-all', async () => { + seed(store) + store.commit('grid/SET_GROUP_BY_COLLAPSE', { mode: 'collapse', paths: [] }) + expect( + store.getters['grid/getGroupByLayout'].items.map((item) => item.type) + ).toEqual(['header']) + + await store.dispatch('grid/setGroupByLayout', 'column') + + expect(store.getters['grid/isGroupByColumnLayout']).toBe(true) + expect( + store.getters['grid/getGroupByLayout'].items.map((item) => item.type) + ).toEqual(['groupSpan', 'rowSection', 'addRow']) + }) + + test('collapse actions are no-ops in column layout', async () => { + const fetchGroupByRowsByScrollTop = vi.fn().mockResolvedValue([]) + const groupByStore = testApp.createStore({ + modules: { + grid: { + ...gridStore, + actions: { ...gridStore.actions, fetchGroupByRowsByScrollTop }, + }, + }, + }) + seed(groupByStore, { groupByLayout: 'column' }) + + await groupByStore.dispatch('grid/toggleGroupCollapse', { + path: { field_2: 'A' }, + view, + fields, + }) + await groupByStore.dispatch('grid/setGroupByCollapseAll', { + view, + fields, + collapse: true, + }) + + expect(groupByStore.getters['grid/getGroupByCollapse']).toEqual({ + mode: 'expand', + paths: [], + }) + expect(fetchGroupByRowsByScrollTop).not.toHaveBeenCalled() + }) + + test.each(['column', 'section'])( + '%s layout selects loaded rows beyond an unloaded group page', + async (layout) => { + const nodes = Object.fromEntries( + [0, 80].flatMap((offset) => + Array.from({ length: 40 }, (_, index) => { + const position = offset + index + return [ + position, + { + path: { field_2: `Group ${position}` }, + depth: 0, + row_count: 1, + sibling_index: position, + row_offset: position, + }, + ] + }) + ) + ) + seed(store, { + groupByLayout: layout, + count: 120, + groupBy: { + ...gridStore.state().groupBy, + pages: { '': { parentPath: {}, totalSiblingCount: 120, nodes } }, + }, + }) + for (const rowId of [100, 101]) { + store.commit('grid/SET_GROUP_BY_SECTION_ROWS', { + sectionKey: groupPathKey(2, `Group ${rowId}`), + startPosition: 0, + rows: [ + { + id: rowId, + field_2: `Group ${rowId}`, + _: { selected: false, selectedFieldId: -1 }, + }, + ], + }) + } + + await store.dispatch('grid/multiSelectStart', { + rowId: 100, + fieldIndex: 0, + }) + await store.dispatch('grid/multiSelectHold', { + rowId: 101, + fieldIndex: 1, + }) + + const firstRowIndex = layout === 'column' ? 100 : 60 + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + expect( + store.getters['grid/getSelectedRows'].map((row) => row.id) + ).toEqual([100, 101]) + + await store.dispatch('grid/correctMultiSelect') + + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + expect( + store.getters['grid/getSelectedRows'].map((row) => row.id) + ).toEqual([100, 101]) + + await store.dispatch('grid/updateMultipleSelectIndexes', { + position: 'tail', + rowIndex: layout === 'column' ? 120 : 80, + fieldIndex: 1, + }) + expect(store.getters['grid/getMultiSelectRowIndexSorted']).toEqual([ + firstRowIndex, + firstRowIndex + 1, + ]) + } + ) + + test('column layout initially fetches descendants despite a saved collapse-all state', async () => { + const nestedFields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + { id: 3, name: 'Role', type: 'text' }, + ] + const nestedGroupBys = [ + { field: 2, order: 'ASC', type: 'default', width: 200 }, + { field: 3, order: 'ASC', type: 'default', width: 200 }, + ] + const nestedView = { ...view, group_bys: nestedGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: nestedGroupBys, + groupByLayout: 'column', + count: 2, + rowHeight: 33, + rowPadding: 0, + windowHeight: 330, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + 3: { hidden: false, order: 2 }, + }, + groupBy: { + ...gridStore.state().groupBy, + pages: {}, + treeNodes: [], + collapse: { mode: 'collapse', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const groupByRequests = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + groupByRequests.push(config.params) + return [ + 200, + { + pages: [ + { + parent: {}, + groups: [ + { + path: { field_2: 'A' }, + depth: 0, + row_count: 2, + children_count: 1, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + }, + { + parent: { field_2: 'A' }, + groups: [ + { + path: { field_2: 'A', field_3: 'Dev' }, + depth: 1, + row_count: 2, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + }, + ], + }, + ] + }) + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply(200, { + count: 2, + results: [ + { + id: 10, + order: '1.00', + field_1: 'Alice', + field_2: 'A', + field_3: 'Dev', + }, + { id: 11, order: '2.00', field_1: 'Ada', field_2: 'A', field_3: 'Dev' }, + ], + }) + + await store.dispatch('grid/fetchGroupByRowsByScrollTop', { + gridId: 1, + view: nestedView, + fields: nestedFields, + scrollTop: 0, + }) + + expect(groupByRequests).toHaveLength(1) + expect(groupByRequests[0].get('include_descendants')).toBe('true') + expect(groupByRequests[0].get('depth')).toBe(null) + expect(store.state.grid.groupBy.collapse).toEqual({ + mode: 'collapse', + paths: [], + }) + }) + + test('column viewport refines sparse group pages until rows are visible', async () => { + const sparseGroupBys = [ + { id: 10, field: 2, order: 'ASC', type: 'default', width: 200 }, + ] + const sparseView = { ...view, group_bys: sparseGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: sparseGroupBys, + groupByLayout: 'column', + count: 12000, + bufferRequestSize: 40, + rowHeight: 33, + rowPadding: 0, + windowHeight: 33, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + }, + groupBy: { + ...gridStore.state().groupBy, + pages: { + '': { + parentPath: {}, + totalSiblingCount: 120, + nodes: {}, + }, + }, + collapse: { mode: 'expand', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const groupPageOffsets = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + const [{ offset }] = JSON.parse(config.params.get('parents')) + groupPageOffsets.push(offset) + const isLastPage = offset === 80 + const rowCount = isLastPage ? 1 : 100 + const rowOffset = isLastPage ? 11960 : 7960 + return [ + 200, + { + pages: [ + { + parent: {}, + groups: Array.from({ length: 40 }, (_, index) => ({ + path: { field_2: `Group ${offset + index}` }, + depth: 0, + row_count: rowCount, + sibling_index: offset + index, + row_offset: rowOffset + index * rowCount, + })), + offset, + limit: 40, + group_count: 120, + }, + ], + }, + ] + }) + + const rowRequests = [] + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply((config) => { + rowRequests.push(config.params) + return [ + 200, + { + count: 12000, + results: [ + { + id: 8001, + order: '8001.00', + field_1: 'Visible row', + field_2: 'Group 40', + }, + ], + }, + ] + }) + + await store.dispatch('grid/fetchGroupByRowsByScrollTop', { + gridId: 1, + view: sparseView, + fields, + scrollTop: 8000 * 33, + }) + + // Loading the initially visible final page changes the estimated heights of the + // earlier sparse pages. The same viewport then falls in page 40 and must be refined + // again before row ranges can be resolved. + expect(groupPageOffsets).toEqual([80, 40]) + expect(rowRequests).toHaveLength(1) + expect(rowRequests[0].get('offset')).toBe('7960') + }) + + test('column layout refresh uses expanded paging but preserves section collapse state', async () => { + const nestedFields = [ + { id: 1, name: 'Name', type: 'text', primary: true }, + { id: 2, name: 'Team', type: 'text' }, + { id: 3, name: 'Role', type: 'text' }, + ] + const nestedGroupBys = [ + { field: 2, order: 'ASC', type: 'default', width: 200 }, + { field: 3, order: 'ASC', type: 'default', width: 200 }, + ] + const nestedView = { ...view, group_bys: nestedGroupBys } + const state = Object.assign(gridStore.state(), { + lastGridId: 1, + activeGroupBys: nestedGroupBys, + groupByLayout: 'column', + count: 2, + rowHeight: 33, + rowPadding: 0, + windowHeight: 330, + fieldOptions: { + 1: { hidden: false, order: 0 }, + 2: { hidden: false, order: 1 }, + 3: { hidden: false, order: 2 }, + }, + groupBy: { + ...gridStore.state().groupBy, + collapse: { mode: 'collapse', paths: [] }, + collapseInitialized: true, + }, + }) + store.replaceState({ ...store.state, grid: state }) + + const rootPage = { + parent: {}, + groups: [ + { + path: { field_2: 'A' }, + depth: 0, + row_count: 2, + children_count: 1, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + } + const childPage = { + parent: { field_2: 'A' }, + groups: [ + { + path: { field_2: 'A', field_3: 'Dev' }, + depth: 1, + row_count: 2, + sibling_index: 0, + row_offset: 0, + }, + ], + offset: 0, + limit: 40, + group_count: 1, + } + const groupByRequests = [] + testApp.mockServer.mock + .onGet('/database/views/grid/1/group-by-data/') + .reply((config) => { + groupByRequests.push(config.params) + const pages = + config.params.get('include_descendants') === 'true' + ? [rootPage, childPage] + : config.params.get('depth') !== null + ? [childPage] + : [rootPage] + return [200, { pages }] + }) + testApp.mockServer.mock.onGet('/database/views/grid/1/').reply(200, { + count: 2, + results: [ + { + id: 10, + order: '1.00', + field_1: 'Alice', + field_2: 'A', + field_3: 'Dev', + }, + { id: 11, order: '2.00', field_1: 'Ada', field_2: 'A', field_3: 'Dev' }, + ], + }) + + await store.dispatch('grid/refreshActiveGroupBys', { + view: nestedView, + fields: nestedFields, + scrollTop: 0, + preserveScroll: true, + }) + + expect(groupByRequests).toHaveLength(1) + expect(groupByRequests[0].get('include_descendants')).toBe('true') + expect(groupByRequests[0].get('depth')).toBe(null) + expect(store.state.grid.groupBy.collapse).toEqual({ + mode: 'collapse', + paths: [], + }) + }) +}) diff --git a/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js b/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js index 38fd321a6e..1c609920fa 100644 --- a/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js +++ b/web-frontend/test/unit/database/utils/copyViewConfiguration.spec.js @@ -1,3 +1,5 @@ +import { readFileSync } from 'fs' +import { resolve } from 'path' import { TestApp } from '@baserow/test/helpers/testApp' import { expect } from 'vitest' @@ -13,6 +15,26 @@ import { ViewSettingsCopyOptionType, } from '@baserow/modules/database/copyViewConfigurationOptionTypes' +// Read rather than imported: the i18n loader turns an imported locale file +// into compiled message ASTs, which the copy below can't be read off of. +const en = JSON.parse( + readFileSync( + resolve(process.cwd(), 'modules/database/locales/en.json'), + 'utf8' + ) +) + +const translate = (key, params = {}) => { + const translation = key + .split('.') + .reduce((value, segment) => value[segment], en) + return Object.entries(params).reduce( + (value, [name, replacement]) => + value.replace(`{${name}}`, String(replacement)), + translation + ) +} + describe('getEnabledCopyOptionKeys', () => { let testApp = null @@ -143,6 +165,19 @@ describe('getEnabledCopyOptionKeys', () => { ) ).not.toContain('field_widths') }) + + test('grid view settings label includes the group layout setting', () => { + const viewSettingsOption = new ViewSettingsCopyOptionType({ + app: { + $registry: testApp.getRegistry(), + $i18n: { t: translate }, + }, + }) + + expect(viewSettingsOption.getName(view('grid'))).toBe( + 'Settings (row height, frozen columns, row identifier, group layout)' + ) + }) }) describe('copyViewConfiguration', () => { diff --git a/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js b/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js index 829a7ab495..96a416e85c 100644 --- a/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js +++ b/web-frontend/test/unit/database/utils/gridGroupByRender.spec.js @@ -3,6 +3,8 @@ import { ROW_HEIGHT, ADD_ROW_HEIGHT, GROUP_GAP, + GROUP_BY_LAYOUT_SECTION, + GROUP_BY_LAYOUT_COLUMN, buildLayout, pathKey, renderViewport, @@ -981,6 +983,505 @@ describe('gridGroupByRender', () => { const renderedHeader = items.find((item) => item.type === 'header') expect(renderedHeader.display).toEqual(display) }) + + describe('column layout', () => { + const fields = [textField(1), textField(2)] + const nodes = [ + { path: { field_1: 'A' }, depth: 0, row_count: 3 }, + { path: { field_1: 'A', field_2: 'X' }, depth: 1, row_count: 2 }, + { path: { field_1: 'A', field_2: 'Y' }, depth: 1, row_count: 1 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + { path: { field_1: 'B', field_2: 'X' }, depth: 1, row_count: 1 }, + ] + const column = { mode: 'expand', paths: [] } + + test('emits spans and gap-free row sections, no headers or per-group add rows', () => { + const layout = buildLayout({ + nodes, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + expect(layout.items.map((i) => i.type)).toEqual([ + 'groupSpan', + 'groupSpan', + 'rowSection', + 'groupSpan', + 'rowSection', + 'groupSpan', + 'groupSpan', + 'rowSection', + 'addRow', + ]) + const spans = layout.items.filter((i) => i.type === 'groupSpan') + expect(spans.map((s) => [s.depth, s.y, s.height])).toEqual([ + [0, 0, 3 * ROW_HEIGHT], + [1, 0, 2 * ROW_HEIGHT], + [1, 2 * ROW_HEIGHT, ROW_HEIGHT], + [0, 3 * ROW_HEIGHT, ROW_HEIGHT], + [1, 3 * ROW_HEIGHT, ROW_HEIGHT], + ]) + const addRow = layout.items.at(-1) + expect(addRow).toMatchObject({ + type: 'addRow', + path: {}, + y: 4 * ROW_HEIGHT, + height: ADD_ROW_HEIGHT, + }) + expect(layout.totalHeight).toBe(4 * ROW_HEIGHT + ADD_ROW_HEIGHT) + expect(layout.geometry.headerHeight).toBe(0) + }) + + test('ignores collapse state', () => { + const layout = buildLayout({ + nodes, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + collapse: { mode: 'collapse', paths: [] }, + }) + expect(layout.items.filter((i) => i.type === 'rowSection')).toHaveLength( + 3 + ) + }) + + test('sizes unloaded sibling ranges by one row per group', () => { + const layout = buildLayout({ + pages: { '': { parentPath: {}, totalSiblingCount: 50, nodes: {} } }, + collapse: column, + fields: [textField(1)], + layout: GROUP_BY_LAYOUT_COLUMN, + pageSize: 40, + }) + const placeholders = layout.items.filter( + (i) => i.type === 'groupPlaceholder' + ) + expect(placeholders.map((p) => p.height)).toEqual([ + 40 * ROW_HEIGHT, + 10 * ROW_HEIGHT, + ]) + expect( + visibleGroupDepthPageInViewport( + layout, + { scrollTop: 41 * ROW_HEIGHT, clientHeight: ROW_HEIGHT }, + 40 + ) + ).toEqual({ depth: 0, offset: 40, limit: 40 }) + }) + + test('uses the total row count to size sparse root group pages', () => { + const layout = buildLayout({ + pages: { + '': { parentPath: {}, totalSiblingCount: 1500, nodes: {} }, + }, + collapse: column, + fields: [textField(1)], + rootRowCount: 3000, + layout: GROUP_BY_LAYOUT_COLUMN, + pageSize: 40, + }) + const placeholders = layout.items.filter( + (item) => item.type === 'groupPlaceholder' + ) + + expect(placeholders).toHaveLength(Math.ceil(1500 / 40)) + expect( + placeholders.reduce((height, item) => height + item.height, 0) + ).toBe(3000 * ROW_HEIGHT) + expect(layout.totalHeight).toBe(3000 * ROW_HEIGHT + ADD_ROW_HEIGHT) + }) + + test.each([0, 1])( + 'anchors sparse loaded groups between uneven gaps at depth %s', + (depth) => { + const parentPath = depth === 0 ? {} : { field_1: 'Parent' } + const parentRowOffset = depth === 0 ? 0 : 20 + const groupField = `field_${depth + 1}` + const groupFields = fields.slice(0, depth + 1) + const makePage = (offset) => + Object.fromEntries( + Array.from({ length: 40 }, (_, index) => { + const position = offset + index + return [ + position, + { + path: { ...parentPath, [groupField]: `Group ${position}` }, + depth, + row_count: position < 40 ? 1 : 100, + sibling_index: position, + row_offset: + parentRowOffset + + (position < 40 ? position : 40 + (position - 40) * 100), + }, + ] + }) + ) + const pages = { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'Earlier' }, + depth: 0, + row_count: 20, + children_count: 1, + row_offset: 0, + }, + 1: { + path: parentPath, + depth: 0, + row_count: 8080, + children_count: 160, + row_offset: parentRowOffset, + }, + }, + }, + [pathKey(parentPath, groupFields)]: { + parentPath, + totalSiblingCount: 160, + nodes: makePage(0), + }, + } + const build = () => + buildLayout({ + pages, + fields: groupFields, + collapse: column, + layout: GROUP_BY_LAYOUT_COLUMN, + rootRowCount: parentRowOffset + 8080, + }) + const viewport = { + scrollTop: (parentRowOffset + 3980) * ROW_HEIGHT, + clientHeight: 60 * ROW_HEIGHT, + } + expect(visibleGroupPagesInViewport(build(), viewport)).toEqual([ + { parentPath, offset: 80, limit: 40 }, + ]) + + // Hydrating the estimated page must anchor it at its actual row offset. + // Otherwise it covers this viewport incorrectly and refinement stops early. + Object.assign( + pages[pathKey(parentPath, groupFields)].nodes, + makePage(80) + ) + const layout = build() + const firstLoadedSection = layout.items.find( + (item) => + item.type === 'rowSection' && item.path[groupField] === 'Group 80' + ) + expect(firstLoadedSection.y).toBe((parentRowOffset + 4040) * ROW_HEIGHT) + expect(visibleGroupPagesInViewport(layout, viewport)).toEqual([ + { parentPath, offset: 40, limit: 40 }, + ]) + expect(layout.totalHeight).toBe( + (parentRowOffset + 8080) * ROW_HEIGHT + ADD_ROW_HEIGHT + ) + + Object.assign( + pages[pathKey(parentPath, groupFields)].nodes, + makePage(40) + ) + const hydratedLayout = build() + const targetViewport = { + scrollTop: (parentRowOffset + 4000) * ROW_HEIGHT, + clientHeight: ROW_HEIGHT, + } + expect(visibleGroupPagesInViewport(hydratedLayout, viewport)).toEqual( + [] + ) + const [section] = visibleSectionsInViewport( + hydratedLayout, + targetViewport, + groupFields + ) + expect(section.absoluteRowOffset + section.startPosition).toBe( + parentRowOffset + 4000 + ) + } + ) + + test('reserves a loaded parent row span while its child page is sparse', () => { + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 100, + children_count: 2, + sibling_index: 0, + row_offset: 0, + }, + 1: { + path: { field_1: 'B' }, + depth: 0, + row_count: 1, + children_count: 1, + sibling_index: 1, + row_offset: 100, + }, + }, + }, + }, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const [a, b] = layout.items.filter( + (item) => item.type === 'groupSpan' && item.depth === 0 + ) + + expect(b.y).toBe(a.y + a.height) + expect(layout.totalHeight).toBe(101 * ROW_HEIGHT + ADD_ROW_HEIGHT) + expect( + visibleGroupPagesInViewport( + layout, + { scrollTop: 50 * ROW_HEIGHT, clientHeight: ROW_HEIGHT }, + 40 + ) + ).toEqual([{ parentPath: { field_1: 'A' }, offset: 0, limit: 40 }]) + }) + + test('distributes missing parent rows across multiple child placeholder pages', () => { + const parentPath = { field_1: 'A' } + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 1, + nodes: { + 0: { + path: parentPath, + depth: 0, + row_count: 100, + children_count: 50, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + [pathKey(parentPath, fields)]: { + parentPath, + totalSiblingCount: 50, + nodes: { + 0: { + path: { ...parentPath, field_2: 'Loaded' }, + depth: 1, + row_count: 10, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + }, + collapse: column, + fields, + pageSize: 40, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const placeholders = layout.items.filter( + (item) => item.type === 'groupPlaceholder' + ) + + expect(placeholders).toHaveLength(2) + expect( + placeholders.reduce((height, item) => height + item.height, 0) + ).toBe(90 * ROW_HEIGHT) + expect(layout.totalHeight).toBe(100 * ROW_HEIGHT + ADD_ROW_HEIGHT) + }) + + test('uses the absolute row offset for identifiers after a sparse branch', () => { + const bPath = { field_1: 'B' } + const bLeafPath = { ...bPath, field_2: 'Only' } + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 2, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 100, + children_count: 2, + sibling_index: 0, + row_offset: 0, + }, + 1: { + path: bPath, + depth: 0, + row_count: 1, + children_count: 1, + sibling_index: 1, + row_offset: 100, + }, + }, + }, + [pathKey(bPath, fields)]: { + parentPath: bPath, + totalSiblingCount: 1, + nodes: { + 0: { + path: bLeafPath, + depth: 1, + row_count: 1, + sibling_index: 0, + row_offset: 100, + }, + }, + }, + }, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const bSection = layout.items.find( + (item) => item.type === 'rowSection' && item.path.field_1 === 'B' + ) + const [renderedRow] = renderViewport({ + layout, + sectionRows: buildSectionRows( + [{ path: bLeafPath, rows: [{ id: 101 }] }], + fields + ), + viewport: { scrollTop: bSection.y, clientHeight: ROW_HEIGHT }, + fields, + }).filter((item) => item.type === 'row') + + expect(renderedRow.globalRowOffset).toBe(100) + }) + + test('marks only the last row of each group as a column boundary', () => { + const f = [textField(1)] + const groupedNodes = [ + { path: { field_1: 'A' }, depth: 0, row_count: 2 }, + { path: { field_1: 'B' }, depth: 0, row_count: 1 }, + ] + const sectionRows = buildSectionRows( + [ + { path: { field_1: 'A' }, rows: [{ id: 1 }, { id: 2 }] }, + { path: { field_1: 'B' }, rows: [{ id: 3 }] }, + ], + f + ) + const renderRows = (layout) => + renderViewport({ + layout, + sectionRows, + viewport: { scrollTop: 0, clientHeight: 1000 }, + fields: f, + }).filter((item) => item.type === 'row') + + const columnRows = renderRows( + buildLayout({ + nodes: groupedNodes, + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + ) + expect(columnRows.map((item) => item.groupEnd)).toEqual([ + false, + true, + true, + ]) + + const sectionLayoutRows = renderRows( + buildLayout({ + nodes: groupedNodes, + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_SECTION, + }) + ) + expect(sectionLayoutRows.every((item) => item.groupEnd === false)).toBe( + true + ) + }) + + test('renderViewport clips spans and renders unloaded ranges as one placeholder block', () => { + const layout = buildLayout({ + pages: { + '': { + parentPath: {}, + totalSiblingCount: 3, + nodes: { + 0: { + path: { field_1: 'A' }, + depth: 0, + row_count: 4, + sibling_index: 0, + row_offset: 0, + }, + }, + }, + }, + collapse: column, + fields: [textField(1)], + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const items = renderViewport({ + layout, + sectionRows: new Map(), + fields: [textField(1)], + viewport: { scrollTop: ROW_HEIGHT, clientHeight: 5 * ROW_HEIGHT }, + }) + expect(items[0]).toMatchObject({ + type: 'groupSpan', + y: 0, + height: 4 * ROW_HEIGHT, + }) + expect(items.filter((i) => i.type === 'placeholder')).toHaveLength(3) + expect( + items.find((i) => i.type === 'groupRangePlaceholder') + ).toMatchObject({ y: 4 * ROW_HEIGHT, height: 2 * ROW_HEIGHT }) + expect(items.some((i) => i.type === 'groupSkeleton')).toBe(false) + }) + + test('resolveGroupByRowMoveTarget ignores spans and resolves the end-of-group slot', () => { + const f = [textField(1)] + const rows = [{ id: 1 }, { id: 2 }] + const layout = buildLayout({ + nodes: [{ path: { field_1: 'A' }, depth: 0, row_count: 2 }], + collapse: column, + fields: f, + layout: GROUP_BY_LAYOUT_COLUMN, + }) + const sectionRows = buildSectionRows( + [{ path: { field_1: 'A' }, rows }], + f + ) + const common = { + layout, + sectionRows, + fields: f, + sourcePath: { field_1: 'A' }, + allowCrossGroup: false, + } + expect( + resolveGroupByRowMoveTarget({ ...common, contentY: 5 }) + ).toMatchObject({ before: rows[0], position: 0 }) + expect( + resolveGroupByRowMoveTarget({ ...common, contentY: 2 * ROW_HEIGHT - 3 }) + ).toMatchObject({ before: null, position: 2 }) + }) + + test('section layout output is unchanged', () => { + const section = buildLayout({ nodes, collapse: column, fields }) + const explicit = buildLayout({ + nodes, + collapse: column, + fields, + layout: GROUP_BY_LAYOUT_SECTION, + }) + expect(explicit.items).toEqual(section.items) + expect(section.items.map((i) => i.type)).toContain('header') + expect(section.items.some((i) => i.type === 'groupSpan')).toBe(false) + }) + }) }) describe('buildLayout cycle safety', () => { diff --git a/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js b/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js new file mode 100644 index 0000000000..fc53866623 --- /dev/null +++ b/web-frontend/test/unit/database/utils/gridGroupByWidths.spec.js @@ -0,0 +1,29 @@ +import { fitGroupByWidths } from '@baserow/modules/database/utils/gridGroupByWidths' + +describe('fitGroupByWidths', () => { + test('keeps configured widths when they fit or the grid is not measured', () => { + const groupBys = [{ width: 90 }, { width: 180 }, { width: 270 }] + + expect(fitGroupByWidths(groupBys, 600, 78)).toEqual([90, 180, 270]) + expect(fitGroupByWidths(groupBys, null, 78)).toEqual([90, 180, 270]) + }) + + test('scales only space above the minimum and does not mutate configured widths', () => { + const groupBys = [ + { width: 78 }, + { width: 120 }, + { width: 200 }, + { width: 260 }, + { width: 342 }, + ] + const configuredWidths = groupBys.map(({ width }) => width) + const renderedWidths = fitGroupByWidths(groupBys, 652, 78) + + expect(renderedWidths[0]).toBe(78) + expect(renderedWidths.every((width) => width >= 78)).toBe(true) + expect( + renderedWidths.reduce((total, width) => total + width, 0) + ).toBeCloseTo(652) + expect(groupBys.map(({ width }) => width)).toEqual(configuredWidths) + }) +})