diff --git a/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py new file mode 100644 index 0000000000..e704a22cf1 --- /dev/null +++ b/backend/src/baserow/contrib/integrations/ai/ai_provider_feature_types.py @@ -0,0 +1,28 @@ +from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT +from baserow.core.ai_provider.registries import AIProviderModelFeatureType +from baserow.core.ai_provider.resolution import ScopedAIProviderState +from baserow.core.generative_ai.registries import generative_ai_model_type_registry +from baserow.core.models import Workspace + + +class AIAgentAIProviderModelFeatureType(AIProviderModelFeatureType): + type = AI_PROVIDER_FEATURE_AI_AGENT + + def get_workspace_availability( + self, + workspace: Workspace | None, + state: ScopedAIProviderState | None = None, + ) -> dict[str, bool | dict[str, list[str]]]: + """ + Return the providers and models available to AI Agent consumers. + + :param workspace: The workspace to resolve, or None for instance scope. + :param state: Optional provider state already loaded for the same scope. + :returns: Whether any eligible models exist and their identifiers grouped + by provider type. + """ + + models = generative_ai_model_type_registry.get_enabled_models_per_type( + workspace, feature_type=self.type, state=state + ) + return {"is_enabled": bool(models), "models": models} diff --git a/backend/src/baserow/contrib/integrations/ai/integration_types.py b/backend/src/baserow/contrib/integrations/ai/integration_types.py index 14d611e512..8307ab3611 100644 --- a/backend/src/baserow/contrib/integrations/ai/integration_types.py +++ b/backend/src/baserow/contrib/integrations/ai/integration_types.py @@ -37,8 +37,10 @@ class SerializedDict(IntegrationDict): required=False, default=dict, help_text="Per-provider AI settings overrides. If a provider key is not " - "present, workspace settings are inherited. If present, these values " - "override workspace settings. Structure: " + "present, workspace settings are inherited. A complete connection uses " + "its own credentials and explicit model list; omitting models inherits " + "available models, while an empty list disables them. An incomplete " + "connection can only restrict inherited model availability. Structure: " '{"openai": {"api_key": "...", "models": [...], "organization": ""}, ...}', ), } @@ -53,12 +55,14 @@ def prepare_values( ) -> Dict[str, Any]: """Validate explicit per-integration provider settings before saving. - Database-only providers are valid here because these overrides are passed - directly to the runtime instead of being stored in legacy workspace settings. + Database-only providers are valid here because complete overrides are passed + atomically to the runtime instead of being stored in legacy workspace settings. :param values: The integration values supplied by the caller. :param user: The user creating or updating the integration. - :return: The normalized values prepared by the base integration type. + :returns: The normalized values prepared by the base integration type. + :raises RequestBodyValidationException: If provider settings fail their + registered serializer's validation. """ if "ai_settings" not in values: @@ -76,6 +80,24 @@ def prepare_values( return super().prepare_values(values, user) + def get_integration_provider_settings( + self, integration: AIIntegration, provider_type: str + ) -> dict[str, Any] | None: + """ + Return the integration-level settings override for a provider. + + :param integration: The AI integration to read the override from. + :param provider_type: The generative AI provider type key. + :returns: The stored override, including an explicit empty dictionary, or + None when no dictionary is stored for this provider. This does not + validate whether the override defines a complete connection. + """ + + provider_settings = integration.ai_settings.get(provider_type) + if isinstance(provider_settings, dict): + return provider_settings + return None + def get_provider_settings( self, integration: AIIntegration, provider_type: str ) -> Dict[str, Any]: @@ -87,15 +109,15 @@ def get_provider_settings( :param integration: The AI integration whose provider settings are requested. :param provider_type: The generative AI provider type. - :return: Explicit or legacy provider settings, or an empty dictionary when + :returns: Explicit or legacy provider settings, or an empty dictionary when database-backed workspace inheritance should be used. """ - # Check if provider has overrides in integration settings - if provider_type in integration.ai_settings: - provider_settings = integration.ai_settings[provider_type] - if isinstance(provider_settings, dict): - return provider_settings + provider_settings = self.get_integration_provider_settings( + integration, provider_type + ) + if provider_settings is not None: + return provider_settings if feature_flag_is_enabled(FF_AI_PROVIDERS): return {} diff --git a/backend/src/baserow/contrib/integrations/ai/service_types.py b/backend/src/baserow/contrib/integrations/ai/service_types.py index d81920eed5..8be80c9c8b 100644 --- a/backend/src/baserow/contrib/integrations/ai/service_types.py +++ b/backend/src/baserow/contrib/integrations/ai/service_types.py @@ -7,6 +7,10 @@ from baserow.contrib.integrations.ai.integration_types import AIIntegrationType from baserow.contrib.integrations.ai.models import AIAgentService, AIOutputType +from baserow.core.ai_provider.constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, + AI_PROVIDER_TYPES, +) from baserow.core.feature_flags import FF_AI_PROVIDERS, feature_flag_is_enabled from baserow.core.formula.serializers import FormulaSerializerField from baserow.core.formula.validator import ensure_string @@ -14,9 +18,13 @@ GenerativeAIPromptError, GenerativeAITypeDoesNotExist, ) -from baserow.core.generative_ai.registries import generative_ai_model_type_registry +from baserow.core.generative_ai.registries import ( + GenerativeAIModelType, + generative_ai_model_type_registry, +) from baserow.core.integrations.exceptions import IntegrationDoesNotExist from baserow.core.integrations.handler import IntegrationHandler +from baserow.core.models import Workspace from baserow.core.services.dispatch_context import DispatchContext from baserow.core.services.exceptions import ( ServiceImproperlyConfiguredDispatchException, @@ -132,20 +140,124 @@ def import_serialized( parent, serialized_values, id_mapping, *args, **kwargs ) + @staticmethod + def _resolve_available_models_and_settings_override( + ai_model_type: GenerativeAIModelType, + workspace: Workspace | None, + integration_settings: dict[str, Any] | None, + providers_enabled: bool, + ) -> tuple[list[str], dict[str, Any] | None]: + """Resolve one integration override without mixing configuration scopes. + + A self-contained override owns its connection and any explicit model + list. Built-in overrides without a model list inherit the active + allowlist. An incomplete connection can only narrow inherited models; + its other fields are ignored so it never borrows credentials. + + :param ai_model_type: The registered provider resolving the model list. + :param workspace: The owning workspace, or None if unavailable. + :param integration_settings: The explicit provider override, or None when + the integration inherits all settings. + :param providers_enabled: Whether to enforce AI Agent eligibility through + database-backed providers rather than the legacy model allowlist. + :returns: The selectable model identifiers and the complete connection + override to pass to the provider, or None to inherit its connection. + """ + + atomic_settings = ( + ai_model_type.get_atomic_settings_override(integration_settings) + if integration_settings is not None + else None + ) + if atomic_settings is not None and ( + ai_model_type.type not in AI_PROVIDER_TYPES + or "models" in integration_settings + ): + if providers_enabled: + available_models = ai_model_type.get_enabled_models_for_feature( + AI_PROVIDER_FEATURE_AI_AGENT, + workspace=workspace, + settings_override=atomic_settings, + ) + else: + available_models = ai_model_type.call_get_enabled_models( + workspace=workspace, + settings_override=atomic_settings, + ) + return available_models, atomic_settings + + if providers_enabled: + available_models = ai_model_type.get_enabled_models_for_feature( + AI_PROVIDER_FEATURE_AI_AGENT, + workspace=workspace, + ) + else: + available_models = ai_model_type.call_get_enabled_models( + workspace=workspace, + ) + + if atomic_settings is not None: + # Legacy callers may override only their connection. Preserve the + # omitted model list's inheritance without inheriting credentials or + # optional connection settings. An explicit empty list stays empty. + return available_models, {**atomic_settings, "models": available_models} + + if integration_settings is not None and "models" in integration_settings: + model_limit = integration_settings["models"] + if not isinstance(model_limit, list): + return [], None + inherited_models = set(available_models) + available_models = [ + model for model in model_limit if model in inherited_models + ] + + return available_models, None + def prepare_values( self, values: Dict[str, Any], user: AbstractUser, instance: Optional[AIAgentService] = None, ) -> Dict[str, Any]: + """ + Validate the effective provider selection before creating or updating. + + With database providers enabled, an unchanged selection is retained even + when its model becomes unavailable. Dispatch still checks availability. + Legacy selections are validated on every update. An unavailable integration + skips the model check. The base type preserves trashed references for undo/redo + but rejects missing IDs; dispatch blocks trashed integrations until restored. + + :param values: The service values supplied for creation or a partial update. + :param user: The user creating or updating the service. + :param instance: The current service when updating, or None when creating. + :returns: The prepared values, with any integration ID resolved by the + base service type. + :raises DRFValidationError: If the selection being validated names an + unknown provider, an unavailable model, or an invalid integration. + """ + ai_type = values.get("ai_generative_ai_type") or ( instance.ai_generative_ai_type if instance else None ) ai_model = values.get("ai_generative_ai_model") or ( instance.ai_generative_ai_model if instance else None ) + integration_id = values.get("integration_id") or ( + instance.integration_id if instance else None + ) + selection_changed = instance is None or ( + ai_type != instance.ai_generative_ai_type + or ai_model != instance.ai_generative_ai_model + or integration_id != instance.integration_id + ) + providers_enabled = feature_flag_is_enabled(FF_AI_PROVIDERS) - if ai_type: + # The relaxed validation gate belongs to the database-backed provider + # feature only. The legacy path has always validated the stored selection + # against the environment/workspace allowlist, including on unrelated + # updates. + if ai_type and (selection_changed or not providers_enabled): try: ai_model_type = generative_ai_model_type_registry.get(ai_type) except GenerativeAITypeDoesNotExist as e: @@ -153,38 +265,37 @@ def prepare_values( {"ai_generative_ai_type": f"AI type '{ai_type}' does not exist."} ) from e - # Get the integration to check available models - integration_id = values.get("integration_id") or ( - instance.integration_id if instance else None - ) if integration_id and ai_model: try: integration = ( IntegrationHandler().get_integration(integration_id).specific ) except IntegrationDoesNotExist: - # The integration has been trashed (e.g. a concurrent edit), so its - # available models can't be resolved. Skip the best-effort model - # validation rather than raising a 500; the trashed integration is - # surfaced as a misconfiguration separately (at dispatch). - integration = None - - if integration is not None: - integration_type = AIIntegrationType() - provider_settings = integration_type.get_provider_settings( + # Preserve trashed references for undo/redo. The base type + # rejects missing IDs; dispatch blocks trashed integrations. + return super().prepare_values(values, user, instance) + + integration_type = AIIntegrationType() + integration_settings = ( + integration_type.get_integration_provider_settings( integration, ai_type ) - available_models = ai_model_type.call_get_enabled_models( - workspace=integration.application.workspace, - settings_override=provider_settings or None, + ) + available_models, _ = ( + self._resolve_available_models_and_settings_override( + ai_model_type, + integration.application.workspace, + integration_settings, + providers_enabled, + ) + ) + if ai_model not in available_models: + raise DRFValidationError( + { + "ai_generative_ai_model": f"Model '{ai_model}' is not " + f"available for provider '{ai_type}'." + } ) - - if available_models and ai_model not in available_models: - raise DRFValidationError( - { - "ai_generative_ai_model": f"Model '{ai_model}' is not available for provider '{ai_type}'." - } - ) return super().prepare_values(values, user, instance) @@ -204,6 +315,23 @@ def dispatch_data( resolved_values: Dict[str, Any], dispatch_context: DispatchContext, ) -> Dict[str, Any]: + """ + Resolve the current provider configuration and execute the AI prompt. + + Published applications recover their owning workspace from the dispatch + context. Model eligibility is rechecked so changes made after saving the + service take effect before a provider receives the prompt. + + :param service: The AI Agent service to execute. + :param resolved_values: Formula results, including the resolved AI prompt. + :param dispatch_context: The current Builder or Automation execution context. + :returns: The provider response under the result key. + :raises ServiceImproperlyConfiguredDispatchException: If required input, + workspace context, provider registration, or model availability is + missing, or choice output has no non-empty options. + :raises UnexpectedDispatchException: If the provider raises a prompt error. + """ + if not service.ai_generative_ai_type: raise ServiceImproperlyConfiguredDispatchException( "The AI provider type is missing." @@ -233,9 +361,14 @@ def dispatch_data( "At least one non-empty choice is required when output type is 'choice'." ) - ai_model_type = generative_ai_model_type_registry.get( - service.ai_generative_ai_type - ) + try: + ai_model_type = generative_ai_model_type_registry.get( + service.ai_generative_ai_type + ) + except GenerativeAITypeDoesNotExist as exc: + raise ServiceImproperlyConfiguredDispatchException( + f"AI provider type '{service.ai_generative_ai_type}' is unavailable." + ) from exc integration = service.integration.specific integration_type = AIIntegrationType() workspace = integration.application.workspace @@ -251,28 +384,27 @@ def dispatch_data( if workflow is not None: workspace = workflow.get_original().automation.workspace - # This returns explicit integration settings, or the legacy workspace fallback - # while database-backed providers are disabled. - provider_settings = integration_type.get_provider_settings( + providers_enabled = feature_flag_is_enabled(FF_AI_PROVIDERS) + integration_settings = integration_type.get_integration_provider_settings( integration, service.ai_generative_ai_type ) - settings_override = provider_settings or None - - providers_enabled = feature_flag_is_enabled(FF_AI_PROVIDERS) - if providers_enabled: - if settings_override is None and workspace is None: - raise ServiceImproperlyConfiguredDispatchException( - "The workspace context for the AI integration is missing." - ) - available_models = ai_model_type.call_get_enabled_models( - workspace=workspace, - settings_override=settings_override, + available_models, settings_override = ( + self._resolve_available_models_and_settings_override( + ai_model_type, + workspace, + integration_settings, + providers_enabled, + ) + ) + if providers_enabled and settings_override is None and workspace is None: + raise ServiceImproperlyConfiguredDispatchException( + "The workspace context for the AI integration is missing." + ) + if service.ai_generative_ai_model not in available_models: + raise ServiceImproperlyConfiguredDispatchException( + f"Model '{service.ai_generative_ai_model}' is not available " + f"for provider '{service.ai_generative_ai_type}'." ) - if service.ai_generative_ai_model not in available_models: - raise ServiceImproperlyConfiguredDispatchException( - f"The AI model '{service.ai_generative_ai_model}' is not " - f"available for provider '{service.ai_generative_ai_type}'." - ) kwargs = {} if service.ai_temperature is not None: diff --git a/backend/src/baserow/contrib/integrations/apps.py b/backend/src/baserow/contrib/integrations/apps.py index 68d008edda..85cb330db1 100644 --- a/backend/src/baserow/contrib/integrations/apps.py +++ b/backend/src/baserow/contrib/integrations/apps.py @@ -23,6 +23,17 @@ def ready(self): integration_type_registry.register(AIIntegrationType()) integration_type_registry.register(SlackBotIntegrationType()) + from baserow.contrib.integrations.ai.ai_provider_feature_types import ( + AIAgentAIProviderModelFeatureType, + ) + from baserow.core.ai_provider.registries import ( + ai_provider_model_feature_type_registry, + ) + + ai_provider_model_feature_type_registry.register( + AIAgentAIProviderModelFeatureType() + ) + from baserow.contrib.integrations.local_baserow.service_types import ( LocalBaserowAggregateRowsUserServiceType, LocalBaserowCreateRowsServiceType, diff --git a/backend/src/baserow/core/ai_provider/constants.py b/backend/src/baserow/core/ai_provider/constants.py index 1a8c205f0c..5c2ab0c6f0 100644 --- a/backend/src/baserow/core/ai_provider/constants.py +++ b/backend/src/baserow/core/ai_provider/constants.py @@ -3,6 +3,7 @@ AI_PROVIDER_FEATURE_AI_FIELDS = "ai_fields" AI_PROVIDER_FEATURE_KUMA = "kuma" +AI_PROVIDER_FEATURE_AI_AGENT = "ai_agent" AI_PROVIDER_FEATURE_MODE_INHERIT = "inherit" AI_PROVIDER_FEATURE_MODE_LEGACY = "legacy" diff --git a/backend/src/baserow/core/ai_provider/handler.py b/backend/src/baserow/core/ai_provider/handler.py index 983224e80b..4c12a0e1dc 100644 --- a/backend/src/baserow/core/ai_provider/handler.py +++ b/backend/src/baserow/core/ai_provider/handler.py @@ -15,6 +15,7 @@ from baserow.core.psycopg import is_unique_violation_error from .constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_MODE_DISABLED, AI_PROVIDER_FEATURE_MODE_INHERIT, @@ -71,11 +72,22 @@ class WorkspaceAIProviderConfig: class AIProviderHandler: @staticmethod def _normalize_feature_types(feature_types: list[str] | None) -> list[str]: + """ + Validate explicit eligibility or preserve the legacy omission default. + + :param feature_types: Requested feature identifiers, or None when omitted. + An explicit empty list leaves the model unavailable to every feature. + :returns: Unique feature identifiers in their original order, or the AI + Fields and AI Agent compatibility default for an omitted value. + :raises AIProviderModelFeatureTypeDoesNotExist: If an explicitly selected + feature type is not registered. + """ + if feature_types is None: # Older API callers predate per-feature model eligibility. Preserve - # their AI Fields-only behaviour instead of silently opting models - # into every feature registered by the running installation. - return [AI_PROVIDER_FEATURE_AI_FIELDS] + # the consumers those models already served instead of silently + # opting them into every feature registered by the installation. + return [AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT] normalized = list(dict.fromkeys(feature_types)) for feature_type in normalized: diff --git a/backend/src/baserow/core/ai_provider/models.py b/backend/src/baserow/core/ai_provider/models.py index ee4637baa6..b2be6cd23b 100644 --- a/backend/src/baserow/core/ai_provider/models.py +++ b/backend/src/baserow/core/ai_provider/models.py @@ -3,6 +3,7 @@ from baserow.core.mixins import CreatedAndUpdatedOnMixin from .constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_TEST_STATUS_FAILURE, AI_PROVIDER_TEST_STATUS_SUCCESS, @@ -10,11 +11,25 @@ def get_default_ai_provider_model_feature_types() -> list[str]: - """Keep API and ORM callers which omit this field backwards compatible.""" + """ + Return the historical default referenced by migration 0119. + + :returns: A new list containing only AI Fields eligibility. + """ return [AI_PROVIDER_FEATURE_AI_FIELDS] +def get_default_ai_provider_model_feature_types_v2() -> list[str]: + """ + Preserve the existing consumers when ORM callers omit model eligibility. + + :returns: A new list containing AI Fields and AI Agent eligibility. + """ + + return [AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT] + + class AIProviderConfig(CreatedAndUpdatedOnMixin, models.Model): """An instance- or workspace-owned AI provider configuration.""" @@ -65,9 +80,9 @@ class TestStatus(models.TextChoices): model_identifier = models.CharField(max_length=255) is_enabled = models.BooleanField(default=True, db_default=True) feature_types = models.JSONField( - default=get_default_ai_provider_model_feature_types, + default=get_default_ai_provider_model_feature_types_v2, blank=True, - db_default=[AI_PROVIDER_FEATURE_AI_FIELDS], + db_default=[AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT], help_text=( "The AI features allowed to select this model. This controls " "eligibility, not which features currently use the model." diff --git a/backend/src/baserow/core/ai_provider/registries.py b/backend/src/baserow/core/ai_provider/registries.py index 595f97531a..20237fbcd3 100644 --- a/backend/src/baserow/core/ai_provider/registries.py +++ b/backend/src/baserow/core/ai_provider/registries.py @@ -8,7 +8,18 @@ class AIProviderModelFeatureType(Instance): - """A Baserow feature for which an AI provider model can be available.""" + """ + A Baserow feature for which an AI provider model can be available. + + ``supports_default_model`` controls how the feature selects its model. + ``True`` means an administrator picks one model for the whole scope + (instance or workspace), stored as an ``AIProviderFeatureSetting`` row + whose RESTRICT foreign key blocks deleting the selected model (Kuma). + ``False`` means each consumer stores its own provider type and model + identifier. Those references create no setting row and do not block deletion; + a removed model surfaces as a validation or dispatch error on the consumer + instead (AI fields, AI Agent nodes). + """ supports_default_model = False required_model_capabilities = (AI_PROVIDER_MODEL_CAPABILITY_TEXT,) diff --git a/backend/src/baserow/core/generative_ai/registries.py b/backend/src/baserow/core/generative_ai/registries.py index 5f2a8367df..4382c302bc 100644 --- a/backend/src/baserow/core/generative_ai/registries.py +++ b/backend/src/baserow/core/generative_ai/registries.py @@ -555,7 +555,15 @@ def _get_complete_legacy_workspace_settings( return self._get_complete_provider_settings(values) def _get_complete_provider_settings(self, values: Any) -> Optional[dict[str, Any]]: - """Validate and normalize one complete provider settings dictionary.""" + """ + Validate and normalize a built-in provider's complete connection. + + :param values: The provider settings to validate, possibly missing or + incomplete legacy JSON. + :returns: The provider's own credentials, normalized model list and every + optional connection setting, with None for absent optional values. + Returns None for incomplete settings or an unsupported provider type. + """ # The database-backed provider feature intentionally supports a closed # set of built-in provider types. GenerativeAIModelType remains an @@ -574,12 +582,39 @@ def _get_complete_provider_settings(self, values: Any) -> Optional[dict[str, Any except InvalidAIProviderSettings: return None + # Include every connection setting, even when its value is absent. The + # returned dictionary is an atomic override: omitting an optional key + # would let provider getters inherit that key from another scope. + extra_settings = provider_values["extra_settings"] return { "api_key": provider_values["api_key"], "models": provider_values["models"], - **provider_values["extra_settings"], + **{ + name: extra_settings.get(name) + for name in AI_PROVIDER_TYPES[self.type]["extra_settings"] + }, } + def get_atomic_settings_override(self, values: Any) -> dict[str, Any] | None: + """Return a self-contained settings override, or ``None`` if incomplete. + + Built-in database-backed providers must supply their own complete + connection so an override can never borrow credentials from another + scope. Out-of-tree providers own their validation and settings contract, + so their dictionaries retain the legacy authoritative behavior. Extensions + with partial/inherited settings should override this hook and return + ``None`` until their connection is complete. + + :param values: The explicit provider settings to consider as an override. + :returns: A new dictionary containing the complete connection, or None + when the values cannot supply one. For out-of-tree providers, the + default implementation copies any dictionary without normalization. + """ + + if self.type not in AI_PROVIDER_TYPES: + return dict(values) if isinstance(values, dict) else None + return self._get_complete_provider_settings(values) + def get_model_settings_override( self, model_name: str, diff --git a/backend/src/baserow/core/migrations/0120_add_ai_agent_provider_model_feature.py b/backend/src/baserow/core/migrations/0120_add_ai_agent_provider_model_feature.py new file mode 100644 index 0000000000..1a6b236683 --- /dev/null +++ b/backend/src/baserow/core/migrations/0120_add_ai_agent_provider_model_feature.py @@ -0,0 +1,64 @@ +from django.db import migrations, models + +import baserow.core.ai_provider.models + + +AI_AGENT_FEATURE = "ai_agent" + + +def add_ai_agent_feature(apps, schema_editor): + AIProviderModel = apps.get_model("core", "AIProviderModel") + provider_models = AIProviderModel.objects.using(schema_editor.connection.alias) + models_to_update = [] + for model in provider_models.only("id", "feature_types"): + feature_types = list(model.feature_types or []) + if AI_AGENT_FEATURE not in feature_types: + model.feature_types = [*feature_types, AI_AGENT_FEATURE] + models_to_update.append(model) + + if models_to_update: + provider_models.bulk_update(models_to_update, ["feature_types"]) + + +def remove_ai_agent_feature(apps, schema_editor): + AIProviderModel = apps.get_model("core", "AIProviderModel") + provider_models = AIProviderModel.objects.using(schema_editor.connection.alias) + models_to_update = [] + for model in provider_models.only("id", "feature_types"): + feature_types = list(model.feature_types or []) + without_ai_agent = [ + feature_type + for feature_type in feature_types + if feature_type != AI_AGENT_FEATURE + ] + if without_ai_agent != feature_types: + model.feature_types = without_ai_agent + models_to_update.append(model) + + if models_to_update: + provider_models.bulk_update(models_to_update, ["feature_types"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0119_aiproviderfeaturesetting_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="aiprovidermodel", + name="feature_types", + field=models.JSONField( + blank=True, + db_default=["ai_fields", "ai_agent"], + default=( + baserow.core.ai_provider.models.get_default_ai_provider_model_feature_types_v2 + ), + help_text=( + "The AI features allowed to select this model. This controls " + "eligibility, not which features currently use the model." + ), + ), + ), + migrations.RunPython(add_ai_agent_feature, remove_ai_agent_feature), + ] diff --git a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py index 122bf52925..74288c7c59 100644 --- a/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py +++ b/backend/tests/baserow/api/ai_provider/test_ai_provider_views.py @@ -511,7 +511,7 @@ def test_provider_crud_never_returns_api_key( "id": response.json()["models"][0]["id"], "model_identifier": "gpt-4o", "is_enabled": True, - "feature_types": ["ai_fields"], + "feature_types": ["ai_fields", "ai_agent"], "last_test_at": None, "last_test_status": None, "last_test_error": "", diff --git a/backend/tests/baserow/contrib/integrations/ai/test_ai_agent_service_type.py b/backend/tests/baserow/contrib/integrations/ai/test_ai_agent_service_type.py index c202fc6216..a13238c06f 100644 --- a/backend/tests/baserow/contrib/integrations/ai/test_ai_agent_service_type.py +++ b/backend/tests/baserow/contrib/integrations/ai/test_ai_agent_service_type.py @@ -1,9 +1,13 @@ import json +from io import StringIO from unittest.mock import patch +from django.core.management import call_command from django.http import HttpRequest import pytest +from rest_framework import serializers +from rest_framework.exceptions import ValidationError as DRFValidationError from baserow.contrib.automation.automation_dispatch_context import ( AutomationDispatchContext, @@ -21,15 +25,21 @@ ) from baserow.contrib.integrations.ai.integration_types import AIIntegrationType from baserow.contrib.integrations.ai.service_types import AIAgentServiceType +from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT from baserow.core.ai_provider.handler import AIProviderHandler from baserow.core.ai_provider.models import AIProviderConfig, AIProviderModel from baserow.core.generative_ai.exceptions import GenerativeAIPromptError +from baserow.core.generative_ai.registries import ( + GenerativeAIModelType, + generative_ai_model_type_registry, +) from baserow.core.integrations.service import IntegrationService from baserow.core.services.exceptions import ( ServiceImproperlyConfiguredDispatchException, UnexpectedDispatchException, ) from baserow.core.services.handler import ServiceHandler +from baserow.core.trash.handler import TrashHandler from baserow.test_utils.helpers import AnyInt from baserow.test_utils.pytest_conftest import FakeDispatchContext @@ -59,6 +69,14 @@ def _prompt( def create_openai_db_provider(workspace, model_identifier="database-model"): + """ + Create an enabled provider and a model available to AI Agent services. + + :param workspace: Owning workspace, or None for an instance provider. + :param model_identifier: Identifier of the model to expose. + :returns: The provider configuration and its model. + """ + provider = AIProviderConfig.objects.create( workspace=workspace, provider_type="openai", @@ -67,10 +85,34 @@ def create_openai_db_provider(workspace, model_identifier="database-model"): model = AIProviderModel.objects.create( provider_config=provider, model_identifier=model_identifier, + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], ) return provider, model +class CustomIntegrationSettingsSerializer(serializers.Serializer): + token = serializers.CharField(required=False) + models = serializers.ListField(child=serializers.CharField(), required=False) + + +class CustomIntegrationGenerativeAIModelType(GenerativeAIModelType): + type = "custom_integration_provider" + supports_legacy_workspace_settings = False + + def get_enabled_models( + self, + workspace=None, + settings_override=None, + feature_type=None, + ): + if settings_override is None: + return [] + return settings_override.get("models", ["custom-model"]) + + def get_settings_serializer(self): + return CustomIntegrationSettingsSerializer + + @pytest.mark.django_db def test_ai_agent_service_creation(data_fixture, settings): settings.BASEROW_OPENAI_API_KEY = "sk-test" @@ -325,6 +367,112 @@ def test_ai_agent_service_inherits_db_provider_in_draft_applications( assert "settings_override" not in prompt.call_args.kwargs +@pytest.mark.django_db +@pytest.mark.parametrize("scope", ["instance", "workspace"]) +@pytest.mark.parametrize("surface", ["builder", "automation"]) +def test_ai_agent_provider_import_and_flag_transition( + data_fixture, settings, scope, surface +): + settings.FEATURE_FLAGS = [] + settings.BASEROW_OPENAI_API_KEY = "environment-key" + settings.BASEROW_OPENAI_MODELS = ["existing-model"] + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + legacy_settings = { + "openai": {"api_key": "workspace-key", "models": ["existing-model"]} + } + if scope == "workspace": + workspace.generative_ai_models_settings = legacy_settings + workspace.save(update_fields=("generative_ai_models_settings",)) + if surface == "builder": + application = data_fixture.create_builder_application( + user=user, workspace=workspace + ) + page = data_fixture.create_builder_page(builder=application) + dispatch_context = BuilderDispatchContext(HttpRequest(), page) + else: + application = data_fixture.create_automation_application( + user=user, workspace=workspace + ) + workflow = data_fixture.create_automation_workflow(automation=application) + dispatch_context = AutomationDispatchContext(workflow, None) + + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service_type = AIAgentServiceType() + values = service_type.prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "existing-model", + "ai_prompt": "'Keep this action working through the cutover'", + }, + user, + ) + service = ServiceHandler().create_service(service_type, **values) + with mock_ai_prompt() as prompt: + result = service_type.dispatch(service, dispatch_context) + assert result.data == {"result": "AI response"} + + call_command( + "migrate_ai_provider_settings", "--scope", scope, "--apply", stdout=StringIO() + ) + provider = AIProviderConfig.objects.get( + provider_type="openai", workspace=workspace if scope == "workspace" else None + ) + model = provider.models.get(model_identifier="existing-model") + assert AI_PROVIDER_FEATURE_AI_AGENT in model.feature_types + AIProviderHandler.update_provider(provider, api_key="database-key") + + openai_type = generative_ai_model_type_registry.get("openai") + legacy_key = "workspace-key" if scope == "workspace" else "environment-key" + for feature_flags, expected_key in ( + ([], legacy_key), + (["ai-providers"], "database-key"), + ): + settings.FEATURE_FLAGS = feature_flags + with mock_ai_prompt() as prompt: + result = service_type.dispatch(service, dispatch_context) + assert result.data == {"result": "AI response"} + assert prompt.call_args.kwargs["workspace"] == workspace + assert ( + openai_type.get_api_key( + workspace, prompt.call_args.kwargs.get("settings_override") + ) + == expected_key + ) + + # Once enabled, an explicit removal must take effect without runtime backfill. + AIProviderHandler.update_model(model, feature_types=["ai_fields"]) + with ( + mock_ai_prompt() as prompt, + pytest.raises( + ServiceImproperlyConfiguredDispatchException, match="not available" + ), + ): + service_type.dispatch(service, dispatch_context) + prompt.assert_not_called() + + settings.FEATURE_FLAGS = [] + with mock_ai_prompt() as prompt: + result = service_type.dispatch(service, dispatch_context) + assert result.data == {"result": "AI response"} + assert ( + openai_type.get_api_key( + workspace, prompt.call_args.kwargs.get("settings_override") + ) + == legacy_key + ) + service.refresh_from_db() + integration.refresh_from_db() + workspace.refresh_from_db() + assert service.ai_generative_ai_model == "existing-model" + assert integration.ai_settings == {} + if scope == "workspace": + assert workspace.generative_ai_models_settings == legacy_settings + + @pytest.mark.django_db def test_ai_agent_service_rejects_model_disabled_after_configuration( data_fixture, settings @@ -1100,3 +1248,872 @@ def test_legacy_workspace_settings_are_materialized_when_publishing( # Settings should be available because they were materialized during export assert provider_settings["api_key"] == "sk-workspace-key" assert provider_settings["models"] == ["gpt-4"] + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +@pytest.mark.parametrize( + "updates", + [ + {"ai_prompt": "'Update a service whose integration was trashed'"}, + {"ai_generative_ai_model": "replacement-model"}, + ], +) +def test_prepare_values_allows_editing_service_with_trashed_integration( + data_fixture, settings, feature_flags, updates +): + settings.FEATURE_FLAGS = feature_flags + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service_type = AIAgentServiceType() + service = ServiceHandler().create_service( + service_type, + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="existing-model", + ) + integration.trashed = True + integration.save(update_fields=["trashed"]) + + assert service_type.prepare_values(dict(updates), user, service) == updates + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_prepare_values_preserves_trashed_integration_until_restore( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={ + "openai": {"api_key": "integration-key", "models": ["existing-model"]} + }, + ) + IntegrationService().delete_integration(user, integration) + + service_type = AIAgentServiceType() + values = service_type.prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "existing-model", + "ai_prompt": "'Resume after the integration is restored'", + }, + user, + ) + assert values["integration"].id == integration.id + assert values["integration"].trashed is True + + service_handler = ServiceHandler() + service = service_handler.create_service(service_type, **values) + service = service_handler.get_service(service.id) + assert service.integration_id == integration.id + assert service.integration is None + with ( + mock_ai_prompt() as prompt, + pytest.raises(ServiceImproperlyConfiguredDispatchException, match="trashed"), + ): + service_handler.dispatch_service(service, FakeDispatchContext()) + prompt.assert_not_called() + + TrashHandler.restore_item(user, "integration", integration.id) + service = service_handler.get_service(service.id) + assert service.integration_id == integration.id + with mock_ai_prompt() as prompt: + result = service_handler.dispatch_service(service, FakeDispatchContext()) + assert result.data == {"result": "AI response"} + prompt.assert_called_once() + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_prepare_values_rejects_nonexistent_integration( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + integration_id = integration.id + integration.delete() + + with pytest.raises(DRFValidationError, match="integration.*does not exist"): + AIAgentServiceType().prepare_values( + { + "integration_id": integration_id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "existing-model", + }, + user, + ) + + +@pytest.mark.django_db +def test_prepare_values_accepts_database_backed_model_with_flag(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="agent-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + + values = AIAgentServiceType().prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "agent-model", + }, + user, + ) + + assert values["ai_generative_ai_model"] == "agent-model" + + +@pytest.mark.django_db +def test_prepare_values_rejects_model_restricted_to_other_features( + data_fixture, settings +): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="fields-only-model", + feature_types=["ai_fields"], + ) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + + with pytest.raises(DRFValidationError): + AIAgentServiceType().prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "fields-only-model", + }, + user, + ) + + +@pytest.mark.django_db +def test_prepare_values_skips_validation_when_selection_unchanged( + data_fixture, settings +): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="stale-model", + ai_output_type="text", + ai_prompt="'Old prompt'", + ) + + values = AIAgentServiceType().prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "stale-model", + "ai_prompt": "'New prompt'", + }, + user, + instance=service, + ) + + assert values["ai_prompt"] == "'New prompt'" + + +@pytest.mark.django_db +def test_prepare_values_validates_changed_selection_on_update(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="agent-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with pytest.raises(DRFValidationError): + AIAgentServiceType().prepare_values( + {"ai_generative_ai_model": "unknown-model"}, + user, + instance=service, + ) + + +@pytest.mark.django_db +def test_dispatch_uses_database_provider_settings_with_flag(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="agent-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="agent-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch( + "baserow.core.generative_ai.generative_ai_model_types.OpenAIGenerativeAIModelType.prompt" + ) as mock_prompt: + mock_prompt.return_value = "Response" + service.get_type().dispatch(service, FakeDispatchContext()) + + call_kwargs = mock_prompt.call_args[1] + assert call_kwargs["workspace"] == application.workspace + assert "settings_override" not in call_kwargs + + +@pytest.mark.django_db +def test_dispatch_rejects_model_restricted_to_other_features(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="fields-only-model", + feature_types=["ai_fields"], + ) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="fields-only-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch( + "baserow.core.generative_ai.generative_ai_model_types.OpenAIGenerativeAIModelType.prompt" + ) as mock_prompt: + with pytest.raises(ServiceImproperlyConfiguredDispatchException): + service.get_type().dispatch(service, FakeDispatchContext()) + mock_prompt.assert_not_called() + + +@pytest.mark.django_db +def test_dispatch_reports_uninstalled_provider_as_unavailable(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="removed-provider", + ai_generative_ai_model="removed-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with pytest.raises( + ServiceImproperlyConfiguredDispatchException, + match="removed-provider.*unavailable", + ): + service.get_type().dispatch(service, FakeDispatchContext()) + + +@pytest.mark.django_db +def test_dispatch_partial_blob_does_not_bypass_feature_gate(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="fields-only-model", + feature_types=["ai_fields"], + ) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"models": ["fields-only-model"]}}, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="fields-only-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch( + "baserow.core.generative_ai.generative_ai_model_types.OpenAIGenerativeAIModelType.prompt" + ) as mock_prompt: + with pytest.raises(ServiceImproperlyConfiguredDispatchException): + service.get_type().dispatch(service, FakeDispatchContext()) + mock_prompt.assert_not_called() + + +@pytest.mark.django_db +def test_dispatch_keeps_env_configured_model_working_with_flag(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + settings.BASEROW_OPENAI_API_KEY = "sk-env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="env-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch( + "baserow.core.generative_ai.generative_ai_model_types.OpenAIGenerativeAIModelType.prompt" + ) as mock_prompt: + mock_prompt.return_value = "Response" + result = service.get_type().dispatch(service, FakeDispatchContext()) + + assert result.data == {"result": "Response"} + assert "settings_override" not in mock_prompt.call_args[1] + + +@pytest.mark.django_db +def test_dispatch_integration_settings_win_over_database_with_flag( + data_fixture, settings +): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="gpt-4", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"api_key": "sk-integration-key", "models": ["gpt-4"]}}, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="gpt-4", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch( + "baserow.core.generative_ai.generative_ai_model_types.OpenAIGenerativeAIModelType.prompt" + ) as mock_prompt: + mock_prompt.return_value = "Response" + service.get_type().dispatch(service, FakeDispatchContext()) + + assert ( + mock_prompt.call_args[1]["settings_override"]["api_key"] == "sk-integration-key" + ) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "feature_flags, inherited_scope, expected_model", + [ + ([], "instance", "env-model"), + ([], "workspace", "workspace-model"), + (["ai-providers"], "instance", "agent-model"), + (["ai-providers"], "workspace", "agent-model"), + ], +) +def test_connection_only_override_inherits_available_models( + data_fixture, settings, feature_flags, inherited_scope, expected_model +): + settings.FEATURE_FLAGS = feature_flags + settings.BASEROW_OPENAI_API_KEY = "env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + settings.BASEROW_OPENAI_BASE_URL = "https://instance.example/v1" + settings.BASEROW_OPENAI_ORGANIZATION = "instance-organization" + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + workspace = application.workspace + if inherited_scope == "workspace": + workspace.generative_ai_models_settings = { + "openai": { + "api_key": "workspace-key", + "models": ["workspace-model"], + "base_url": "https://workspace.example/v1", + "organization": "workspace-organization", + } + } + workspace.save(update_fields=("generative_ai_models_settings",)) + if feature_flags: + provider, _ = create_openai_db_provider( + workspace if inherited_scope == "workspace" else None, + "agent-model", + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="fields-only-model", + feature_types=["ai_fields"], + ) + + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"api_key": "integration-key"}}, + ) + integration.refresh_from_db() + assert "models" not in integration.ai_settings["openai"] + service_type = AIAgentServiceType() + service_values = { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": expected_model, + "ai_prompt": "'Use my connection with inherited models'", + } + service = ServiceHandler().create_service( + service_type, **service_type.prepare_values(dict(service_values), user) + ) + + with mock_ai_prompt() as prompt: + result = service_type.dispatch(service, FakeDispatchContext()) + + assert result.data == {"result": "AI response"} + settings_override = prompt.call_args.kwargs["settings_override"] + assert settings_override["models"] == [expected_model] + openai_type = generative_ai_model_type_registry.get("openai") + assert openai_type.get_api_key(workspace, settings_override) == "integration-key" + assert openai_type.get_base_url(workspace, settings_override) is None + assert openai_type.get_organization(workspace, settings_override) is None + + # An omitted model list still respects the inherited allowlist, including + # feature eligibility for database-backed models. + unavailable_model = "fields-only-model" if feature_flags else "unknown-model" + service_values["ai_generative_ai_model"] = unavailable_model + with pytest.raises(DRFValidationError, match="not available"): + service_type.prepare_values(service_values, user) + service.ai_generative_ai_model = unavailable_model + with ( + mock_ai_prompt() as prompt, + pytest.raises(ServiceImproperlyConfiguredDispatchException), + ): + service_type.dispatch(service, FakeDispatchContext()) + prompt.assert_not_called() + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_complete_override_preserves_explicit_empty_models( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + settings.BASEROW_OPENAI_API_KEY = "env-key" + settings.BASEROW_OPENAI_MODELS = ["configured-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + if feature_flags: + create_openai_db_provider(application.workspace, "configured-model") + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"api_key": "integration-key", "models": []}}, + ) + integration.refresh_from_db() + assert integration.ai_settings["openai"]["models"] == [] + service_type = AIAgentServiceType() + service_values = { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "configured-model", + "ai_prompt": "'Do not inherit the model list'", + } + with pytest.raises(DRFValidationError, match="not available"): + service_type.prepare_values(dict(service_values), user) + service = ServiceHandler().create_service(service_type, **service_values) + with ( + mock_ai_prompt() as prompt, + pytest.raises(ServiceImproperlyConfiguredDispatchException), + ): + service_type.dispatch(service, FakeDispatchContext()) + prompt.assert_not_called() + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_complete_override_does_not_inherit_optional_connection_settings( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + settings.BASEROW_OPENAI_BASE_URL = "https://attacker.example/env" + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + workspace.generative_ai_models_settings = { + "openai": { + "api_key": "workspace-key", + "base_url": "https://attacker.example/workspace", + "models": ["workspace-model"], + } + } + workspace.save(update_fields=("generative_ai_models_settings",)) + provider = AIProviderConfig.objects.create( + workspace=workspace, + provider_type="openai", + api_key="database-key", + extra_settings={"base_url": "https://attacker.example/database"}, + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="database-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + application = data_fixture.create_builder_application( + user=user, workspace=workspace + ) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={ + "openai": { + "api_key": "integration-key", + "models": ["integration-model"], + } + }, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="integration-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with mock_ai_prompt() as prompt: + service.get_type().dispatch(service, FakeDispatchContext()) + + settings_override = prompt.call_args.kwargs["settings_override"] + assert settings_override["api_key"] == "integration-key" + assert settings_override["base_url"] is None + assert settings_override["organization"] is None + openai_type = generative_ai_model_type_registry.get("openai") + assert openai_type.get_base_url(workspace, settings_override) is None + assert openai_type.get_organization(workspace, settings_override) is None + + +@pytest.mark.django_db +def test_prepare_values_rejects_unknown_env_model_without_provider_flag( + data_fixture, settings +): + settings.FEATURE_FLAGS = [] + settings.BASEROW_OPENAI_API_KEY = "sk-env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, AIIntegrationType(), application=application, ai_settings={} + ) + + with pytest.raises(DRFValidationError, match="unknown-model.*not available"): + AIAgentServiceType().prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "unknown-model", + }, + user, + ) + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_prepare_values_rejects_partial_override_model_not_in_legacy_allowlist( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + settings.BASEROW_OPENAI_API_KEY = "sk-env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"models": ["attacker-model"]}}, + ) + + with pytest.raises(DRFValidationError, match="attacker-model.*not available"): + AIAgentServiceType().prepare_values( + { + "integration_id": integration.id, + "ai_generative_ai_type": "openai", + "ai_generative_ai_model": "attacker-model", + }, + user, + ) + + +@pytest.mark.django_db +def test_dispatch_does_not_forward_partial_integration_override(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + create_openai_db_provider(application.workspace) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={ + "openai": { + "base_url": "https://attacker.example/v1", + "models": ["database-model"], + } + }, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="database-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with mock_ai_prompt() as prompt: + result = service.get_type().dispatch(service, FakeDispatchContext()) + + assert result.data == {"result": "AI response"} + assert prompt.call_args.kwargs["workspace"] == application.workspace + assert "settings_override" not in prompt.call_args.kwargs + + +@pytest.mark.django_db +def test_dispatch_does_not_mix_legacy_credentials_with_partial_override( + data_fixture, settings +): + settings.FEATURE_FLAGS = [] + settings.BASEROW_OPENAI_API_KEY = "sk-env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={ + "openai": { + "base_url": "https://attacker.example/v1", + "models": ["env-model"], + } + }, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="env-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with mock_ai_prompt() as prompt: + result = service.get_type().dispatch(service, FakeDispatchContext()) + + assert result.data == {"result": "AI response"} + assert prompt.call_args.kwargs["workspace"] == application.workspace + assert "settings_override" not in prompt.call_args.kwargs + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +def test_dispatch_rejects_partial_override_model_not_in_legacy_allowlist( + data_fixture, settings, feature_flags +): + settings.FEATURE_FLAGS = feature_flags + settings.BASEROW_OPENAI_API_KEY = "sk-env-key" + settings.BASEROW_OPENAI_MODELS = ["env-model"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={"openai": {"models": ["attacker-model"]}}, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="attacker-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with mock_ai_prompt() as prompt: + with pytest.raises(ServiceImproperlyConfiguredDispatchException): + service.get_type().dispatch(service, FakeDispatchContext()) + + prompt.assert_not_called() + + +@pytest.mark.django_db +def test_dispatch_rejects_model_removed_from_complete_integration_override( + data_fixture, settings +): + settings.FEATURE_FLAGS = ["ai-providers"] + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={ + "openai": { + "api_key": "sk-integration-key", + "models": ["selected-model"], + } + }, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type="openai", + ai_generative_ai_model="selected-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + IntegrationService().update_integration( + user, + integration, + ai_settings={ + "openai": { + "api_key": "sk-integration-key", + "models": ["replacement-model"], + } + }, + ) + service.refresh_from_db() + + with ( + mock_ai_prompt() as prompt, + pytest.raises( + ServiceImproperlyConfiguredDispatchException, + match="selected-model.*not available", + ), + ): + service.get_type().dispatch(service, FakeDispatchContext()) + + prompt.assert_not_called() + + +@pytest.mark.django_db +@pytest.mark.parametrize("feature_flags", [[], ["ai-providers"]]) +@pytest.mark.parametrize( + "integration_settings", + [ + {"token": "custom-token", "models": ["custom-model"]}, + {"token": "custom-token"}, + ], +) +def test_dispatch_preserves_custom_provider_integration_override( + data_fixture, settings, feature_flags, integration_settings +): + settings.FEATURE_FLAGS = feature_flags + model_type = CustomIntegrationGenerativeAIModelType() + generative_ai_model_type_registry.register(model_type) + try: + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration = IntegrationService().create_integration( + user, + AIIntegrationType(), + application=application, + ai_settings={model_type.type: integration_settings}, + ) + service = ServiceHandler().create_service( + AIAgentServiceType(), + integration_id=integration.id, + ai_generative_ai_type=model_type.type, + ai_generative_ai_model="custom-model", + ai_output_type="text", + ai_prompt="'Test'", + ) + + with patch.object( + model_type, "prompt", return_value="Custom response" + ) as prompt: + result = service.get_type().dispatch(service, FakeDispatchContext()) + + assert result.data == {"result": "Custom response"} + assert prompt.call_args.kwargs["settings_override"] == integration_settings + finally: + generative_ai_model_type_registry.unregister(model_type) diff --git a/backend/tests/baserow/contrib/integrations/ai/test_ai_integration_type.py b/backend/tests/baserow/contrib/integrations/ai/test_ai_integration_type.py index 3a4bf01312..d5dee133f6 100644 --- a/backend/tests/baserow/contrib/integrations/ai/test_ai_integration_type.py +++ b/backend/tests/baserow/contrib/integrations/ai/test_ai_integration_type.py @@ -503,3 +503,52 @@ def test_ai_integration_settings_hierarchy(data_fixture, settings): provider_settings = integration_type.get_provider_settings(integration, "openai") assert provider_settings["api_key"] == "sk-workspace-key" assert provider_settings["models"] == ["gpt-4"] + + +@pytest.mark.django_db +def test_get_integration_provider_settings_returns_blob_or_none(data_fixture): + user = data_fixture.create_user() + application = data_fixture.create_builder_application(user=user) + integration_type = AIIntegrationType() + integration = IntegrationService().create_integration( + user, + integration_type, + application=application, + ai_settings={"openai": {"api_key": "sk-blob", "models": ["gpt-4"]}}, + ) + + blob = integration_type.get_integration_provider_settings(integration, "openai") + assert blob == {"api_key": "sk-blob", "models": ["gpt-4"]} + assert ( + integration_type.get_integration_provider_settings(integration, "anthropic") + is None + ) + + +@pytest.mark.django_db +def test_get_integration_provider_settings_ignores_workspace_settings( + data_fixture, settings +): + settings.FEATURE_FLAGS = [] + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + workspace.generative_ai_models_settings = { + "openai": {"api_key": "sk-workspace", "models": ["gpt-4"]} + } + workspace.save() + application = data_fixture.create_builder_application( + user=user, workspace=workspace + ) + integration_type = AIIntegrationType() + integration = IntegrationService().create_integration( + user, integration_type, application=application, ai_settings={} + ) + + assert ( + integration_type.get_integration_provider_settings(integration, "openai") + is None + ) + assert integration_type.get_provider_settings(integration, "openai") == { + "api_key": "sk-workspace", + "models": ["gpt-4"], + } diff --git a/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py new file mode 100644 index 0000000000..5c7ea6e7da --- /dev/null +++ b/backend/tests/baserow/contrib/integrations/ai/test_ai_provider_feature_types.py @@ -0,0 +1,58 @@ +import pytest + +from baserow.core.ai_provider.constants import AI_PROVIDER_FEATURE_AI_AGENT +from baserow.core.ai_provider.handler import AIProviderHandler +from baserow.core.ai_provider.models import AIProviderConfig, AIProviderModel +from baserow.core.ai_provider.registries import ( + ai_provider_model_feature_type_registry, +) + + +def test_ai_agent_feature_type_is_registered(): + feature_type = ai_provider_model_feature_type_registry.get( + AI_PROVIDER_FEATURE_AI_AGENT + ) + assert feature_type.supports_default_model is False + + +@pytest.mark.django_db +def test_ai_agent_availability_lists_only_feature_models(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + workspace = data_fixture.create_workspace() + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="agent-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + AIProviderModel.objects.create( + provider_config=provider, + model_identifier="fields-only-model", + feature_types=["ai_fields"], + ) + + availability = ai_provider_model_feature_type_registry.get_workspace_availability( + workspace + )[AI_PROVIDER_FEATURE_AI_AGENT] + + assert availability["is_enabled"] is True + assert availability["models"] == {"openai": ["agent-model"]} + + +@pytest.mark.django_db +def test_agent_selection_does_not_block_model_deletion(data_fixture, settings): + settings.FEATURE_FLAGS = ["ai-providers"] + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="database-key" + ) + model = AIProviderModel.objects.create( + provider_config=provider, + model_identifier="agent-model", + feature_types=[AI_PROVIDER_FEATURE_AI_AGENT], + ) + + AIProviderHandler.delete_model(model) + + assert not AIProviderModel.objects.filter(id=model.id).exists() diff --git a/backend/tests/baserow/core/ai_provider/test_handler.py b/backend/tests/baserow/core/ai_provider/test_handler.py index d9b93276d2..5961ddc899 100644 --- a/backend/tests/baserow/core/ai_provider/test_handler.py +++ b/backend/tests/baserow/core/ai_provider/test_handler.py @@ -7,6 +7,7 @@ from pydantic_ai.models.test import TestModel from baserow.core.ai_provider.constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_KUMA, AI_PROVIDER_FEATURE_MODE_DISABLED, @@ -76,14 +77,31 @@ def test_create_model_does_not_misreport_unexpected_integrity_error(): @pytest.mark.django_db -def test_omitted_model_feature_types_remain_ai_fields_only(): +def test_omitted_model_feature_types_remain_available_to_legacy_consumers(): provider = AIProviderHandler.create_provider( "openai", api_key="secret", models_data=[{"model_identifier": "gpt-5.4"}], ) - assert provider.models.get().feature_types == [AI_PROVIDER_FEATURE_AI_FIELDS] + assert provider.models.get().feature_types == [ + AI_PROVIDER_FEATURE_AI_FIELDS, + AI_PROVIDER_FEATURE_AI_AGENT, + ] + + +@pytest.mark.django_db +def test_orm_model_default_remains_available_to_legacy_consumers(): + provider = AIProviderConfig.objects.create(provider_type="openai", api_key="secret") + + model = AIProviderModel.objects.create( + provider_config=provider, model_identifier="gpt-5.4" + ) + + assert model.feature_types == [ + AI_PROVIDER_FEATURE_AI_FIELDS, + AI_PROVIDER_FEATURE_AI_AGENT, + ] @pytest.mark.django_db diff --git a/backend/tests/baserow/core/management/test_migrate_ai_provider_settings.py b/backend/tests/baserow/core/management/test_migrate_ai_provider_settings.py index 3ddb7c2f10..05fac49cca 100644 --- a/backend/tests/baserow/core/management/test_migrate_ai_provider_settings.py +++ b/backend/tests/baserow/core/management/test_migrate_ai_provider_settings.py @@ -5,7 +5,11 @@ import pytest -from baserow.core.ai_provider.constants import PROVIDER_ENVIRONMENT_SETTINGS +from baserow.core.ai_provider.constants import ( + AI_PROVIDER_FEATURE_AI_AGENT, + AI_PROVIDER_FEATURE_AI_FIELDS, + PROVIDER_ENVIRONMENT_SETTINGS, +) from baserow.core.ai_provider.handler import AIProviderHandler from baserow.core.ai_provider.models import AIProviderConfig, AIProviderModel from baserow.core.ai_provider.registries import ( @@ -54,8 +58,8 @@ def test_command_previews_then_imports_without_printing_secrets(settings): "gpt-4o-mini", ] assert list(provider.models.values_list("feature_types", flat=True)) == [ - ["ai_fields"], - ["ai_fields"], + [AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT], + [AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT], ] assert "Imported 1 missing instance provider(s)." in out.getvalue() assert "super-secret" not in out.getvalue() @@ -77,7 +81,10 @@ def test_command_imports_on_oss_only_installations(settings, monkeypatch): ) model = AIProviderModel.objects.get() - assert model.feature_types == ["ai_fields"] + assert model.feature_types == [ + AI_PROVIDER_FEATURE_AI_FIELDS, + AI_PROVIDER_FEATURE_AI_AGENT, + ] @pytest.mark.django_db @@ -281,7 +288,8 @@ def test_workspace_scope_migrates_every_current_legacy_provider_setting( == expected_models ) assert list(provider.models.values_list("feature_types", flat=True)) == [ - ["ai_fields"] for _ in expected_models + [AI_PROVIDER_FEATURE_AI_FIELDS, AI_PROVIDER_FEATURE_AI_AGENT] + for _ in expected_models ] model_type = generative_ai_model_type_registry.get(provider_type) diff --git a/backend/tests/baserow/core/migrations/test_core_migrations.py b/backend/tests/baserow/core/migrations/test_core_migrations.py index 4f377d47a9..fe9a248aa7 100644 --- a/backend/tests/baserow/core/migrations/test_core_migrations.py +++ b/backend/tests/baserow/core/migrations/test_core_migrations.py @@ -1,4 +1,4 @@ -from django.db import IntegrityError +from django.db import IntegrityError, connection from django.utils import timezone import pytest @@ -121,3 +121,117 @@ def test_0119_initializes_ai_provider_model_features_and_capabilities( } assert untested_model.feature_types == ["ai_fields"] assert untested_model.last_test_capabilities == {} + + +@pytest.mark.once_per_day_in_ci +def test_0120_makes_existing_ai_provider_models_available_to_ai_agents_and_reverses( + migrator, teardown_table_metadata, settings +): + settings.FEATURE_FLAGS = [] + old_state = migrator.migrate( + [("core", "0119_aiproviderfeaturesetting_and_more")] + ) + AIProviderConfig = old_state.apps.get_model("core", "AIProviderConfig") + AIProviderModel = old_state.apps.get_model("core", "AIProviderModel") + provider = AIProviderConfig.objects.create( + provider_type="openai", api_key="test-key" + ) + historical_default_model = AIProviderModel.objects.create( + provider_config=provider, + model_identifier="historical-default-model", + last_test_capabilities={ + "text": {"status": "success", "error": ""} + }, + ) + assert historical_default_model.feature_types == ["ai_fields"] + + models = { + feature_types[0] if feature_types else "none": AIProviderModel.objects.create( + provider_config=provider, + model_identifier=f"{index}-model", + feature_types=feature_types, + last_test_capabilities={ + "text": {"status": "success", "error": ""} + }, + ) + for index, feature_types in enumerate( + ( + ["ai_fields"], + ["kuma"], + [], + ["ai_agent", "ai_fields"], + ["extension_feature", "kuma"], + ) + ) + } + models["historical_default"] = historical_default_model + + new_state = migrator.migrate( + [("core", "0120_add_ai_agent_provider_model_feature")] + ) + NewAIProviderModel = new_state.apps.get_model("core", "AIProviderModel") + + expected_features = { + "ai_fields": ["ai_fields", "ai_agent"], + "kuma": ["kuma", "ai_agent"], + "none": ["ai_agent"], + "ai_agent": ["ai_agent", "ai_fields"], + "extension_feature": ["extension_feature", "kuma", "ai_agent"], + "historical_default": ["ai_fields", "ai_agent"], + } + for key, model in models.items(): + migrated_model = NewAIProviderModel.objects.get(id=model.id) + assert migrated_model.feature_types == expected_features[key] + assert migrated_model.last_test_capabilities == { + "text": {"status": "success", "error": ""} + } + + default_model = NewAIProviderModel.objects.create( + provider_config_id=provider.id, model_identifier="new-orm-default" + ) + assert default_model.feature_types == ["ai_fields", "ai_agent"] + # Bypass the ORM's Python default to verify the actual database default. + with connection.cursor() as cursor: + cursor.execute( + "INSERT INTO core_aiprovidermodel (provider_config_id, model_identifier) " + "VALUES (%s, %s) RETURNING id", + [provider.id, "new-database-default"], + ) + database_default_id = cursor.fetchone()[0] + database_default_model = NewAIProviderModel.objects.get(id=database_default_id) + assert database_default_model.feature_types == ["ai_fields", "ai_agent"] + + rolled_back_state = migrator.migrate( + [("core", "0119_aiproviderfeaturesetting_and_more")] + ) + RolledBackAIProviderModel = rolled_back_state.apps.get_model( + "core", "AIProviderModel" + ) + expected_rolled_back_features = { + "ai_fields": ["ai_fields"], + "kuma": ["kuma"], + "none": [], + "ai_agent": ["ai_fields"], + "extension_feature": ["extension_feature", "kuma"], + "historical_default": ["ai_fields"], + } + for key, model in models.items(): + rolled_back_model = RolledBackAIProviderModel.objects.get(id=model.id) + assert rolled_back_model.feature_types == expected_rolled_back_features[key] + assert rolled_back_model.last_test_capabilities == { + "text": {"status": "success", "error": ""} + } + + default_model = RolledBackAIProviderModel.objects.create( + provider_config_id=provider.id, model_identifier="restored-orm-default" + ) + assert default_model.feature_types == ["ai_fields"] + with connection.cursor() as cursor: + cursor.execute( + "INSERT INTO core_aiprovidermodel (provider_config_id, model_identifier) " + "VALUES (%s, %s) RETURNING id", + [provider.id, "restored-database-default"], + ) + database_default_id = cursor.fetchone()[0] + database_default_model = RolledBackAIProviderModel.objects.get(id=database_default_id) + assert database_default_model.feature_types == ["ai_fields"] diff --git a/docs/development/feature-flags.md b/docs/development/feature-flags.md index 7064d645d1..6190a76ba0 100644 --- a/docs/development/feature-flags.md +++ b/docs/development/feature-flags.md @@ -14,80 +14,59 @@ Add/remove features flags to the list below: ### Preparing the `ai-providers` feature -There are two sources of legacy configuration and both must be imported: AI provider -**environment variables** become instance-level providers, and each workspace's legacy -`generative_ai_models_settings` JSON becomes workspace-owned providers. Run the whole -sequence with the flag still disabled, because until a workspace's legacy JSON is -imported the new UI reports no workspace provider while that JSON is still what -resolves at runtime. - -1. Before deploying the release, keep `ai-providers` disabled. If the installation - uses `FEATURE_FLAGS=*`, first make a separate configuration-only rollout of the - currently installed release with an explicit list of the other required flags; - there is no negative override for one flag. Wait for every wildcard-configured - web and worker process to drain before deploying the new image. Do not combine - this configuration change with a rolling image update: old processes would still - enable `ai-providers`. If the platform cannot roll out configuration separately, - stop the old processes before starting the new release with the explicit list. -2. Deploy the release and let the old web and worker processes drain. Pause changes - to instance and workspace AI provider settings until the import is complete and - the flag has been enabled; otherwise a workspace can change its legacy JSON after - the command has read it. - The new Google and Groq provider types have no environment variables and are - configured in the admin UI only. Older frontend bundles do not have those - provider types in their registry and cannot safely render a workspace payload - containing them, so add either provider only after every old frontend process - has drained. Draining frontend processes does not replace JavaScript already - loaded in a browser tab: before adding either provider, require active users to - reload Baserow (or close and reopen it) onto the new frontend assets. Keep the - settings-write pause in place until that client cutover is complete. -3. Preview both scopes. The command writes nothing without `--apply`: - -```bash -just b manage migrate_ai_provider_settings --scope instance -just b manage migrate_ai_provider_settings --scope workspace -``` - -4. Review every warning. Repair or explicitly accept each incomplete legacy setting - and each difference from an existing database provider before proceeding. The - importer preserves the database provider in a conflict, and an incomplete - workspace override can inherit the instance provider after the flag is enabled. - Then apply each scope atomically, instance first: - -```bash -just b manage migrate_ai_provider_settings --scope instance --apply -just b manage migrate_ai_provider_settings --scope workspace --apply -``` - -5. Redeploy or restart every web, backend, and worker process with `ai-providers` - enabled, then wait for every feature-disabled process to drain before ending - the settings-write pause. This prevents one generation from resolving legacy - settings while another resolves the imported database settings. Wildcard - installations can now restore `FEATURE_FLAGS=*`. -6. Republish any Application Builder site or Automation workflow that uses an AI - integration without its own provider override. Publications created before the - switch contain a snapshot of the inherited legacy workspace settings; republishing - replaces that snapshot with live database-backed workspace inheritance. Explicit - per-integration provider overrides remain self-contained and do not need this step. - -The command never prints credentials and preserves provider types already configured -at the selected scope, so both imports are safe to run again — only missing -provider types are imported. - -The command does not import Kuma's legacy model or provider-native credentials. -Kuma continues to use that legacy configuration when its database selection is -unconfigured or invalid. An explicit instance or workspace disable remains -authoritative and does not fall back. An administrator can clear an instance -database selection with **Use legacy environment model**, which displays the -configured model, before or after the feature gate is retired. - -When retiring `ai-providers`, make the database-backed paths unconditional in both -the backend and frontend; a missing flag must not route clients back to legacy-only -payloads. Remove only the feature gates in that release and retain the legacy provider -and Kuma resolution fallbacks: self-hosted installations can skip the manual import -sequence, and some legacy Kuma providers cannot yet be represented by database-backed -providers. Retiring those fallbacks requires a separate migration which covers every -supported provider and credential source. +Migration `core.0120` runs in both flag states. It adds AI Agent eligibility to +existing models, preserves their other features, and defaults omitted feature +selections to AI Fields and AI Agent. Keeping the flag disabled requires no provider +imports or republishing. + +Review [integration override compatibility](../testing/kuma-model-settings-test-plan.md#65-explicit-integration-overrides) +before upgrading: these rules apply in both flag states, including to existing +publications. + +To enable database-backed providers on an existing installation: + +1. Pause AI settings changes and deploy with the flag disabled. Drain old web and + worker processes. With `FEATURE_FLAGS=*`, first roll out an explicit list of the + other flags and drain wildcard processes before deploying; alternatively, stop + the old processes first. +2. Preview both imports: environment settings become instance providers, and legacy + workspace JSON becomes workspace providers. Review warnings and reconcile conflicts. + + ```bash + just b manage migrate_ai_provider_settings --scope instance + just b manage migrate_ai_provider_settings --scope workspace + ``` + +3. Apply instance settings first, then workspace settings: + + ```bash + just b manage migrate_ai_provider_settings --scope instance --apply + just b manage migrate_ai_provider_settings --scope workspace --apply + ``` + + Each scope is atomic and imports only missing providers, without printing + credentials. Repeating an import does not synchronize changes to existing providers. +4. Restart all web, backend, and worker processes with `ai-providers` enabled and + drain the previous processes. Reload administrator/editor tabs before resuming + settings changes, and all browser tabs before adding new provider types such as + Google or Groq. +5. Republish sites and workflows containing inherited legacy AI settings snapshots + to adopt live database credentials and eligibility. Review pending draft changes + first: republishing makes them live too. Explicit complete integration overrides + remain independent. + +For installations already using the flag, pause settings changes, upgrade with old +processes stopped, and reload administrator/editor tabs before resuming. Keep the +flag enabled unless usable legacy settings have been verified. + +For rollback, retain the schema and verify legacy settings before disabling the +flag. Database changes are not copied back; deleting an imported workspace provider +also removes its legacy JSON. Verify published sites and workflows too. + +Kuma's legacy model and provider-native credentials are not imported; see +[AI assistant configuration](../installation/ai-assistant.md#2-minimal-enablement). +Use the [transition test plan](../testing/kuma-model-settings-test-plan.md#11-transition-from-legacy-settings-to-database-providers) +to rehearse adoption and rollback. ## Enabling feature flags diff --git a/docs/installation/ai-assistant.md b/docs/installation/ai-assistant.md index 8812c2267e..f0ff4c83f2 100644 --- a/docs/installation/ai-assistant.md +++ b/docs/installation/ai-assistant.md @@ -14,7 +14,8 @@ server. disable Kuma in its workspace AI provider settings. - `BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL` remains the legacy fallback while the `ai-providers` feature is disabled, or while its Kuma selection is unconfigured - or invalid. An explicit instance or workspace disable remains authoritative. + or invalid. While the feature is enabled, an explicit instance or workspace + disable remains authoritative. - The assistant has been mostly tested with the `gpt-oss-120b` family. Other models can work as well. @@ -22,55 +23,23 @@ server. For a fresh database-backed setup, enable `ai-providers`, then add a provider and its models in the admin UI. On each model, choose whether it is available to Kuma, -AI Fields, or both, then select the Kuma model in the **AI features** section. -Availability permits a feature to choose a model; it does not force AI Fields to -use Kuma's model. Use **Test model** to check every selected feature. AI Fields -check for a text response, while Kuma also checks tool calling. - -For every existing installation, deploy this release with `ai-providers` still -disabled and wait for the previous web and worker processes to drain. If the -installation uses the `*` catch-all flag, first roll out an explicit list of the -other flags to every process on the currently installed release. Wait for all -wildcard-configured processes to drain before deploying the new image; do not combine -these changes in one rolling update. If configuration cannot be rolled out -separately, stop the old processes before starting the new release with the explicit -flag list. Pause changes to instance and workspace AI settings through the import and -feature switch. -The database-backed Google and Groq providers are configured through the admin UI, -not new provider environment variables. Add either provider after the old frontend -processes have drained; older -bundles cannot render these provider types. Also require active users to reload -Baserow, or close and reopen their tabs, before either provider can appear in API -or realtime payloads: draining the frontend processes does not replace JavaScript -already loaded by a browser. Keep the settings-write pause in place until that -client cutover is complete. -Preview both scopes before applying them, then enable the feature: - -```bash -baserow migrate_ai_provider_settings --scope instance -baserow migrate_ai_provider_settings --scope workspace -baserow migrate_ai_provider_settings --scope instance --apply -baserow migrate_ai_provider_settings --scope workspace --apply -``` - -Review every warning before enabling the feature. Repair or explicitly accept -incomplete legacy settings and differences from an existing database provider. The -importer keeps an existing database provider in a conflict, while an incomplete -workspace override can inherit the instance provider after the switch. Keep the -settings-write pause in place while you redeploy or restart every web, backend, and -worker process with `ai-providers` enabled. Wait for every feature-disabled process -to drain before ending the pause; otherwise different process generations can resolve -different settings, or a workspace can change its legacy JSON after the command reads -it. - -After the switch, republish each Application Builder site or Automation workflow that -uses an AI integration without its own provider override. Older publications contain -a snapshot of the inherited legacy workspace settings, while a new publication uses -the live database-backed workspace provider. Integrations with an explicit provider -override remain self-contained and do not need to be republished for this reason. - -This command imports Baserow's legacy AI provider configuration; it does not -import `BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL` or the provider-native credentials +AI Fields, AI Agent actions, or any combination of them, then select the +Kuma model in the **AI features** section. Availability permits a feature to choose +a model; it does not force AI Fields or AI Agent actions to use Kuma's model. Use +**Test model** to check every selected feature. AI Fields and AI Agent actions check +for a text response, while Kuma also checks tool calling. + +For an existing installation, see the +[AI provider upgrade and import instructions](../development/feature-flags.md#preparing-the-ai-providers-feature). +Schema migrations run during the normal upgrade. Provider imports and republishing +are needed when adopting database-backed settings, not just to upgrade with the +feature disabled. Integrations with explicit provider overrides retain their own +connection settings; check the compatibility notes for model lists, partial overrides, +and optional endpoints. Review pending draft changes before republishing a site or +workflow, since those changes will also become live. + +The `migrate_ai_provider_settings` command imports legacy AI provider configuration; +it does not import `BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL` or the provider-native credentials used by Kuma. The assistant therefore stays on its legacy fallback until an administrator configures the same provider connection in the database, marks and tests a model for Kuma, and explicitly selects it. A database selection is diff --git a/docs/testing/kuma-model-settings-test-plan.md b/docs/testing/kuma-model-settings-test-plan.md index 0595eea36a..f585060a00 100644 --- a/docs/testing/kuma-model-settings-test-plan.md +++ b/docs/testing/kuma-model-settings-test-plan.md @@ -48,8 +48,9 @@ label was reworded since, treat the wording as indicative and the behaviour as binding. Only features that need one default model appear in the **AI features** section. -Today that is **Kuma** only — AI fields pick their model per field, so the -`ai_fields` checkbox controls *eligibility*, not a default. +Today that is **Kuma** only — AI fields and AI Agent services used by Automation +nodes and Application Builder actions pick their models per consumer, so their +checkboxes control *eligibility*, not a default. ### Helper: what the backend actually resolves @@ -87,7 +88,7 @@ EOF Run it after every settings change: ```bash -WORKSPACE_IDS=1,2 just dc-dev exec -T backend -e WORKSPACE_IDS \ +WORKSPACE_IDS=1,2 just dc-dev exec -T -e WORKSPACE_IDS backend \ /baserow/venv/bin/python /baserow/backend/src/baserow/manage.py shell < /tmp/kuma_state.py ``` @@ -136,15 +137,16 @@ refused instead — see 4.5. 1. Sign in as **admin**, go to Admin → **AI providers**. 2. **Add provider** → pick a provider type, enter the API key. 3. In the model rows, enter a model identifier and note the new **Available for** - checkboxes (**AI fields**, **Kuma**), both ticked by default. -4. Add a second model, untick **Kuma** on it, save. -5. Add a third model through the provider menu → **Add model**, and untick both boxes. + checkboxes (**AI agent**, **AI fields**, **Kuma**), all ticked by default. +4. Add a second model, untick **AI agent** and **Kuma** on it, save. +5. Add a third model through the provider menu → **Add model**, and untick all three + boxes. Verify: - The provider is created with all three models. -- Each model row shows `Available for: AI fields, Kuma` and `Available for: AI fields`. - The order follows the feature registry (premium before enterprise), not the order - you ticked the boxes. +- The first two model rows show all three feature labels and **AI fields** only, + respectively. Feature order is not semantically meaningful and can reflect how the + row was created. - A model with no boxes ticked saves, and its row shows `Not available to any listed feature`. @@ -163,7 +165,7 @@ Verify: `POST /api/ai-providers/models/test/` (its `feature_results` lists one entry per selected feature) or `AIProviderModel.last_test_capabilities`, which holds a `text` and a `tools` key for a Kuma-eligible model and only `text` otherwise. -- A model without tool support passes as an AI fields model, and shows +- A model without tool support passes as an AI fields or AI Agent model, and shows **Some tests failed** when it is also marked for Kuma — hover the badge for the per-feature breakdown. - With a bogus identifier the row shows **Test failed** and the tooltip lists a @@ -175,8 +177,8 @@ This branch adds **Google Gemini** and **Groq** as provider types. Verify: - Both appear in the **Add provider** type list and accept an API key. -- Their models can be marked for Kuma and AI fields, tested, and selected like any - other provider's. +- Their models can be marked for Kuma, AI fields, and AI Agent services, tested, and + selected like any other provider's. - They are configurable through the admin/workspace AI providers UI only. The legacy endpoint is `PATCH` (a `PUT` returns `405`), and a `google` or `groq` key is rejected with `400 ERROR_REQUEST_BODY_VALIDATION`, *"Your request body had the @@ -205,7 +207,9 @@ Verify: `{"feature_types": ["not-a-feature"]}` returns `400 ERROR_AI_PROVIDER_MODEL_FEATURE_TYPE_DOES_NOT_EXIST`. - Omitting `feature_types` entirely is the back-compat path for older API callers: the - model is created with `["ai_fields"]` only, **not** with every registered feature. + model is created with `["ai_fields", "ai_agent"]`, the two per-consumer features + that used provider models before eligibility was introduced, **not** with every + registered feature. - `{"feature_types": []}` is accepted, and the resulting model is offered by no feature (see 6.1). @@ -388,9 +392,9 @@ survive the confirm dialog. ### 4.2 Removing the Kuma checkbox from the selected model Verify: same rejection, under the title *"The AI provider model could not be -saved."*; the modal stays open and the row keeps **Available for: AI fields, Kuma**. -Removing only the **AI fields** checkbox is allowed, because no default-model -feature depends on it. +saved."*; the modal stays open and the row keeps all three feature labels. +Removing only the **AI agent** or **AI fields** checkbox is allowed, because +neither is a default-model feature. ### 4.3 Disabling the selected model or its provider @@ -494,14 +498,18 @@ and that Kuma there also falls back rather than switching off. --- -## 6. AI fields are a separate feature +## 6. Per-consumer features are separate ### 6.1 Eligibility is per feature 1. Create an AI field in a table of the workspace. +2. Add an **AI prompt** node in Automation and an **AI prompt** action in Application + Builder, using AI integrations without provider overrides. Verify: - Models with **AI fields** unticked are not offered in the field's model dropdown. +- Models with **AI agent** unticked are not offered as new choices in either + **AI prompt** model dropdown. - Models with only **AI fields** ticked are not offered in the Kuma **AI features** dropdown. - A model with no feature ticked appears nowhere. @@ -513,8 +521,9 @@ curl -s http://localhost:8000/api/workspaces/ -H "Authorization: JWT " \ | python3 -c "import json,sys; [print(w['id'], json.dumps(w['ai_features'])) for w in json.load(sys.stdin)]" ``` -`ai_features.ai_fields.models` must exclude every Kuma-only model, and a model with no -feature ticked must be absent from both feature lists. The legacy +`ai_features.ai_fields.models` must exclude every Kuma-only model, +`ai_features.ai_agent.models` must exclude every model without AI Agent eligibility, +and a model with no feature ticked must be absent from all feature lists. The legacy `generative_ai_models_enabled` field still lists **all** of them, including the no-feature model — which is exactly why the frontend must read `ai_features` while the flag is on, and why section 10's legacy path still offers those models. @@ -544,6 +553,40 @@ Verify: among the options. That is today's behaviour, not a defect to file — but it means a tester must not expect an explicit "model unavailable" message here. +### 6.4 Removing AI Agent eligibility from a model in use + +1. Select a working model in both **AI prompt** consumers from 6.1 and run them. +2. Untick **AI agent** on that model while keeping the editor tabs open. + +Verify: + +- The change is allowed. The saved provider and model identifiers remain visible, + with an unavailable-model error, and the unavailable model cannot be selected anew. +- Editing the prompt preserves the saved selection. The node/action reports a + configuration error, and backend execution refuses the model before calling its + provider. +- Restoring eligibility clears the error and allows execution without reselecting + the model. Repeat with model disable/enable and provider disable/enable; use a + provider without a Kuma default so its protection does not block this check. + +### 6.5 Explicit integration overrides + +Use API-created AI integrations to test these built-in provider overrides in both +feature-flag states, in Automation and Application Builder: + +| Override | Expected connection and model list | +|---|---| +| Complete connection with `models: ["custom-model"]` | Own connection and explicit list, independent of database eligibility | +| Complete connection without `models` | Own connection and inherited allowed list, filtered for AI Agent while the flag is on | +| Complete connection with `models: []` | No available models | +| Incomplete connection with a model list | Inherited connection; only models also present in the inherited allowed list | +| Incomplete connection without `models` | Inherited connection and allowed list | + +For a complete connection, verify that omitted optional settings such as OpenAI's +`base_url` and `organization` are not inherited. For an incomplete connection, verify +that its partial credentials or endpoint never reach the provider. An attempted model +outside the effective list must fail during selection validation and execution. + --- ## 7. Realtime propagation @@ -681,6 +724,43 @@ and verify nothing from this feature leaks into the old path: uses it; stored feature settings are ignored (the helper prints `source=legacy`). - AI fields offer the env-var and legacy workspace models, unfiltered by `feature_types`. +- Automation AI Agent nodes and Application Builder AI Agent actions likewise use + the legacy integration/workspace/environment model lists without feature filtering. + A complete provider connection with no `models` key inherits that model list; + explicit `models: []` remains empty. Partial overrides cannot introduce models + outside the inherited list or supply connection settings from another scope. + +--- + +## 11. Transition from legacy settings to database providers + +Rehearse on a disposable installation using the +[upgrade and import sequence](../development/feature-flags.md#preparing-the-ai-providers-feature). +Start with the flag off, working instance environment settings, a different workspace +connection, and inherited **AI prompt** consumers in both Automation and Application +Builder. Include a publication created while the flag is off. + +Verify: + +- Upgrading while the flag stays off preserves execution without imports or + republishing, subject to the explicit-override compatibility rules in 6.5. +- Migration `core.0120` adds `ai_agent` once to existing provider models, including + models with an empty feature list, preserving other features and enabled states. +- Instance and workspace import previews do not write. Applying each scope creates + missing providers with `ai_fields` and `ai_agent` eligibility; repeating the import + leaves existing configurations unchanged. +- After enabling the flag and reloading editors, saved selections resolve with their + expected instance or workspace credentials. An enabled workspace model overrides + a matching instance model; other instance models remain inherited. A disabled + workspace model suppresses that identifier; disabling its provider reveals the + inherited instance layer. +- A publication containing a complete legacy snapshot keeps using that snapshot + until republished. Review its draft first, then republish and verify that it follows + live workspace credential and eligibility changes. An explicit complete integration + override remains independent. +- Before turning the flag off again, verify equivalent legacy settings exist. With + the schema retained, flag-off execution returns to those settings in drafts and + publications; database eligibility changes no longer filter the legacy model list. --- diff --git a/web-frontend/modules/automation/components/AutomationHeader.vue b/web-frontend/modules/automation/components/AutomationHeader.vue index b9b4d37127..856c1ced34 100644 --- a/web-frontend/modules/automation/components/AutomationHeader.vue +++ b/web-frontend/modules/automation/components/AutomationHeader.vue @@ -209,6 +209,7 @@ export default defineComponent({ const isInError = nodeType.isInError({ service: node.service, workspace: workspace.value, + application: props.automation, }) return nodeType.isWorkflowAction === true && !isInError }) diff --git a/web-frontend/modules/automation/components/form/SimulateDispatchNodeForm.vue b/web-frontend/modules/automation/components/form/SimulateDispatchNodeForm.vue index 7ba36e2151..37785cdc7e 100644 --- a/web-frontend/modules/automation/components/form/SimulateDispatchNodeForm.vue +++ b/web-frontend/modules/automation/components/form/SimulateDispatchNodeForm.vue @@ -116,6 +116,7 @@ const cantBeTestedReason = computed(() => { nodeType.value.isInError({ service: props.node.service, workspace: workspace.value, + application: automation.value, }) ) { return $i18n.t('simulateDispatch.errorNodeNotConfigured') @@ -135,6 +136,7 @@ const cantBeTestedReason = computed(() => { previousNodeType.isInError({ service: previousNode.service, workspace: workspace.value, + application: automation.value, }) ) { return $i18n.t('simulateDispatch.errorPreviousNodeNotConfigured', { diff --git a/web-frontend/modules/automation/components/workflow/WorkflowNodeContent.vue b/web-frontend/modules/automation/components/workflow/WorkflowNodeContent.vue index 38961db223..2f80233d15 100644 --- a/web-frontend/modules/automation/components/workflow/WorkflowNodeContent.vue +++ b/web-frontend/modules/automation/components/workflow/WorkflowNodeContent.vue @@ -268,6 +268,7 @@ const isInError = computed(() => { return nodeType.value.isInError({ service: props.node.service, workspace: workspace.value, + application: automation.value, }) }) @@ -281,6 +282,7 @@ const errorMessage = computed(() => { service: props.node.service, node: props.node, workspace: workspace.value, + application: automation.value, }) }) diff --git a/web-frontend/modules/automation/nodeTypes.js b/web-frontend/modules/automation/nodeTypes.js index 50f529dcf4..1181459d5f 100644 --- a/web-frontend/modules/automation/nodeTypes.js +++ b/web-frontend/modules/automation/nodeTypes.js @@ -225,31 +225,37 @@ export class NodeType extends Registerable { * Returns whether the node is in-error or not. * By default, this is derived from the service type's `isInError` * method, but can be overridden by the node type. + * @param {object} context The node's service and application context. + * @param {object} context.service The service of the node. + * @param {object|null} [context.workspace=null] The owning workspace. + * @param {object|null} [context.application=null] The owning automation. * @returns {boolean} - Whether the properties are in-error. */ - isInError({ service, workspace = null }) { + isInError({ service, workspace = null, application = null }) { if (workspace && this.isDeactivated({ workspace })) { return true } - return this.serviceType.isInError({ service }) + return this.serviceType.isInError({ service, workspace, application }) } /** * Returns the error message we should show when the node is in-error. * By default, this is derived from the service type's `getErrorMessage` * method, but can be overridden by the node type. - * @param {object} service - The service of the node. - * @param {object} node - The node for which the - * error message is being retrieved. - * @returns {string} - The error message. + * @param {object} context The node and its service and application context. + * @param {object} context.service The service of the node. + * @param {object} context.node The node whose error is being retrieved. + * @param {object|null} [context.workspace=null] The owning workspace. + * @param {object|null} [context.application=null] The owning automation. + * @returns {string|null} The error message, or null when valid. */ - getErrorMessage({ service, node, workspace = null }) { + getErrorMessage({ service, node, workspace = null, application = null }) { const deactivatedReason = workspace && this.isDeactivatedReason({ workspace }) if (deactivatedReason) { return deactivatedReason } - return this.serviceType.getErrorMessage({ service }) + return this.serviceType.getErrorMessage({ service, workspace, application }) } /** diff --git a/web-frontend/modules/builder/workflowActionTypes.js b/web-frontend/modules/builder/workflowActionTypes.js index 5e09a1568e..7128773528 100644 --- a/web-frontend/modules/builder/workflowActionTypes.js +++ b/web-frontend/modules/builder/workflowActionTypes.js @@ -361,18 +361,22 @@ export class WorkflowActionServiceType extends WorkflowActionType { return null } + /** + * Validate the service using workspace and integration configuration in the + * editor. Preview and public pages do not load integration settings. + * + * @param {object} workflowAction The workflow action with its service. + * @param {object} applicationContext The owning workspace, builder, and mode. + * @returns {string|null} The first service or action configuration error. + */ getErrorMessage(workflowAction, applicationContext) { + const isEditing = applicationContext?.mode === 'editing' const serviceError = this.serviceType.getErrorMessage({ service: workflowAction.service, - // Pass the builder so the service type can resolve the service's integration - // and flag the action as in-error when that integration has been trashed. - // Editor only: integrations are never loaded in preview/public mode, so - // there the check would flag every configured action as misconfigured and - // hide its element. - application: - applicationContext.mode === 'editing' - ? applicationContext.builder - : undefined, + // Outside the editor, missing integration overrides must not turn a valid + // action into a configuration error and hide its element. + workspace: isEditing ? applicationContext.workspace : undefined, + application: isEditing ? applicationContext.builder : undefined, }) if (serviceError) { diff --git a/web-frontend/modules/core/generativeAIModelTypes.js b/web-frontend/modules/core/generativeAIModelTypes.js index 1aa20a8871..c45c40de77 100644 --- a/web-frontend/modules/core/generativeAIModelTypes.js +++ b/web-frontend/modules/core/generativeAIModelTypes.js @@ -51,11 +51,61 @@ export class GenerativeAIModelType extends Registerable { return this.getSettings().find((setting) => setting.key === key) || null } + /** + * Settings which must be present for an integration override to own the + * provider connection instead of inheriting it from the workspace. + * + * @returns {string[]} The required connection setting keys. + */ + getRequiredIntegrationSettings() { + return this.getSetting('api_key') ? ['api_key'] : [] + } + + /** + * Whether an integration settings object contains its own complete + * connection. Partial objects may narrow model availability, but must never + * be combined with credentials inherited from another scope. + * + * @param {object|null} settings The integration's settings for this provider. + * @returns {boolean} Whether every required connection setting is nonempty. + */ + isIntegrationSettingsComplete(settings) { + if (!settings || typeof settings !== 'object') { + return false + } + return this.getRequiredIntegrationSettings().every((key) => { + const value = settings[key] + return typeof value === 'string' ? value.trim() !== '' : Boolean(value) + }) + } + + /** + * Whether the backend owns this provider's settings contract in + * `AI_PROVIDER_TYPES`, which lets an integration override that omits `models` + * inherit the workspace allowlist. + * + * @returns {boolean} Whether Baserow ships this provider type itself. + */ + isBuiltInProviderType() { + return false + } + getModelIdentifierDescription() { return null } } +/** + * Base class for the provider types Baserow ships and the backend knows in + * `AI_PROVIDER_TYPES`. Plugin providers extend `GenerativeAIModelType` directly + * and keep their own authoritative integration model list. + */ +export class BuiltInGenerativeAIModelType extends GenerativeAIModelType { + isBuiltInProviderType() { + return true + } +} + const modelSettings = (label, description) => ({ key: 'models', label, @@ -71,7 +121,7 @@ const modelSettings = (label, description) => ({ }, }) -export class OpenAIModelType extends GenerativeAIModelType { +export class OpenAIModelType extends BuiltInGenerativeAIModelType { static getType() { return 'openai' } @@ -124,7 +174,7 @@ export class OpenAIModelType extends GenerativeAIModelType { } } -export class AnthropicModelType extends GenerativeAIModelType { +export class AnthropicModelType extends BuiltInGenerativeAIModelType { static getType() { return 'anthropic' } @@ -168,7 +218,7 @@ export class AnthropicModelType extends GenerativeAIModelType { } } -export class MistralModelType extends GenerativeAIModelType { +export class MistralModelType extends BuiltInGenerativeAIModelType { static getType() { return 'mistral' } @@ -212,7 +262,7 @@ export class MistralModelType extends GenerativeAIModelType { } } -export class OllamaModelType extends GenerativeAIModelType { +export class OllamaModelType extends BuiltInGenerativeAIModelType { static getType() { return 'ollama' } @@ -237,6 +287,13 @@ export class OllamaModelType extends GenerativeAIModelType { ] } + /** + * @returns {string[]} Ollama requires its own host to override the connection. + */ + getRequiredIntegrationSettings() { + return ['host'] + } + getModelIdentifierDescription() { return this.app.$i18n.t( 'generativeAIModelType.ollamaModelIdentifierDescription' @@ -256,7 +313,7 @@ export class OllamaModelType extends GenerativeAIModelType { } } -export class OpenRouterModelType extends GenerativeAIModelType { +export class OpenRouterModelType extends BuiltInGenerativeAIModelType { static getType() { return 'openrouter' } @@ -303,7 +360,7 @@ export class OpenRouterModelType extends GenerativeAIModelType { } } -export class GoogleModelType extends GenerativeAIModelType { +export class GoogleModelType extends BuiltInGenerativeAIModelType { static getType() { return 'google' } @@ -347,7 +404,7 @@ export class GoogleModelType extends GenerativeAIModelType { } } -export class GroqModelType extends GenerativeAIModelType { +export class GroqModelType extends BuiltInGenerativeAIModelType { static getType() { return 'groq' } diff --git a/web-frontend/modules/integrations/ai/aiProviderModelFeatureTypes.js b/web-frontend/modules/integrations/ai/aiProviderModelFeatureTypes.js new file mode 100644 index 0000000000..cb470697f8 --- /dev/null +++ b/web-frontend/modules/integrations/ai/aiProviderModelFeatureTypes.js @@ -0,0 +1,24 @@ +import { AIProviderModelFeatureType } from '@baserow/modules/core/aiProviderModelFeatureTypes' + +export class AIAgentAIProviderModelFeatureType extends AIProviderModelFeatureType { + /** + * @returns {string} The eligibility identifier shared with the backend. + */ + static getType() { + return 'ai_agent' + } + + /** + * @returns {string} The localized feature name used in provider settings. + */ + getName() { + return this.$t('aiProviderModelFeature.aiAgent') + } + + /** + * @returns {string} The localized description of AI Agent eligibility. + */ + getDescription() { + return this.$t('aiProviderModelFeature.aiAgentDescription') + } +} diff --git a/web-frontend/modules/integrations/ai/components/services/AIAgentServiceForm.vue b/web-frontend/modules/integrations/ai/components/services/AIAgentServiceForm.vue index f3d31ffa96..cf6a89104b 100644 --- a/web-frontend/modules/integrations/ai/components/services/AIAgentServiceForm.vue +++ b/web-frontend/modules/integrations/ai/components/services/AIAgentServiceForm.vue @@ -38,6 +38,7 @@ v-if="values.ai_generative_ai_type" small-label :label="$t('aiAgentServiceForm.modelLabel')" + :error="selectedModelUnavailable" required class="margin-bottom-2" > @@ -50,9 +51,17 @@ :key="model" :name="model" :value="model" + :disabled=" + selectedModelUnavailable && model === values.ai_generative_ai_model + " > + } Workspace models available to AI Agent + * under the active feature flag, grouped by provider type. + */ + workspaceEnabledModels() { + return getEnabledModelsForAIProviderFeature( + this.workspace, + AIAgentAIProviderModelFeatureType.getType(), + this.aiProvidersEnabled + ) + }, + /** + * @returns {Array<{type: string, name: string}>} Installed providers with + * effective models, excluding any unavailable saved selection. + */ + baseAvailableProviders() { if (!this.integration) { return [] } - const workspaceEnabled = - this.workspace?.generative_ai_models_enabled || {} + const workspaceEnabled = this.workspaceEnabledModels const integrationSettings = this.integration.ai_settings || {} const allProviders = this.$registry.getAll('generativeAIModel') return Object.keys(allProviders) - .filter((type) => { - // Provider is available if it's configured at any level: - // 1. Has env vars (checked by backend when getting enabled models) - // 2. Set on workspace level - // 3. Set on integration level - - // Check if provider has models in integration settings - if (integrationSettings[type]) { - const models = integrationSettings[type].models || [] - if (models.length > 0) { - return true - } - } - - // Check if provider has models in workspace settings - if (workspaceEnabled[type] && workspaceEnabled[type].length > 0) { - return true - } - - return false - }) + .filter( + (type) => + getEffectiveAIAgentModels({ + workspaceModels: workspaceEnabled[type] || [], + integrationSettings: integrationSettings[type], + modelType: allProviders[type], + }).length > 0 + ) .map((type) => { const modelType = this.$registry.get('generativeAIModel', type) return { @@ -253,33 +273,79 @@ export default { } }) }, - availableModels() { + /** + * @returns {Array<{type: string, name: string}>} Provider options, retaining + * an unavailable saved provider for diagnosis when eligibility is enforced. + */ + availableProviders() { + const providers = [...this.baseAvailableProviders] + + const current = this.values.ai_generative_ai_type + if ( + this.aiProvidersEnabled && + current && + !providers.some((provider) => provider.type === current) + ) { + const allProviders = this.$registry.getAll('generativeAIModel') + const modelType = allProviders[current] + providers.push({ + type: current, + name: modelType ? modelType.getName() : current, + }) + } + return providers + }, + /** + * @returns {string[]} Selectable models for the integration and provider, + * excluding any unavailable saved model. + */ + baseAvailableModels() { if (!this.integration || !this.values.ai_generative_ai_type) { return [] } const integrationSettings = this.integration.ai_settings || {} - const workspaceEnabled = - this.workspace?.generative_ai_models_enabled || {} + const providerType = this.values.ai_generative_ai_type + const modelType = + this.$registry.getAll('generativeAIModel')[providerType] || null + return getEffectiveAIAgentModels({ + workspaceModels: this.workspaceEnabledModels[providerType] || [], + integrationSettings: integrationSettings[providerType], + modelType, + }) + }, + /** + * @returns {string[]} Model options, retaining an unavailable saved model + * for diagnosis when eligibility is enforced. + */ + availableModels() { + const models = this.baseAvailableModels - // If provider is overridden in integration, use integration models. - if (integrationSettings[this.values.ai_generative_ai_type]) { - return ( - integrationSettings[this.values.ai_generative_ai_type].models || [] - ) + const current = this.values.ai_generative_ai_model + if (this.aiProvidersEnabled && current && !models.includes(current)) { + return [...models, current] } - - // Otherwise use workspace models - return workspaceEnabled[this.values.ai_generative_ai_type] || [] + return models + }, + /** + * @returns {boolean} Whether eligibility prevents using the saved model. + */ + selectedModelUnavailable() { + const current = this.values.ai_generative_ai_model + return Boolean( + this.aiProvidersEnabled && + current && + !this.baseAvailableModels.includes(current) + ) }, maxTemperature() { if (!this.values.ai_generative_ai_type) { return 2 } - const modelType = this.$registry.get( - 'generativeAIModel', - this.values.ai_generative_ai_type - ) + const modelType = + this.$registry.getAll('generativeAIModel')[ + this.values.ai_generative_ai_type + ] || null return modelType ? modelType.getMaxTemperature() : 2 }, outputTypeOptions() { @@ -303,7 +369,7 @@ export default { 'values.integration_id'(newValue, oldValue) { if (oldValue && newValue !== oldValue) { // Check if current provider is still available - const availableProviderTypes = this.availableProviders.map( + const availableProviderTypes = this.baseAvailableProviders.map( (p) => p.type ) if ( @@ -315,7 +381,7 @@ export default { this.values.ai_generative_ai_model = null } else if (this.values.ai_generative_ai_type) { // Provider still available, check if model is still available - const models = this.availableModels + const models = this.baseAvailableModels if ( this.values.ai_generative_ai_model && !models.includes(this.values.ai_generative_ai_model) @@ -333,7 +399,7 @@ export default { */ 'values.ai_generative_ai_type'(newValue, oldValue) { if (oldValue && newValue !== oldValue) { - const models = this.availableModels + const models = this.baseAvailableModels this.values.ai_generative_ai_model = models.length > 0 ? models[0] : null } diff --git a/web-frontend/modules/integrations/ai/serviceTypes.js b/web-frontend/modules/integrations/ai/serviceTypes.js index 916fe237e3..36d8ecc4c7 100644 --- a/web-frontend/modules/integrations/ai/serviceTypes.js +++ b/web-frontend/modules/integrations/ai/serviceTypes.js @@ -4,6 +4,10 @@ import { } from '@baserow/modules/core/serviceTypes' import { AIIntegrationType } from '@baserow/modules/integrations/ai/integrationTypes' import AIAgentServiceForm from '@baserow/modules/integrations/ai/components/services/AIAgentServiceForm' +import { getEnabledModelsForAIProviderFeature } from '@baserow/modules/core/aiProviderModelFeatureTypes' +import { AIAgentAIProviderModelFeatureType } from '@baserow/modules/integrations/ai/aiProviderModelFeatureTypes' +import { FF_AI_PROVIDERS } from '@baserow/modules/core/plugins/featureFlags' +import { getEffectiveAIAgentModels } from '@baserow/modules/integrations/ai/utils' export class AIAgentServiceType extends WorkflowActionServiceTypeMixin( ServiceType @@ -36,7 +40,64 @@ export class AIAgentServiceType extends WorkflowActionServiceTypeMixin( return service.schema } - getErrorMessage({ service }) { + /** + * Resolve the selected provider's models for service validation while the + * AI providers feature flag is enabled. + * + * @param {object} context The service and its owning application context. + * @param {object} context.service The AI Agent service to validate. + * @param {object|null} context.workspace The workspace with model availability. + * @param {object|null} context.application The application used to look up the + * service's integration, when available. + * @returns {string[]|null} Available models, or null when the workspace or + * integration has not loaded and availability cannot yet be checked. + */ + getEffectiveModels({ service, workspace, application }) { + if (!workspace) { + return null + } + + const providerType = service.ai_generative_ai_type + const modelType = + this.app.$registry.getAll('generativeAIModel')[providerType] || null + const workspaceModels = getEnabledModelsForAIProviderFeature( + workspace, + AIAgentAIProviderModelFeatureType.getType(), + true + )[providerType] + + let integrationSettings = null + if (application && service.integration_id) { + const integration = this.app.$store.getters[ + 'integration/getIntegrationById' + ](application, service.integration_id) + // Avoid reporting a false configuration error while the application's + // integrations are still loading. + if (!integration) { + return null + } + integrationSettings = integration.ai_settings?.[providerType] + } + + return getEffectiveAIAgentModels({ + workspaceModels: workspaceModels || [], + integrationSettings, + modelType, + }) + } + + /** + * Validate configuration and, when enabled, AI Agent model availability. + * + * @param {object} context The service and its owning application context. + * @param {object|undefined} context.service The service, possibly redacted on + * a public page or absent before configuration. + * @param {object|null} [context.workspace=null] The owning workspace. + * @param {object|null} [context.application=null] The owning application. + * @returns {string|null} The first configuration error, or null when valid or + * when the relevant configuration is not available to the client. + */ + getErrorMessage({ service, workspace = null, application = null }) { if (service === undefined) { return null } @@ -52,6 +113,19 @@ export class AIAgentServiceType extends WorkflowActionServiceTypeMixin( if (!service.ai_generative_ai_model) { return this.app.$i18n.t('serviceType.errorNoAIModelSelected') } + if (this.app.$featureFlagIsEnabled(FF_AI_PROVIDERS)) { + const effectiveModels = this.getEffectiveModels({ + service, + workspace, + application, + }) + if ( + effectiveModels !== null && + !effectiveModels.includes(service.ai_generative_ai_model) + ) { + return this.app.$i18n.t('serviceType.errorAIModelUnavailable') + } + } if (!service.ai_prompt.formula) { return this.app.$i18n.t('serviceType.errorNoPromptProvided') } @@ -66,9 +140,17 @@ export class AIAgentServiceType extends WorkflowActionServiceTypeMixin( return this.app.$i18n.t('serviceType.errorNoChoicesProvided') } } - return super.getErrorMessage({ service }) + return super.getErrorMessage({ service, workspace, application }) } + /** + * Describe the selected provider and model, including configuration errors. + * + * @param {object} service The AI Agent service to describe. + * @param {object|null} application The application used to resolve the + * workspace and integration, when available. + * @returns {string} The model selection and its first validation error. + */ getDescription(service, application) { let description = this.name @@ -76,8 +158,13 @@ export class AIAgentServiceType extends WorkflowActionServiceTypeMixin( description += ` - ${service.ai_generative_ai_model}` } - if (this.isInError({ service })) { - description += ` - ${this.getErrorMessage({ service })}` + const workspaceId = application?.workspace?.id ?? application?.workspace + const workspace = workspaceId + ? this.app.$store.getters['workspace/get'](workspaceId) + : null + const validationContext = { service, workspace, application } + if (this.isInError(validationContext)) { + description += ` - ${this.getErrorMessage(validationContext)}` } return description diff --git a/web-frontend/modules/integrations/ai/utils.js b/web-frontend/modules/integrations/ai/utils.js new file mode 100644 index 0000000000..c76562273b --- /dev/null +++ b/web-frontend/modules/integrations/ai/utils.js @@ -0,0 +1,54 @@ +/** + * Resolve the AI Agent models shown by the client using the same precedence as + * the backend. + * + * A complete integration override owns its connection. Built-in provider types + * inherit the model allowlist when their override omits models; an explicit + * list, including an empty one, remains authoritative. A partial override can + * only narrow the feature-filtered workspace list. + * + * @param {object} options The inputs for one provider's model resolution. + * @param {string[]} [options.workspaceModels=[]] Effective workspace models, + * already filtered for AI Agent eligibility when the provider flag is enabled. + * @param {object|null} [options.integrationSettings=null] Provider settings from + * the selected integration. + * @param {GenerativeAIModelType|null} [options.modelType=null] The registered + * provider type, or null when its extension is no longer installed. + * @returns {string[]} The available model identifiers. Treat the returned list + * as read-only because it can be the original workspace or integration list. + */ +export function getEffectiveAIAgentModels({ + workspaceModels = [], + integrationSettings = null, + modelType = null, +}) { + if (!modelType) { + return [] + } + + if (!integrationSettings || typeof integrationSettings !== 'object') { + return workspaceModels + } + + const integrationModels = Array.isArray(integrationSettings.models) + ? integrationSettings.models + : [] + const hasModels = Object.prototype.hasOwnProperty.call( + integrationSettings, + 'models' + ) + + if (modelType.isIntegrationSettingsComplete(integrationSettings)) { + if (!hasModels && modelType.isBuiltInProviderType()) { + return workspaceModels + } + return integrationModels + } + + if (!hasModels) { + return workspaceModels + } + + const enabledModels = new Set(workspaceModels) + return integrationModels.filter((model) => enabledModels.has(model)) +} diff --git a/web-frontend/modules/integrations/locales/en.json b/web-frontend/modules/integrations/locales/en.json index fbe7b54b36..9bc7b3b2df 100644 --- a/web-frontend/modules/integrations/locales/en.json +++ b/web-frontend/modules/integrations/locales/en.json @@ -18,6 +18,10 @@ "inheritingWorkspace": "Inheriting workspace AI settings", "overridingProviders": "Overriding {count} provider|Overriding {count} providers" }, + "aiProviderModelFeature": { + "aiAgent": "AI agent", + "aiAgentDescription": "Run AI Agent actions in Automation and Application Builder workflows." + }, "slackBotIntegrationType": { "slackBotSummary": "Slack Bot", "slackBotNoToken": "Slack Bot - Not configured", @@ -95,6 +99,7 @@ "errorNoIntegrationSelected": "No integration selected", "errorNoAIProviderSelected": "No AI provider selected", "errorNoAIModelSelected": "No AI model selected", + "errorAIModelUnavailable": "The selected AI model is disabled or no longer available", "errorNoPromptProvided": "No prompt provided", "errorNoChoicesProvided": "No choices provided for choice output type", "slackWriteMessage": "Send a Slack message", @@ -113,7 +118,7 @@ "aiForm": { "description": "Configure AI provider settings for this integration. By default, workspace AI settings are inherited.", "workspaceSettingsTitle": "Workspace AI Settings", - "workspaceSettingsDescription": "This integration inherits AI provider settings from your workspace by default. You can override specific providers below.", + "workspaceSettingsDescription": "This integration inherits AI provider settings from your workspace by default. A provider override is self-contained: include its own credentials or host, because connection fields are never combined with inherited credentials.", "overrideWorkspaceSettings": "Override workspace settings for this provider", "inherited": "Inherited", "overridden": "Overridden" diff --git a/web-frontend/modules/integrations/plugin.js b/web-frontend/modules/integrations/plugin.js index d04ab95637..7ac5717786 100644 --- a/web-frontend/modules/integrations/plugin.js +++ b/web-frontend/modules/integrations/plugin.js @@ -32,6 +32,7 @@ import { import { AIAgentServiceType } from '@baserow/modules/integrations/ai/serviceTypes' import { SlackWriteMessageServiceType } from '@baserow/modules/integrations/slack/serviceTypes' import { SlackBotIntegrationType } from '@baserow/modules/integrations/slack/integrationTypes' +import { AIAgentAIProviderModelFeatureType } from '@baserow/modules/integrations/ai/aiProviderModelFeatureTypes' export default defineNuxtPlugin({ dependsOn: ['core'], @@ -45,6 +46,11 @@ export default defineNuxtPlugin({ $registry.register('integration', new AIIntegrationType(context)) $registry.register('integration', new SlackBotIntegrationType(context)) + $registry.register( + 'aiProviderModelFeature', + new AIAgentAIProviderModelFeatureType(context) + ) + $registry.register('service', new LocalBaserowGetRowServiceType(context)) $registry.register('service', new LocalBaserowListRowsServiceType(context)) $registry.register( diff --git a/web-frontend/test/unit/automation/nodeTypes.spec.js b/web-frontend/test/unit/automation/nodeTypes.spec.js index bec4525304..bd28e3bdd9 100644 --- a/web-frontend/test/unit/automation/nodeTypes.spec.js +++ b/web-frontend/test/unit/automation/nodeTypes.spec.js @@ -30,6 +30,46 @@ describe('NodeType.getHistoryLabel', () => { }) }) +describe('NodeType service validation context', () => { + test('forwards the workspace and application to the service type', () => { + const service = { id: 1 } + const workspace = { id: 2 } + const application = { id: 3 } + const serviceType = { + isDeactivatedReason: vi.fn(() => null), + isInError: vi.fn(() => true), + getErrorMessage: vi.fn(() => 'Unavailable model'), + } + + class ContextAwareNodeType extends NodeType { + static getType() { + return 'context-aware' + } + + get serviceType() { + return serviceType + } + } + + const nodeType = new ContextAwareNodeType({ app: {} }) + + expect(nodeType.isInError({ service, workspace, application })).toBe(true) + expect(nodeType.getErrorMessage({ service, workspace, application })).toBe( + 'Unavailable model' + ) + expect(serviceType.isInError).toHaveBeenCalledWith({ + service, + workspace, + application, + }) + expect(serviceType.getErrorMessage).toHaveBeenCalledWith({ + service, + workspace, + application, + }) + }) +}) + describe('CoreRouterNodeType.getHistoryLabel', () => { const makeApp = () => ({ $i18n: { diff --git a/web-frontend/test/unit/builder/workflowActionTypes.spec.js b/web-frontend/test/unit/builder/workflowActionTypes.spec.js index 10b4fd7dde..36d8b1538a 100644 --- a/web-frontend/test/unit/builder/workflowActionTypes.spec.js +++ b/web-frontend/test/unit/builder/workflowActionTypes.spec.js @@ -135,6 +135,85 @@ describe('Builder workflow action types', () => { expect(workflowActionType.getErrorMessage(workflowAction, {})).toBeNull() }) + test('validates AI Agent models using the builder integration override', () => { + const workflowActionType = testApp + .getRegistry() + .get('workflowAction', 'ai_agent') + const integration = { + id: 5, + type: 'ai', + ai_settings: { + openai: { api_key: 'integration-key', models: ['own-model'] }, + }, + } + const context = { + mode: 'editing', + builder: { id: 1, integrations: [integration] }, + workspace: { + id: 2, + ai_features: { ai_agent: { models: { openai: ['db-model'] } } }, + }, + } + const workflowAction = { + type: 'ai_agent', + service: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'own-model', + ai_prompt: { formula: 'Prompt' }, + ai_output_type: 'text', + }, + } + + expect( + workflowActionType.getErrorMessage(workflowAction, context) + ).toBeNull() + + integration.ai_settings = {} + + expect(workflowActionType.getErrorMessage(workflowAction, context)).toBe( + 'serviceType.errorAIModelUnavailable' + ) + + workflowAction.service.ai_generative_ai_model = 'db-model' + + expect( + workflowActionType.getErrorMessage(workflowAction, context) + ).toBeNull() + }) + + test.each(['preview', 'public'])( + 'does not reject an AI Agent integration model in %s without loaded overrides', + (mode) => { + const workflowActionType = testApp + .getRegistry() + .get('workflowAction', 'ai_agent') + const context = { + mode, + builder: { id: 1, integrations: [] }, + workspace: { + id: 2, + ai_features: { ai_agent: { models: { openai: ['db-model'] } } }, + }, + } + const workflowAction = { + type: 'ai_agent', + service: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'own-model', + ai_prompt: { formula: 'Prompt' }, + ai_output_type: 'text', + }, + } + + expect( + workflowActionType.getErrorMessage(workflowAction, context) + ).toBeNull() + expect(workflowActionType.isInError(workflowAction, context)).toBe(false) + } + ) + test('open page action is in error when saved page parameters are outdated', () => { const workflowActionType = testApp .getRegistry() diff --git a/web-frontend/test/unit/core/components/aiProviderFormModal.spec.js b/web-frontend/test/unit/core/components/aiProviderFormModal.spec.js index 4b7e029cf1..7b5c839c16 100644 --- a/web-frontend/test/unit/core/components/aiProviderFormModal.spec.js +++ b/web-frontend/test/unit/core/components/aiProviderFormModal.spec.js @@ -129,7 +129,7 @@ describe('AIProviderFormModal', () => { models: [ { model_identifier: 'gpt-5.6', - feature_types: ['ai_fields', 'kuma'], + feature_types: ['ai_agent', 'ai_fields', 'kuma'], }, ], }) diff --git a/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js b/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js index 53de68d069..d8854e92a3 100644 --- a/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js +++ b/web-frontend/test/unit/core/components/aiProviderModelFormModal.spec.js @@ -83,7 +83,7 @@ describe('AIProviderModelFormModal', () => { providerId: 1, values: { model_identifier: 'claude-sonnet-5', - feature_types: ['ai_fields', 'kuma'], + feature_types: ['ai_agent', 'ai_fields', 'kuma'], }, }) expect(dispatch).not.toHaveBeenCalledWith( @@ -159,7 +159,7 @@ describe('AIProviderModelFormModal', () => { providerId: 1, values: { model_identifier: 'custom-model', - feature_types: ['ai_fields', 'kuma'], + feature_types: ['ai_agent', 'ai_fields', 'kuma'], }, }) expect(dispatch).not.toHaveBeenCalledWith( diff --git a/web-frontend/test/unit/core/generativeAIModelTypes.spec.js b/web-frontend/test/unit/core/generativeAIModelTypes.spec.js index 2d4bbd78a0..47055efa36 100644 --- a/web-frontend/test/unit/core/generativeAIModelTypes.spec.js +++ b/web-frontend/test/unit/core/generativeAIModelTypes.spec.js @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from 'vitest' import { TestApp } from '@baserow/test/helpers/testApp' +import { GenerativeAIModelType } from '@baserow/modules/core/generativeAIModelTypes' import GenerativeAIWorkspaceSettings from '@baserow/modules/core/components/workspace/GenerativeAIWorkspaceSettings' describe('Generative AI model types', () => { @@ -32,6 +33,28 @@ describe('Generative AI model types', () => { ]) }) + test('marks every registered provider as a built-in provider type', () => { + const modelTypes = testApp.getRegistry().getOrderedList('generativeAIModel') + + expect( + modelTypes + .filter((modelType) => !modelType.isBuiltInProviderType()) + .map((modelType) => modelType.getType()) + ).toEqual([]) + }) + + test('does not treat a plugin provider as a built-in provider type', () => { + class ExtensionModelType extends GenerativeAIModelType { + static getType() { + return 'extension' + } + } + + expect( + new ExtensionModelType({ app: testApp.getApp() }).isBuiltInProviderType() + ).toBe(false) + }) + test('keeps database-only providers out of legacy workspace settings', () => { const registry = testApp.getRegistry() @@ -51,6 +74,34 @@ describe('Generative AI model types', () => { expect(registry.exists('generativeAIModel', 'groq')).toBe(true) }) + test('only treats self-contained integration connections as complete', () => { + const registry = testApp.getRegistry() + const openai = registry.get('generativeAIModel', 'openai') + const ollama = registry.get('generativeAIModel', 'ollama') + + expect( + openai.isIntegrationSettingsComplete({ + base_url: 'https://example.com/v1', + models: ['model'], + }) + ).toBe(false) + expect( + openai.isIntegrationSettingsComplete({ + api_key: 'secret', + models: ['model'], + }) + ).toBe(true) + expect(ollama.isIntegrationSettingsComplete({ models: ['model'] })).toBe( + false + ) + expect( + ollama.isIntegrationSettingsComplete({ + host: 'http://localhost:11434', + models: ['model'], + }) + ).toBe(true) + }) + test.each([ { providerType: 'google', diff --git a/web-frontend/test/unit/integrations/ai/aiAgentServiceForm.spec.js b/web-frontend/test/unit/integrations/ai/aiAgentServiceForm.spec.js new file mode 100644 index 0000000000..389affd3d0 --- /dev/null +++ b/web-frontend/test/unit/integrations/ai/aiAgentServiceForm.spec.js @@ -0,0 +1,570 @@ +import { defineComponent, reactive, ref, unref } from 'vue' +import { mountSuspended } from '@nuxt/test-utils/runtime' +import { enableAutoUnmount, flushPromises } from '@vue/test-utils' + +import AIAgentServiceForm from '@baserow/modules/integrations/ai/components/services/AIAgentServiceForm' +import { AIAgentServiceType } from '@baserow/modules/integrations/ai/serviceTypes' + +const FormGroupStub = defineComponent({ + name: 'FormGroup', + props: { + label: { type: String, default: null }, + error: { type: Boolean, default: false }, + }, + template: + '
', +}) +const DropdownStub = defineComponent({ + name: 'Dropdown', + props: { modelValue: { type: [String, Number], default: null } }, + emits: ['update:modelValue'], + template: + '', +}) +const DropdownItemStub = defineComponent({ + name: 'DropdownItem', + props: { + name: { type: String, required: true }, + value: { type: String, required: true }, + disabled: { type: Boolean, default: false }, + }, + template: + '', +}) +const IntegrationDropdownStub = defineComponent({ + name: 'IntegrationDropdown', + props: { + modelValue: { type: Number, default: null }, + integrations: { type: Array, required: true }, + }, + emits: ['update:modelValue'], + template: + '', +}) +const PassthroughStub = defineComponent({ template: '
' }) + +enableAutoUnmount(afterEach) + +const openAIModelType = { + getType: () => 'openai', + getName: () => 'OpenAI', + getMaxTemperature: () => 2, + isIntegrationSettingsComplete: (settings) => Boolean(settings.api_key), + isBuiltInProviderType: () => true, +} +const anthropicModelType = { + getType: () => 'anthropic', + getName: () => 'Anthropic', + getMaxTemperature: () => 1, + isIntegrationSettingsComplete: (settings) => Boolean(settings.api_key), + isBuiltInProviderType: () => true, +} +const modelTypes = { openai: openAIModelType, anthropic: anthropicModelType } + +const workspace = { + id: 1, + generative_ai_models_enabled: { openai: ['legacy-model'] }, + ai_features: { ai_agent: { models: { openai: ['db-model'] } } }, +} + +async function mountForm({ + featureFlagEnabled, + integration, + integrations = [integration], + defaultValues, + workspace: workspaceValue = workspace, +}) { + return await mountSuspended(AIAgentServiceForm, { + props: { + application: { id: 1, workspace: { id: 1 } }, + defaultValues, + }, + global: { + stubs: { + FormGroup: FormGroupStub, + Dropdown: DropdownStub, + DropdownItem: DropdownItemStub, + IntegrationDropdown: IntegrationDropdownStub, + InjectedFormulaInput: PassthroughStub, + RadioGroup: PassthroughStub, + FormInput: PassthroughStub, + Button: PassthroughStub, + }, + mocks: { + $t: (key) => key, + $featureFlagIsEnabled: () => unref(featureFlagEnabled), + $store: { + getters: { + 'integration/getIntegrations': () => integrations, + 'integration/getIntegrationById': (application, id) => + integrations.find((candidate) => candidate.id === id), + 'workspace/get': () => workspaceValue, + }, + }, + $registry: { + getAll: () => modelTypes, + get: (namespace, type) => { + if (namespace !== 'generativeAIModel') { + return {} + } + if (!modelTypes[type]) { + throw new Error(`Missing model type: ${type}`) + } + return modelTypes[type] + }, + }, + }, + }, + }) +} + +describe('AIAgentServiceForm', () => { + test('lists ai_agent feature models when the flag is enabled', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="db-model"]').exists()).toBe(true) + expect(wrapper.find('[data-value="legacy-model"]').exists()).toBe(false) + }) + + test('lists legacy workspace models when the flag is disabled', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: false, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="legacy-model"]').exists()).toBe(true) + expect(wrapper.find('[data-value="db-model"]').exists()).toBe(false) + }) + + test('preserves the selected model through flag activation and availability changes', async () => { + const featureFlagEnabled = ref(false) + const workspaceValue = reactive({ + ...workspace, + ai_features: { ai_agent: { models: { openai: ['db-model'] } } }, + }) + const wrapper = await mountForm({ + featureFlagEnabled, + workspace: workspaceValue, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'legacy-model', + }, + }) + await flushPromises() + + const modelField = wrapper.get( + '[data-label="aiAgentServiceForm.modelLabel"]' + ) + expect(modelField.get('select').element.value).toBe('legacy-model') + expect(modelField.attributes('data-error')).toBe('false') + + featureFlagEnabled.value = true + await flushPromises() + + expect(modelField.get('select').element.value).toBe('legacy-model') + expect(modelField.attributes('data-error')).toBe('true') + expect( + modelField.get('[data-value="legacy-model"]').attributes('aria-disabled') + ).toBe('true') + expect(modelField.get('[data-value="db-model"]').exists()).toBe(true) + + workspaceValue.ai_features.ai_agent.models.openai.push('legacy-model') + await flushPromises() + + expect(modelField.get('select').element.value).toBe('legacy-model') + expect(modelField.attributes('data-error')).toBe('false') + expect( + modelField.get('[data-value="legacy-model"]').attributes('aria-disabled') + ).toBe('false') + + featureFlagEnabled.value = false + await flushPromises() + + expect(modelField.get('select').element.value).toBe('legacy-model') + expect(modelField.find('[data-value="db-model"]').exists()).toBe(false) + expect(modelField.attributes('data-error')).toBe('false') + }) + + test('marks a selected model unavailable when its feature eligibility is removed', async () => { + const workspaceValue = reactive({ + ...workspace, + ai_features: { ai_agent: { models: { openai: ['db-model'] } } }, + }) + const wrapper = await mountForm({ + featureFlagEnabled: true, + workspace: workspaceValue, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'db-model', + }, + }) + await flushPromises() + + workspaceValue.ai_features.ai_agent.models = {} + await flushPromises() + + const modelField = wrapper.get( + '[data-label="aiAgentServiceForm.modelLabel"]' + ) + expect(modelField.get('select').element.value).toBe('db-model') + expect(modelField.attributes('data-error')).toBe('true') + expect(modelField.text()).toContain('selectAIModelForm.modelUnavailable') + expect(modelField.find('[data-value="legacy-model"]').exists()).toBe(false) + }) + + test('limits partial integration settings to legacy workspace models when the flag is disabled', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: false, + integration: { + id: 5, + type: 'ai', + ai_settings: { + openai: { models: ['integration-only-model', 'legacy-model'] }, + }, + }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="legacy-model"]').exists()).toBe(true) + expect(wrapper.find('[data-value="integration-only-model"]').exists()).toBe( + false + ) + }) + + test('complete integration settings override the workspace models', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { + id: 5, + type: 'ai', + ai_settings: { + openai: { api_key: 'integration-key', models: ['blob-model'] }, + }, + }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="blob-model"]').exists()).toBe(true) + expect(wrapper.find('[data-value="db-model"]').exists()).toBe(false) + }) + + test.each([ + { + featureFlagEnabled: true, + available: 'db-model', + excluded: 'legacy-model', + }, + { + featureFlagEnabled: false, + available: 'legacy-model', + excluded: 'db-model', + }, + ])( + 'inherits available models for an own-key override without models when flag is $featureFlagEnabled', + async ({ featureFlagEnabled, available, excluded }) => { + const wrapper = await mountForm({ + featureFlagEnabled, + integration: { + id: 5, + type: 'ai', + ai_settings: { openai: { api_key: 'integration-key' } }, + }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: available, + }, + }) + await flushPromises() + + expect( + wrapper.get(`[data-value="${available}"]`).attributes('aria-disabled') + ).toBe('false') + expect(wrapper.find(`[data-value="${excluded}"]`).exists()).toBe(false) + expect( + wrapper + .get('[data-label="aiAgentServiceForm.modelLabel"]') + .attributes('data-error') + ).toBe('false') + } + ) + + test.each([true, false])( + 'keeps an explicit empty own-key model list empty when flag is %s', + async (featureFlagEnabled) => { + const wrapper = await mountForm({ + featureFlagEnabled, + integration: { + id: 5, + type: 'ai', + ai_settings: { openai: { api_key: 'integration-key', models: [] } }, + }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="db-model"]').exists()).toBe(false) + expect(wrapper.find('[data-value="legacy-model"]').exists()).toBe(false) + } + ) + + test('limits partial integration model settings to workspace ai_agent models', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { + id: 5, + type: 'ai', + ai_settings: { + openai: { models: ['integration-only-model', 'db-model'] }, + }, + }, + defaultValues: { integration_id: 5, ai_generative_ai_type: 'openai' }, + }) + await flushPromises() + + expect(wrapper.find('[data-value="db-model"]').exists()).toBe(true) + expect(wrapper.find('[data-value="integration-only-model"]').exists()).toBe( + false + ) + }) + + test('keeps a stale model visible but marks it unavailable', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'gone-model', + }, + }) + await flushPromises() + + const staleOption = wrapper.get('[data-value="gone-model"]') + expect(staleOption.attributes('aria-disabled')).toBe('true') + expect(staleOption.text()).toContain('gone-model') + + const modelField = wrapper.get( + '[data-label="aiAgentServiceForm.modelLabel"]' + ) + expect(modelField.attributes('data-error')).toBe('true') + expect(modelField.text()).toContain('selectAIModelForm.modelUnavailable') + }) + + test('keeps a stale provider visible while editing', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { id: 5, type: 'ai', ai_settings: {} }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'anthropic', + ai_generative_ai_model: 'gone-model', + }, + }) + await flushPromises() + + expect(wrapper.get('[data-value="anthropic"]').text()).toBe('Anthropic') + }) + + test('marks a provider from an uninstalled extension as unavailable', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integration: { id: 5, type: 'ai', ai_settings: {} }, + workspace: { + ...workspace, + ai_features: { + ai_agent: { models: { 'removed-provider': ['removed-model'] } }, + }, + }, + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'removed-provider', + ai_generative_ai_model: 'removed-model', + }, + }) + await flushPromises() + + expect(wrapper.get('[data-value="removed-model"]').exists()).toBe(true) + expect( + wrapper + .get('[data-label="aiAgentServiceForm.modelLabel"]') + .attributes('data-error') + ).toBe('true') + }) + + test('re-validates the selection when switching integration with the flag enabled', async () => { + const wrapper = await mountForm({ + featureFlagEnabled: true, + integrations: [ + { + id: 5, + type: 'ai', + ai_settings: { + openai: { api_key: 'first-key', models: ['gpt-4'] }, + }, + }, + { + id: 6, + type: 'ai', + ai_settings: { + openai: { api_key: 'second-key', models: ['gpt-3.5'] }, + }, + }, + ], + defaultValues: { + integration_id: 5, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: 'gpt-4', + }, + }) + await flushPromises() + + await wrapper + .get('[data-label="aiAgentServiceForm.integrationLabel"] select') + .setValue('6') + await flushPromises() + + expect( + wrapper.get('[data-label="aiAgentServiceForm.modelLabel"] select').element + .value + ).toBe('gpt-3.5') + expect(wrapper.find('[data-value="gpt-4"]').exists()).toBe(false) + }) +}) + +describe('AIAgentServiceType', () => { + const makeServiceType = (integration = null, featureFlagEnabled = true) => + new AIAgentServiceType({ + app: { + $featureFlagIsEnabled: () => featureFlagEnabled, + $i18n: { t: (key) => key }, + $registry: { getAll: () => modelTypes }, + $store: { + getters: { + 'integration/getIntegrationById': () => integration, + }, + }, + }, + }) + + const makeService = (model, integrationId = null) => ({ + integration_id: integrationId, + ai_generative_ai_type: 'openai', + ai_generative_ai_model: model, + ai_prompt: { formula: 'Prompt' }, + ai_output_type: 'text', + }) + + test('reports a model unavailable to the workspace ai_agent feature', () => { + const serviceType = makeServiceType() + + expect( + serviceType.getErrorMessage({ + service: makeService('gone-model'), + workspace, + }) + ).toBe('serviceType.errorAIModelUnavailable') + expect( + serviceType.getErrorMessage({ + service: makeService('db-model'), + workspace, + }) + ).toBeNull() + }) + + test('does not enforce database model availability when the flag is disabled', () => { + const serviceType = makeServiceType(null, false) + + expect( + serviceType.getErrorMessage({ + service: makeService('legacy-model'), + workspace, + }) + ).toBeNull() + }) + + test('defers availability validation until the integration is loaded', () => { + const application = { id: 1 } + const serviceType = makeServiceType() + + expect( + serviceType.getErrorMessage({ + service: makeService('blob-model', 5), + workspace, + application, + }) + ).toBeNull() + }) + + test('uses a complete integration override when checking availability', () => { + const application = { id: 1 } + const serviceType = makeServiceType({ + id: 5, + ai_settings: { + openai: { api_key: 'integration-key', models: ['blob-model'] }, + }, + }) + expect( + serviceType.getErrorMessage({ + service: makeService('blob-model', 5), + workspace, + application, + }) + ).toBeNull() + }) + + test('validates an own-key override without models against the ai_agent allowlist', () => { + const application = { id: 1 } + const serviceType = makeServiceType({ + id: 5, + ai_settings: { openai: { api_key: 'integration-key' } }, + }) + + expect( + serviceType.getErrorMessage({ + service: makeService('db-model', 5), + workspace, + application, + }) + ).toBeNull() + expect( + serviceType.getErrorMessage({ + service: makeService('legacy-model', 5), + workspace, + application, + }) + ).toBe('serviceType.errorAIModelUnavailable') + }) + + test('reports a provider from an uninstalled extension as unavailable', () => { + const serviceType = makeServiceType() + + expect( + serviceType.getErrorMessage({ + service: { + ...makeService('removed-model'), + ai_generative_ai_type: 'removed-provider', + }, + workspace: { + ...workspace, + ai_features: { + ai_agent: { models: { 'removed-provider': ['removed-model'] } }, + }, + }, + }) + ).toBe('serviceType.errorAIModelUnavailable') + }) +}) diff --git a/web-frontend/test/unit/integrations/ai/utils.spec.js b/web-frontend/test/unit/integrations/ai/utils.spec.js new file mode 100644 index 0000000000..e4c26fb2d2 --- /dev/null +++ b/web-frontend/test/unit/integrations/ai/utils.spec.js @@ -0,0 +1,78 @@ +import { TestApp } from '@baserow/test/helpers/testApp' +import { GenerativeAIModelType } from '@baserow/modules/core/generativeAIModelTypes' +import { getEffectiveAIAgentModels } from '@baserow/modules/integrations/ai/utils' + +describe('AI Agent integration model resolution', () => { + let testApp + + beforeEach(() => { + testApp = new TestApp() + }) + + afterEach(async () => { + await testApp.afterEach() + }) + + test.each([ + 'openai', + 'anthropic', + 'google', + 'groq', + 'mistral', + 'ollama', + 'openrouter', + ])( + '%s inherits omitted models while preserving explicit lists', + (providerType) => { + const modelType = testApp + .getRegistry() + .get('generativeAIModel', providerType) + const connection = + providerType === 'ollama' + ? { host: 'http://localhost:11434' } + : { api_key: 'integration-key' } + const resolve = (integrationSettings) => + getEffectiveAIAgentModels({ + workspaceModels: ['shared-model'], + integrationSettings, + modelType, + }) + + expect(resolve(connection)).toEqual(['shared-model']) + expect(resolve({ ...connection, models: [] })).toEqual([]) + expect(resolve({ ...connection, models: ['own-model'] })).toEqual([ + 'own-model', + ]) + expect(resolve({ models: ['own-model', 'shared-model'] })).toEqual([ + 'shared-model', + ]) + } + ) + + test('does not add inherited models to an authoritative extension override', () => { + class ExtensionModelType extends GenerativeAIModelType { + static getType() { + return 'extension' + } + } + const modelType = new ExtensionModelType({ app: testApp.getApp() }) + + expect( + getEffectiveAIAgentModels({ + workspaceModels: ['shared-model'], + integrationSettings: {}, + modelType, + }) + ).toEqual([]) + }) + + test('excludes a removed provider even when workspace models are stale', () => { + expect( + getEffectiveAIAgentModels({ + workspaceModels: ['removed-model'], + integrationSettings: { models: ['removed-model'] }, + modelType: null, + }) + ).toEqual([]) + }) +})