Skip to content

Redis-Backed Semantic Search (Phase 4) #3663

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ajtiwari07 wants to merge 13 commits into
base: main
Choose a base branch
from
Open

Redis-Backed Semantic Search (Phase 4) #3663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3ccdef9
DAB Phase 4 Implementation
ajtiwari07 Jun 5, 2026
e7c997c
More features and tests
ajtiwari07 Jun 10, 2026
990087d
DAB Phase 4 Implementation
ajtiwari07 Jun 5, 2026
8c3a221
More features and tests
ajtiwari07 Jun 10, 2026
a92b1c2
Merge branch 'redis-dab-phase4' of https://github.com/ajtiwari07/data…
ajtiwari07 Jun 10, 2026
6fb288c
Fix duplicate semantic search helper after merge
ajtiwari07 Jun 10, 2026
eede53f
Remove separate embeddings config within semantic search
ajtiwari07 Jun 11, 2026
3cbca26
Fix semantic search flow tests dictionary readonly usage
ajtiwari07 Jun 11, 2026
601d837
Fix test failure
ajtiwari07 Jun 11, 2026
f352133
Fix PR comments Part 1
ajtiwari07 Jun 11, 2026
6afa920
Fix linting issues
ajtiwari07 Jun 11, 2026
6f2c786
Linting issues part 2
ajtiwari07 Jun 11, 2026
938b0be
Fix tests failures
ajtiwari07 Jun 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions schemas/dab.draft.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@
}
}
},

"compression": {
"type": "object",
"description": "Configures HTTP response compression settings.",
Expand Down Expand Up @@ -1487,6 +1488,67 @@
}
}
},
"semantic-search": {
"type": "object",
"description": "Semantic search configuration for this entity.",
"additionalProperties": false,
"properties": {
"enabled": {
"$ref": "#/$defs/boolean-or-string",

@JerryNixon JerryNixon Jun 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

semantic-search.enabled does not honor boolean-or-string. The schema allows enabled as boolean-or-string, but EntitySemanticSearchOptions.Enabled is a bool. @env() / @akv() configs will not round-trip/resolve correctly. Use the existing DAB boolean-or-string pattern and add env/AKV tests.

"description": "Enables semantic search for this entity.",
"default": false
},
"redis-index-name": {
"type": "string",
"description": "Name of the Redis vector index used for semantic search for this entity."
},
"redis-index-type": {
"type": "string",
"description": "Redis index storage type used by the semantic search index.",
"enum": ["hash", "json"],
"default": "hash"
},
"redis-index-multiplier": {
"type": "integer",
"description": "Multiplier applied to requested result count when querying Redis.",
"default": 2,
"minimum": 1,
"maximum": 10
},
"similarity-threshold": {
"type": "number",
"description": "Minimum Redis similarity value required for a semantic match.",
"default": 0.8,
"minimum": 0.0,
"maximum": 1.0
},
"input-description": {
"type": "string",
"description": "Description surfaced in API metadata for semantic search input.",
"default": "Natural language value used for semantic search."
},
"output-description": {
"type": "string",
"description": "Description surfaced in API metadata for semantic distance output.",
"default": "Semantic distance score returned by semantic search."
}
},
"allOf": [
{
"if": {
"properties": {
"enabled": {
"const": true
}
},
"required": ["enabled"]
},
"then": {
"required": ["redis-index-name"]
}
}
]
},
"mcp": {
"oneOf": [
{
Expand Down
32 changes: 23 additions & 9 deletions src/Cli/Commands/AddOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,22 @@ public AddOptions(
string? cacheTtlSeconds,
string? cacheLevel,
string? healthEnabled,
string? description,
IEnumerable<string>? parametersNameCollection,
IEnumerable<string>? parametersDescriptionCollection,
IEnumerable<string>? parametersRequiredCollection,
IEnumerable<string>? parametersDefaultCollection,
IEnumerable<string>? fieldsNameCollection,
IEnumerable<string>? fieldsAliasCollection,
IEnumerable<string>? fieldsDescriptionCollection,
IEnumerable<bool>? fieldsPrimaryKeyCollection,
string? semanticSearchEnabled = null,
string? semanticSearchRedisIndexName = null,
string? semanticSearchRedisIndexType = null,
string? semanticSearchRedisIndexMultiplier = null,
string? semanticSearchSimilarityThreshold = null,
string? semanticSearchInputDescription = null,
string? semanticSearchOutputDescription = null,
string? description = null,
IEnumerable<string>? parametersNameCollection = null,
IEnumerable<string>? parametersDescriptionCollection = null,
IEnumerable<string>? parametersRequiredCollection = null,
IEnumerable<string>? parametersDefaultCollection = null,
IEnumerable<string>? fieldsNameCollection = null,
IEnumerable<string>? fieldsAliasCollection = null,
IEnumerable<string>? fieldsDescriptionCollection = null,
IEnumerable<bool>? fieldsPrimaryKeyCollection = null,
string? mcpDmlTools = null,
string? mcpCustomTool = null,
string? config = null
Expand All @@ -66,6 +73,13 @@ public AddOptions(
cacheTtlSeconds,
cacheLevel,
healthEnabled,
semanticSearchEnabled,
semanticSearchRedisIndexName,
semanticSearchRedisIndexType,
semanticSearchRedisIndexMultiplier,
semanticSearchSimilarityThreshold,
semanticSearchInputDescription,
semanticSearchOutputDescription,
description,
parametersNameCollection,
parametersDescriptionCollection,
Expand Down
10 changes: 10 additions & 0 deletions src/Cli/Commands/ConfigureOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ public ConfigureOptions(
int? runtimePaginationMaxPageSize = null,
int? runtimePaginationDefaultPageSize = null,
bool? runtimePaginationNextLinkRelative = null,
string? runtimeCacheLevel2Provider = null,
Comment thread
ajtiwari07 marked this conversation as resolved.
string? runtimeCacheLevel2ConnectionString = null,
CompressionLevel? runtimeCompressionLevel = null,
bool? runtimeHealthEnabled = null,
int? runtimeHealthCacheTtlSeconds = null,
Expand Down Expand Up @@ -161,6 +163,8 @@ public ConfigureOptions(
RuntimePaginationMaxPageSize = runtimePaginationMaxPageSize;
RuntimePaginationDefaultPageSize = runtimePaginationDefaultPageSize;
RuntimePaginationNextLinkRelative = runtimePaginationNextLinkRelative;
RuntimeCacheLevel2Provider = runtimeCacheLevel2Provider;
RuntimeCacheLevel2ConnectionString = runtimeCacheLevel2ConnectionString;
// Compression
RuntimeCompressionLevel = runtimeCompressionLevel;
// Health
Expand Down Expand Up @@ -342,6 +346,12 @@ public ConfigureOptions(
[Option("runtime.pagination.next-link-relative", Required = false, HelpText = "Use relative URLs for pagination next links. Default: false (boolean).")]
public bool? RuntimePaginationNextLinkRelative { get; }

[Option("runtime.cache.level-2.provider", Required = false, HelpText = "Set level-2 cache provider. Allowed values: redis.")]
public string? RuntimeCacheLevel2Provider { get; }

[Option("runtime.cache.level-2.connection-string", Required = false, HelpText = "Set level-2 cache connection string.")]
public string? RuntimeCacheLevel2ConnectionString { get; }

[Option("runtime.compression.level", Required = false, HelpText = "Set the response compression level. Allowed values: optimal (default), fastest, none.")]
public CompressionLevel? RuntimeCompressionLevel { get; }

Expand Down
35 changes: 35 additions & 0 deletions src/Cli/Commands/EntityOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ public EntityOptions(
string? cacheTtlSeconds,
string? cacheLevel,
string? healthEnabled,
string? semanticSearchEnabled,
string? semanticSearchRedisIndexName,
string? semanticSearchRedisIndexType,
string? semanticSearchRedisIndexMultiplier,
string? semanticSearchSimilarityThreshold,
string? semanticSearchInputDescription,
string? semanticSearchOutputDescription,
string? description,
IEnumerable<string>? parametersNameCollection,
IEnumerable<string>? parametersDescriptionCollection,
Expand Down Expand Up @@ -58,6 +65,13 @@ public EntityOptions(
CacheTtlSeconds = cacheTtlSeconds;
CacheLevel = cacheLevel;
HealthEnabled = healthEnabled;
SemanticSearchEnabled = semanticSearchEnabled;
SemanticSearchRedisIndexName = semanticSearchRedisIndexName;
SemanticSearchRedisIndexType = semanticSearchRedisIndexType;
SemanticSearchRedisIndexMultiplier = semanticSearchRedisIndexMultiplier;
SemanticSearchSimilarityThreshold = semanticSearchSimilarityThreshold;
SemanticSearchInputDescription = semanticSearchInputDescription;
SemanticSearchOutputDescription = semanticSearchOutputDescription;
Description = description;
ParametersNameCollection = parametersNameCollection;
ParametersDescriptionCollection = parametersDescriptionCollection;
Expand Down Expand Up @@ -120,6 +134,27 @@ public EntityOptions(
[Option("health.enabled", Required = false, HelpText = "Enable health checks for this entity. Default: true (boolean).")]
public string? HealthEnabled { get; }

[Option("semantic-search.enabled", Required = false, HelpText = "Enable semantic search for this entity. Accepted values are true/false.")]
public string? SemanticSearchEnabled { get; }

[Option("semantic-search.redis-index-name", Required = false, HelpText = "Name of the Redis vector index to use for semantic search.")]
public string? SemanticSearchRedisIndexName { get; }

[Option("semantic-search.redis-index-type", Required = false, HelpText = "Redis index type for semantic search. Allowed values: hash, json.")]
public string? SemanticSearchRedisIndexType { get; }

[Option("semantic-search.redis-index-multiplier", Required = false, HelpText = "Multiplier for Redis candidate retrieval. Allowed values: 1-10.")]
public string? SemanticSearchRedisIndexMultiplier { get; }

[Option("semantic-search.similarity-threshold", Required = false, HelpText = "Default semantic similarity threshold. Allowed values: 0.0-1.0.")]
public string? SemanticSearchSimilarityThreshold { get; }

[Option("semantic-search.input-description", Required = false, HelpText = "Description for semantic search input metadata.")]
public string? SemanticSearchInputDescription { get; }

[Option("semantic-search.output-description", Required = false, HelpText = "Description for semantic distance output metadata.")]
public string? SemanticSearchOutputDescription { get; }

[Option("description", Required = false, HelpText = "Description of the entity.")]
public string? Description { get; }

Expand Down
32 changes: 23 additions & 9 deletions src/Cli/Commands/UpdateOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,22 @@ public UpdateOptions(
string? cacheTtlSeconds,
string? cacheLevel,
string? healthEnabled,
string? description,
IEnumerable<string>? parametersNameCollection,
IEnumerable<string>? parametersDescriptionCollection,
IEnumerable<string>? parametersRequiredCollection,
IEnumerable<string>? parametersDefaultCollection,
IEnumerable<string>? fieldsNameCollection,
IEnumerable<string>? fieldsAliasCollection,
IEnumerable<string>? fieldsDescriptionCollection,
IEnumerable<bool>? fieldsPrimaryKeyCollection,
string? semanticSearchEnabled = null,
string? semanticSearchRedisIndexName = null,
string? semanticSearchRedisIndexType = null,
string? semanticSearchRedisIndexMultiplier = null,
string? semanticSearchSimilarityThreshold = null,
string? semanticSearchInputDescription = null,
string? semanticSearchOutputDescription = null,
string? description = null,
IEnumerable<string>? parametersNameCollection = null,
IEnumerable<string>? parametersDescriptionCollection = null,
IEnumerable<string>? parametersRequiredCollection = null,
IEnumerable<string>? parametersDefaultCollection = null,
IEnumerable<string>? fieldsNameCollection = null,
IEnumerable<string>? fieldsAliasCollection = null,
IEnumerable<string>? fieldsDescriptionCollection = null,
IEnumerable<bool>? fieldsPrimaryKeyCollection = null,
string? mcpDmlTools = null,
string? mcpCustomTool = null,
string? config = null)
Expand All @@ -72,6 +79,13 @@ public UpdateOptions(
cacheTtlSeconds,
cacheLevel,
healthEnabled,
semanticSearchEnabled,
semanticSearchRedisIndexName,
semanticSearchRedisIndexType,
semanticSearchRedisIndexMultiplier,
semanticSearchSimilarityThreshold,
semanticSearchInputDescription,
semanticSearchOutputDescription,
description,
parametersNameCollection,
parametersDescriptionCollection,
Expand Down
59 changes: 58 additions & 1 deletion src/Cli/ConfigGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,14 @@ public static bool TryAddNewEntity(AddOptions options, RuntimeConfig initialRunt
EntityGraphQLOptions graphqlOptions = ConstructGraphQLTypeDetails(options.GraphQLType, graphQLOperationsForStoredProcedures);
EntityCacheOptions? cacheOptions = ConstructCacheOptions(options.CacheEnabled, options.CacheTtlSeconds, options.CacheLevel);
EntityHealthCheckConfig? entityHealthOptions = ConstructEntityHealthOptions(options.HealthEnabled);
EntitySemanticSearchOptions? semanticSearchOptions = ConstructSemanticSearchOptions(
options.SemanticSearchEnabled,
options.SemanticSearchRedisIndexName,
options.SemanticSearchRedisIndexType,
options.SemanticSearchRedisIndexMultiplier,
options.SemanticSearchSimilarityThreshold,
options.SemanticSearchInputDescription,
options.SemanticSearchOutputDescription);
EntityMcpOptions? mcpOptions = null;

if (options.McpDmlTools is not null || options.McpCustomTool is not null)
Expand All @@ -495,6 +503,7 @@ public static bool TryAddNewEntity(AddOptions options, RuntimeConfig initialRunt
Relationships: null,
Mappings: null,
Cache: cacheOptions,
SemanticSearch: semanticSearchOptions,
Description: string.IsNullOrWhiteSpace(options.Description) ? null : options.Description,
Mcp: mcpOptions,
Health: entityHealthOptions);
Expand Down Expand Up @@ -1002,7 +1011,9 @@ private static bool TryUpdateConfiguredRuntimeOptions(

// Cache: Enabled and TTL
if (options.RuntimeCacheEnabled != null ||
options.RuntimeCacheTTL != null)
options.RuntimeCacheTTL != null ||
options.RuntimeCacheLevel2Provider != null ||
options.RuntimeCacheLevel2ConnectionString != null)
{
RuntimeCacheOptions? updatedCacheOptions = runtimeConfig?.Runtime?.Cache ?? new();
bool status = TryUpdateConfiguredCacheValues(options, ref updatedCacheOptions);
Expand Down Expand Up @@ -1559,6 +1570,43 @@ private static bool TryUpdateConfiguredCacheValues(
}
}

// Runtime.Cache.level-2.provider
updatedValue = options?.RuntimeCacheLevel2Provider;
if (updatedValue != null)
{
string provider = ((string)updatedValue).Trim();
if (!string.Equals(provider, "redis", StringComparison.OrdinalIgnoreCase))
{
_logger.LogError("Failed to update Runtime.Cache.level-2.provider as '{updatedValue}'. Supported value is 'redis'.", updatedValue);
return false;
}

RuntimeCacheLevel2Options currentLevel2 = updatedCacheOptions?.Level2 ?? new();
updatedCacheOptions = updatedCacheOptions! with
{
Level2 = currentLevel2 with
{
Provider = provider.ToLowerInvariant()
}
};
_logger.LogInformation("Updated RuntimeConfig with Runtime.Cache.level-2.provider as '{updatedValue}'", updatedValue);
}

// Runtime.Cache.level-2.connection-string
updatedValue = options?.RuntimeCacheLevel2ConnectionString;
if (updatedValue != null)
{
RuntimeCacheLevel2Options currentLevel2 = updatedCacheOptions?.Level2 ?? new();
updatedCacheOptions = updatedCacheOptions! with
{
Level2 = currentLevel2 with
{
ConnectionString = (string)updatedValue
}
};
_logger.LogInformation("Updated RuntimeConfig with Runtime.Cache.level-2.connection-string.");
}

return true;
}
catch (Exception ex)
Expand Down Expand Up @@ -2327,6 +2375,14 @@ public static bool TryUpdateExistingEntity(UpdateOptions options, RuntimeConfig
EntityActionFields? updatedFields = GetFieldsForOperation(options.FieldsToInclude, options.FieldsToExclude);
EntityCacheOptions? updatedCacheOptions = ConstructCacheOptions(options.CacheEnabled, options.CacheTtlSeconds, options.CacheLevel) ?? entity.Cache;
EntityHealthCheckConfig? updatedEntityHealthOptions = ConstructEntityHealthOptions(options.HealthEnabled) ?? entity.Health;
EntitySemanticSearchOptions? updatedSemanticSearchOptions = ConstructSemanticSearchOptions(
options.SemanticSearchEnabled,
options.SemanticSearchRedisIndexName,
options.SemanticSearchRedisIndexType,
options.SemanticSearchRedisIndexMultiplier,
options.SemanticSearchSimilarityThreshold,
options.SemanticSearchInputDescription,
options.SemanticSearchOutputDescription);

// Determine if the entity is or will be a stored procedure
bool isStoredProcedureAfterUpdate = doOptionsRepresentStoredProcedure || (isCurrentEntityStoredProcedure && options.SourceType is null);
Expand Down Expand Up @@ -2588,6 +2644,7 @@ public static bool TryUpdateExistingEntity(UpdateOptions options, RuntimeConfig
Relationships: updatedRelationships,
Mappings: updatedMappings,
Cache: updatedCacheOptions,
SemanticSearch: updatedSemanticSearchOptions ?? entity.SemanticSearch,
Description: string.IsNullOrWhiteSpace(options.Description) ? entity.Description : options.Description,
Mcp: updatedMcpOptions,
Health: updatedEntityHealthOptions
Expand Down
Loading
Loading