diff --git a/docs-mintlify/api-reference/api.yaml b/docs-mintlify/api-reference/api.yaml index 3a13a5356622b..bfd4f3acdc16e 100644 --- a/docs-mintlify/api-reference/api.yaml +++ b/docs-mintlify/api-reference/api.yaml @@ -23,9 +23,19 @@ tags: - name: Workbooks - name: Notifications - name: Workspace + - name: User Attributes - name: App Theme - name: Embed - name: Embed Tenants + - name: Dbt Sync + - name: Env Variables + - name: OAuth Integrations + - name: OIDC Token Configs + - name: Regions + - name: Tenant Settings + - name: User Attribute Values + - name: User OAuth Tokens + - name: Users Admin paths: /v1/app-config: get: @@ -110,6 +120,24 @@ paths: tags: - Deployments /v1/deployments/{deploymentId}: + delete: + operationId: deleteDeployment + parameters: + - in: path + name: deploymentId + required: true + schema: + type: integer + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/DeploymentDeleteResponse' + description: '' + summary: Delete deployment + tags: + - Deployments get: operationId: getDeployment parameters: @@ -128,6 +156,110 @@ paths: summary: Get deployment tags: - Deployments + put: + operationId: updateDeployment + parameters: + - in: path + name: deploymentId + required: true + schema: + type: integer + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateDeploymentInput' + description: UpdateDeploymentInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Deployment' + description: '' + summary: Update deployment + tags: + - Deployments + /v1/deployments/{deploymentId}/dbt-sync: + post: + operationId: startDbtSync + parameters: + - in: path + name: deploymentId + required: true + schema: + type: integer + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/StartDbtSyncInput' + description: StartDbtSyncInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/DbtSyncResponse' + description: '' + summary: Start a dbt sync for a deployment + tags: + - Dbt Sync + /v1/deployments/{deploymentId}/env-vars: + get: + operationId: getEnvVariables + parameters: + - in: path + name: deploymentId + required: true + schema: + type: integer + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvVariablesListResponse' + description: '' + summary: Get env variables + tags: + - Env Variables + x-mint: + content: >- + Lists the deployment environment variables. Values of secret-named variables (containing + PASS, SECRET, TOKEN, KEY, or CREDENTIAL) are returned as `[ENCRYPTED]`. + put: + operationId: setEnvVariables + parameters: + - in: path + name: deploymentId + required: true + schema: + type: integer + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/SetEnvVariablesInput' + description: SetEnvVariablesInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvVariablesListResponse' + description: '' + summary: Set env variables + tags: + - Env Variables + x-mint: + content: >- + Upserts deployment environment variables by name; variables not included keep their + existing values. Passing the `[ENCRYPTED]` placeholder (the masked read value) is rejected + — omit variables you do not intend to change. /v1/deployments/{deploymentId}/environments: get: operationId: getDeploymentEnvironments @@ -954,6 +1086,24 @@ paths: oneOf: - type: integer - type: 'null' + - in: query + name: folderId + schema: + oneOf: + - type: integer + - type: 'null' + - in: query + name: externalWorkbookId + schema: + oneOf: + - type: string + - type: 'null' + - in: query + name: search + schema: + oneOf: + - type: string + - type: 'null' - in: query name: limit schema: @@ -1108,6 +1258,46 @@ paths: summary: Update report tags: - Reports + /v1/deployments/{deploymentId}/reports/{reportId}/connect-workbook: + put: + operationId: connectReportToWorkbook + parameters: + - in: path + name: deploymentId + required: true + schema: + type: number + - in: path + name: reportId + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectReportToWorkbookInput' + description: ConnectReportToWorkbookInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Report' + description: '' + summary: Connect a report to the calling spreadsheet + tags: + - Reports + x-mint: + content: |- + Link a report to the caller's own spreadsheet by recording its placement + (workbook id + result location) so the add-in can list and refresh it there. + + This upserts only the placement for the given workbook and never changes the + report's query, name, or other definition — so, like refresh, it requires only + read access to the report. A user who can view a report can place its data into + their own sheet without being its creator or having edit rights. /v1/deployments/{deploymentId}/reports/{reportId}/refresh: put: operationId: refreshReport @@ -1416,6 +1606,37 @@ paths: summary: Update workbook dashboard tags: - Workbooks + /v1/deployments/{deploymentId}/workbooks/{workbookId}/dashboard/ai-widget-thread: + post: + operationId: updatePublishedDashboardAiWidgetThread + parameters: + - in: path + name: deploymentId + required: true + schema: + type: number + - in: path + name: workbookId + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdatePublishedAiWidgetThreadInput' + description: UpdatePublishedAiWidgetThreadInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Dashboard' + description: '' + summary: Update published dashboard AI widget thread + tags: + - Workbooks /v1/deployments/{deploymentId}/workbooks/{workbookId}/duplicate: post: operationId: duplicateWorkbook @@ -1655,63 +1876,26 @@ paths: Moving a folder into one of its own descendants, or exceeding the maximum folder depth, is rejected with `400`. Returns `404` if the item does not exist or belongs to a different deployment. - /v1/embed-tenants/{embedTenantName}/groups: - get: - operationId: getGroups - parameters: - - in: path - name: embedTenantName - required: true - schema: - type: string - - in: query - name: after - schema: - oneOf: - - type: string - - type: 'null' - - in: query - name: first - schema: - oneOf: - - minimum: 1 - type: integer - - type: 'null' - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/UserGroupsConnectionResponse' - description: '' - summary: Get groups - tags: - - Embed Tenants - /v1/embed-tenants/{embedTenantName}/groups/{id}: + /v1/embed-tenants/{embedTenantName}: delete: - operationId: deleteGroup + operationId: deleteEmbedTenant parameters: - in: path name: embedTenantName required: true schema: type: string - - in: path - name: id - required: true - schema: - type: number responses: '200': content: application/json: {} description: Successful response - summary: Delete group + summary: Delete embed tenant tags: - Embed Tenants - /v1/embed-tenants/{embedTenantName}/user-attributes: + /v1/embed-tenants/{embedTenantName}/groups: get: - operationId: getUserAttributes + operationId: getGroups parameters: - in: path name: embedTenantName @@ -1736,14 +1920,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UserAttributesConnectionResponse' + $ref: '#/components/schemas/UserGroupsConnectionResponse' description: '' - summary: Get user attributes + summary: Get groups tags: - Embed Tenants - /v1/embed-tenants/{embedTenantName}/user-attributes/{id}: + /v1/embed-tenants/{embedTenantName}/groups/{id}: delete: - operationId: deleteUserAttribute + operationId: deleteGroup parameters: - in: path name: embedTenantName @@ -1760,7 +1944,7 @@ paths: content: application/json: {} description: Successful response - summary: Delete user attribute + summary: Delete group tags: - Embed Tenants /v1/embed/dashboard/{publicId}: @@ -1829,11 +2013,31 @@ paths: The session captures the embed context that will be baked into the embed token once redeemed — the target `deploymentId`, the end user's identity (`externalId` / `email` / - `userProfile`), their `groups` and `userAttributes`, and an optional `securityContext`. - Exchange the returned `sessionId` for a signed embed JWT via `POST + `userProfile`), their group memberships, `userAttributes`, and an optional + `securityContext`. Exchange the returned `sessionId` for a signed embed JWT via `POST /api/v1/embed/session/token` (single use). + The end user can be assigned to two independent kinds of group, which serve different + purposes: + + + - **`groups`** — global, tenant-wide groups for **data-model access control**. Their names + are passed verbatim into the Cube security context (`cubeCloud.groups`), where your data + model's `access_policy` rules use them to gate cubes, views, members, and + row-/column-level filters. They must already exist in the tenant. + + - **`tenantGroups`** — per-embed-tenant groups for **sharing and organizing content within + a single embed tenant** (e.g. sharing a workbook or dashboard with a group in Creator + Mode). Requires `creatorMode: true` and `embedTenantName`; create them inline via + `tenantGroupDefinitions`. In the security context they are namespaced as + `system:tenant:{embedTenantName}:group:{group}`, so they never collide with a same-named + global group. + + + See the individual request-body fields for the full contract. + + `deploymentId` is required and the caller must have read access to it. Embedding must be enabled for the tenant, otherwise `403` is returned. /v1/embed/session/token: @@ -1871,43 +2075,533 @@ paths: This endpoint is unauthenticated — it is called from the embedding client and the session id itself is the credential. Returns `401` if the session id is unknown or has already been redeemed. -components: - securitySchemes: - bearerAuth: - type: http - scheme: bearer - description: 'Token authentication. Send `Authorization: Bearer `.' - schemas: - AddNotificationRecipientsInput: - properties: - recipients: - description: Recipients to subscribe (1–1000 per request) - items: - $ref: '#/components/schemas/NotificationRecipientInput' - maxItems: 1000 - minItems: 1 - type: array - required: - - recipients - type: object - AppConfigResponse: - properties: - appTheme: - oneOf: - - $ref: '#/components/schemas/AppTheme' - - type: 'null' - applyThemeGlobally: - type: boolean - creatorMode: - oneOf: - - $ref: '#/components/schemas/CreatorMode' - - type: 'null' - required: - - applyThemeGlobally - type: object - AppTheme: - properties: - dark: + /v1/oauth-integrations: + get: + operationId: listOAuthIntegrations + responses: + '200': + content: + application/json: + schema: + items: + $ref: '#/components/schemas/OAuthIntegration' + type: array + description: '' + summary: List OAuth integrations + tags: + - OAuth Integrations + post: + operationId: createOAuthIntegration + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateOAuthIntegrationInput' + description: CreateOAuthIntegrationInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OAuthIntegration' + description: '' + summary: Create OAuth integration + tags: + - OAuth Integrations + /v1/oauth-integrations/{id}: + delete: + operationId: deleteOAuthIntegration + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + '200': + content: + application/json: {} + description: Successful response + summary: Delete OAuth integration + tags: + - OAuth Integrations + get: + operationId: getOAuthIntegration + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OAuthIntegration' + description: '' + summary: Get OAuth integration + tags: + - OAuth Integrations + put: + operationId: updateOAuthIntegration + parameters: + - in: path + name: id + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateOAuthIntegrationInput' + description: UpdateOAuthIntegrationInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OAuthIntegration' + description: '' + summary: Update OAuth integration + tags: + - OAuth Integrations + /v1/oidc-token-configs: + get: + operationId: listOidcTokenConfigs + responses: + '200': + content: + application/json: + schema: + items: + $ref: '#/components/schemas/OidcTokenConfig' + type: array + description: '' + summary: List OIDC token configs + tags: + - OIDC Token Configs + post: + operationId: createOidcTokenConfig + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateOidcTokenConfigInput' + description: CreateOidcTokenConfigInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OidcTokenConfig' + description: '' + summary: Create OIDC token config + tags: + - OIDC Token Configs + /v1/oidc-token-configs/{id}: + delete: + operationId: deleteOidcTokenConfig + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + '200': + content: + application/json: {} + description: Successful response + summary: Delete OIDC token config + tags: + - OIDC Token Configs + get: + operationId: getOidcTokenConfig + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OidcTokenConfig' + description: '' + summary: Get OIDC token config + tags: + - OIDC Token Configs + put: + operationId: updateOidcTokenConfig + parameters: + - in: path + name: id + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateOidcTokenConfigInput' + description: UpdateOidcTokenConfigInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/OidcTokenConfig' + description: '' + summary: Update OIDC token config + tags: + - OIDC Token Configs + /v1/regions: + get: + operationId: listRegions + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/RegionsListResponse' + description: '' + summary: List regions + tags: + - Regions + /v1/tenant/settings: + get: + operationId: getTenantSettings + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/TenantSettings' + description: '' + summary: Get tenant settings + tags: + - Tenant Settings + put: + operationId: updateTenantSettings + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TenantSettingsInput' + description: TenantSettingsInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/TenantSettings' + description: '' + summary: Update tenant settings + tags: + - Tenant Settings + /v1/user-attribute-values: + post: + operationId: upsertUserAttributeValue + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttributeValueCreateInput' + description: UserAttributeValueCreateInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttributeValueDTO' + description: '' + summary: Upsert user attribute value + tags: + - User Attribute Values + /v1/user-attribute-values/{userId}: + get: + operationId: getUserAttributeValues + parameters: + - in: path + name: userId + required: true + schema: + type: number + responses: + '200': + content: + application/json: + schema: + items: + $ref: '#/components/schemas/UserAttributeValueDTO' + type: array + description: '' + summary: Get user attribute values + tags: + - User Attribute Values + /v1/user-attributes: + get: + operationId: getUserAttributes + parameters: + - in: query + name: offset + schema: + oneOf: + - type: integer + - type: 'null' + - in: query + name: limit + schema: + oneOf: + - type: integer + - type: 'null' + - in: query + name: name + schema: + oneOf: + - type: string + - type: 'null' + - in: query + name: type + schema: + oneOf: + - $ref: '#/components/schemas/GetUserAttributesQueryType' + - type: 'null' + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttributesListResponse' + description: '' + summary: Get user attributes + tags: + - User Attributes + post: + operationId: createUserAttribute + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttributeCreateInput' + description: UserAttributeCreateInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttribute' + description: '' + summary: Create user attribute + tags: + - User Attributes + /v1/user-attributes/{id}: + put: + operationId: updateUserAttribute + parameters: + - in: path + name: id + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttributeUpdateInput' + description: UserAttributeUpdateInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UserAttribute' + description: '' + summary: Update user attribute + tags: + - User Attributes + /v1/user-oauth-tokens: + get: + operationId: listUserOAuthTokens + responses: + '200': + content: + application/json: + schema: + items: + $ref: '#/components/schemas/UserOAuthToken' + type: array + description: '' + summary: List user OAuth tokens + tags: + - User OAuth Tokens + /v1/user-oauth-tokens/{integrationId}: + delete: + operationId: revokeUserOAuthToken + parameters: + - in: path + name: integrationId + required: true + schema: + type: number + responses: + '200': + content: + application/json: {} + description: Successful response + summary: Revoke user OAuth token + tags: + - User OAuth Tokens + get: + operationId: getUserOAuthToken + parameters: + - in: path + name: integrationId + required: true + schema: + type: number + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UserOAuthToken' + description: '' + summary: Get user OAuth token + tags: + - User OAuth Tokens + /v1/user-oauth-tokens/{integrationId}/initiate: + post: + operationId: initiateOAuthFlow + parameters: + - in: path + name: integrationId + required: true + schema: + type: number + responses: + '200': + content: + application/json: {} + description: Successful response + summary: Initiate OAuth flow + tags: + - User OAuth Tokens + /v1/users: + post: + operationId: createUser + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UserCreateInput' + description: UserCreateInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/User' + description: '' + summary: Create user + tags: + - Users Admin + /v1/users/{id}: + delete: + operationId: deleteUser + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + '200': + content: + application/json: {} + description: Successful response + summary: Delete user + tags: + - Users Admin + put: + operationId: updateUser + parameters: + - in: path + name: id + required: true + schema: + type: number + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UserUpdateInput' + description: UserUpdateInput + required: false + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/User' + description: '' + summary: Update user + tags: + - Users Admin +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + description: 'Token authentication. Send `Authorization: Bearer `.' + schemas: + AddNotificationRecipientsInput: + properties: + recipients: + description: Recipients to subscribe (1–1000 per request) + items: + $ref: '#/components/schemas/NotificationRecipientInput' + maxItems: 1000 + minItems: 1 + type: array + required: + - recipients + type: object + AppConfigResponse: + properties: + appTheme: + oneOf: + - $ref: '#/components/schemas/AppTheme' + - type: 'null' + applyThemeGlobally: + type: boolean + creatorMode: + oneOf: + - $ref: '#/components/schemas/CreatorMode' + - type: 'null' + embedding: + oneOf: + - $ref: '#/components/schemas/EmbedSettings' + - type: 'null' + required: + - applyThemeGlobally + type: object + AppTheme: + properties: + dark: $ref: '#/components/schemas/AppThemeScheme' light: $ref: '#/components/schemas/AppThemeScheme' @@ -1980,6 +2674,20 @@ components: - percent - number type: string + ConnectReportToWorkbookInput: + properties: + endResultCell: + oneOf: + - type: string + - type: 'null' + externalWorkbookId: + type: string + resultLocation: + type: string + required: + - externalWorkbookId + - resultLocation + type: object ControlThemeBorderSectionInput: properties: color: @@ -2163,6 +2871,77 @@ components: - MONTHLY - CUSTOM type: string + CreateOAuthIntegrationInput: + properties: + authUrl: + format: url + type: string + clientId: + type: string + clientSecret: + type: string + config: + additionalProperties: true + type: object + name: + type: string + redirectUri: + format: url + type: string + scopes: + items: + type: string + type: array + tokenUrl: + format: url + type: string + type: + type: string + required: + - clientSecret + - name + - type + - authUrl + - tokenUrl + - clientId + - redirectUri + type: object + CreateOidcTokenConfigInput: + properties: + audienceType: + type: string + customAudience: + oneOf: + - type: string + - type: 'null' + customClaims: + oneOf: + - type: object + - type: 'null' + isEnabled: + oneOf: + - type: boolean + - type: 'null' + name: + oneOf: + - pattern: ^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$ + type: string + - type: 'null' + subFormat: + oneOf: + - pattern: ^(?:[A-Za-z0-9:_-]|\{(?:deployment_id|component|region)\})+$ + type: string + maxLength: 200 + - type: 'null' + targetEnvVar: + oneOf: + - pattern: ^[A-Z_][A-Z0-9_]*$ + type: string + maxLength: 128 + - type: 'null' + required: + - audienceType + type: object CreateReportInput: properties: endResultCell: @@ -2258,6 +3037,21 @@ components: type: string CreatorMode: properties: + localizedTitles: + oneOf: + - items: + $ref: '#/components/schemas/LocalizedValue' + type: array + maxItems: 50 + - type: 'null' + showGeneratedSql: + oneOf: + - type: boolean + - type: 'null' + showSemanticSql: + oneOf: + - type: boolean + - type: 'null' showWorkspaceTitle: oneOf: - type: boolean @@ -2295,12 +3089,20 @@ components: items: $ref: '#/components/schemas/ReportSnapshot' type: array + restrictDataDownload: + oneOf: + - type: boolean + - type: 'null' status: $ref: '#/components/schemas/DashboardDtoStatus' title: oneOf: - type: string - type: 'null' + useBoardDashboards: + oneOf: + - type: boolean + - type: 'null' versionId: type: integer versionNumber: @@ -2470,11 +3272,11 @@ components: - type: string - type: 'null' widgets: - items: - $ref: '#/components/schemas/DashboardWidgetInput' - type: array - required: - - widgets + oneOf: + - items: + $ref: '#/components/schemas/DashboardWidgetInput' + type: array + - type: 'null' type: object DashboardDtoStatus: enum: @@ -2858,6 +3660,19 @@ components: - w - h type: object + DbtSyncResponse: + properties: + branchName: + type: string + syncJobId: + type: string + workflowId: + type: string + required: + - syncJobId + - workflowId + - branchName + type: object Deployment: properties: creationStep: @@ -2874,6 +3689,13 @@ components: - deploymentUrl - creationStep type: object + DeploymentDeleteResponse: + properties: + success: + type: boolean + required: + - success + type: object DeploymentEnvironment: properties: api_credentials: @@ -3045,6 +3867,37 @@ components: - type: boolean - type: 'null' type: object + EmbedSessionSettings: + properties: + showDashboardChat: + oneOf: + - type: boolean + description: >- + Whether embedded published dashboards viewed with this session show the AI chat + (agent panel and launcher bubble). Omit to inherit the account-wide embed setting + (shown by default); `false` hides the chat even if it is enabled account-wide, + `true` shows it even if it is disabled account-wide. Only affects the dashboard + surface. + - type: 'null' + type: object + EmbedSettings: + properties: + locale: + oneOf: + - type: string + - type: 'null' + showDashboardChat: + oneOf: + - type: boolean + - type: 'null' + type: object + EmbedTenantProfile: + properties: + displayName: + oneOf: + - type: string + - type: 'null' + type: object EmbedTheme: properties: analyticsChat: @@ -3108,6 +3961,38 @@ components: type: string - type: 'null' type: object + EnvVariableDto: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + EnvVariablesListResponse: + properties: + data: + items: + $ref: '#/components/schemas/EnvVariableDto' + type: array + required: + - data + type: object + FillMissingRows: + properties: + excludedDimensions: + oneOf: + - items: + type: string + type: array + - type: 'null' + member: + type: string + required: + - member + type: object Folder: properties: createdAt: @@ -3210,14 +4095,15 @@ components: - pattern: ^[a-z][a-z0-9-]{0,34}[a-z0-9]$ type: string - type: 'null' - embedTheme: + embedTenantProfile: oneOf: - - $ref: '#/components/schemas/EmbedTheme' + - $ref: '#/components/schemas/EmbedTenantProfile' type: object - type: 'null' - ephemeralTtlSeconds: + embedTheme: oneOf: - - type: number + - $ref: '#/components/schemas/EmbedTheme' + type: object - type: 'null' externalId: oneOf: @@ -3228,12 +4114,29 @@ components: - items: $ref: '#/components/schemas/GroupDefinition' type: array + deprecated: true + description: >- + Deprecated and ignored. Global groups can no longer be created through this endpoint + — define them beforehand via the Cube Cloud UI or admin API. Still accepted for + backward compatibility (no error), but it has no effect. To create per-embed-tenant + groups, use `tenantGroupDefinitions`. - type: 'null' groups: oneOf: - items: type: string type: array + description: >- + Global user groups — defined once at the tenant level and shared across every embed + tenant — to assign this embed user to. Use `groups` for **data-model access + control**: each name is placed verbatim into the Cube security context as + `cubeCloud.groups`, where your data model's `access_policy` rules reference it to + gate cubes, views, members, and row-/column-level filters. The groups must already + exist in the tenant (create them via the Cube Cloud UI or admin API beforehand) — + this endpoint never creates global groups, and names that do not resolve to an + existing group are rejected. Global groups are NOT shown in an embed tenant’s + Creator Mode UI. To share or organize content inside a single embed tenant, use + `tenantGroups` instead. - type: 'null' internalId: oneOf: @@ -3258,6 +4161,44 @@ components: - type: object additionalProperties: true - type: 'null' + settings: + oneOf: + - $ref: '#/components/schemas/EmbedSessionSettings' + type: object + description: >- + Per-session overrides for embed behavior. Each key is tri-state: omit it to inherit + the account-wide setting, or set `true`/`false` to force the behavior for every + embed viewed with this session, taking precedence over the account-wide setting. + - type: 'null' + tenantGroupDefinitions: + oneOf: + - items: + $ref: '#/components/schemas/GroupDefinition' + type: array + description: >- + Idempotently create or update the per-embed-tenant groups referenced by + `tenantGroups`, before they are assigned. Requires `creatorMode: true` and + `embedTenantName`. Use this to declare a tenant’s groups in the same call that + assigns them, so you do not need a separate admin request. Applies only to + per-embed-tenant groups; global groups must be defined beforehand. + - type: 'null' + tenantGroups: + oneOf: + - items: + type: string + type: array + description: >- + Per-embed-tenant user groups — scoped to the single embed tenant named by + `embedTenantName` — to assign this embed user to. Use `tenantGroups` for **content + sharing and organization within one embed tenant**: for example, so a creator can + share a workbook, dashboard, or folder with a group of that tenant’s users. These + are the only groups shown in the embed tenant’s Creator Mode UI. Requires + `creatorMode: true` and `embedTenantName`. Define the groups beforehand — or in the + same request — via `tenantGroupDefinitions`. In the Cube security context they + appear namespaced as `system:tenant:{embedTenantName}:group:{groupName}`, so a + tenant group can never collide with — or be mistaken for — a global `groups` entry + of the same name. For organization-wide data-model access policies, use `groups`. + - type: 'null' userAttributeDefinitions: oneOf: - items: @@ -3303,6 +4244,14 @@ components: - ASC - DESC type: string + GetUserAttributesQueryType: + enum: + - string + - number + - boolean + - string_array + - number_array + type: string GetWorkspaceObjectsQueryOrderByDirection: enum: - ASC @@ -3326,6 +4275,18 @@ components: required: - name type: object + LocalizedValue: + properties: + locale: + maxLength: 35 + type: string + value: + maxLength: 1000 + type: string + required: + - locale + - value + type: object MoveWorkspaceObjectInput: properties: folderId: @@ -3556,6 +4517,103 @@ components: required: - items type: object + OAuthIntegration: + properties: + authUrl: + format: url + type: string + clientId: + type: string + config: + oneOf: + - type: object + additionalProperties: true + - type: 'null' + createdAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + id: + type: integer + name: + type: string + redirectUri: + format: url + type: string + scopes: + oneOf: + - items: {} + type: array + - type: 'null' + tokenUrl: + format: url + type: string + type: + type: string + updatedAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + required: + - id + - name + - type + - authUrl + - tokenUrl + - clientId + - redirectUri + - createdAt + - updatedAt + type: object + OidcTokenConfig: + properties: + audience: + type: string + audienceType: + type: string + createdAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + customClaims: + oneOf: + - type: object + - type: 'null' + id: + type: integer + isEnabled: + type: boolean + name: + type: string + subFormat: + oneOf: + - type: string + - type: 'null' + targetEnvVar: + oneOf: + - type: string + - type: 'null' + updatedAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + required: + - id + - audienceType + - audience + - name + - isEnabled + - createdAt + - updatedAt + type: object PageInfo: properties: endCursor: @@ -3652,6 +4710,49 @@ components: required: - workbookId type: object + RegionDto: + properties: + id: + type: integer + isDedicated: + oneOf: + - type: boolean + - type: 'null' + isHybrid: + oneOf: + - type: boolean + - type: 'null' + isPublic: + oneOf: + - type: boolean + - type: 'null' + name: + type: string + provider: + oneOf: + - type: string + - type: 'null' + title: + oneOf: + - type: string + - type: 'null' + useAiEngineer: + oneOf: + - type: boolean + - type: 'null' + required: + - id + - name + type: object + RegionsListResponse: + properties: + data: + items: + $ref: '#/components/schemas/RegionDto' + type: array + required: + - data + type: object RemoveNotificationRecipientInput: properties: embedTenantName: @@ -3694,6 +4795,14 @@ components: type: object Report: properties: + canEdit: + oneOf: + - type: boolean + - type: 'null' + canManage: + oneOf: + - type: boolean + - type: 'null' createdAt: oneOf: - format: date @@ -3716,6 +4825,12 @@ components: oneOf: - type: string - type: 'null' + externalWorkbookPlacements: + oneOf: + - items: + $ref: '#/components/schemas/ReportPlacement' + type: array + - type: 'null' folderId: oneOf: - type: integer @@ -3815,12 +4930,30 @@ components: - WORKBOOK - REPORT type: string + ReportPlacement: + properties: + endResultCell: + oneOf: + - type: string + - type: 'null' + resultLocation: + type: string + workbookId: + type: string + required: + - workbookId + - resultLocation + type: object ReportSnapshot: properties: description: oneOf: - type: string - type: 'null' + fillMissingRows: + oneOf: + - $ref: '#/components/schemas/FillMissingRows' + - type: 'null' id: type: integer kind: @@ -3938,6 +5071,8 @@ components: - DeploymentRead - DeploymentUpdate - DeploymentDelete + - SecretsManage + - DownloadData - PlaygroundRead - SchemaRead - SchemaUpdate @@ -3986,12 +5121,90 @@ components: type: array id: type: integer - name: + name: + type: string + required: + - id + - name + - actions + type: object + SetEnvVariablesInput: + properties: + env_variables: + items: + $ref: '#/components/schemas/EnvVariableDto' + type: array + required: + - env_variables + type: object + StartDbtSyncInput: + properties: + branchName: + oneOf: + - type: string + - type: 'null' + type: object + TenantSettings: + properties: + auditLogEnabled: + type: boolean + auditLogEnabledCurrent: + type: boolean + auditLogEnabledTimestamp: + type: integer + hasSupportAccess: + type: boolean + maintenanceWindowDay: type: string + maintenanceWindowEnabled: + type: boolean + maintenanceWindowTime: + type: string + oidcEnabled: + type: boolean + useAIBIUserInterface: + type: boolean required: - - id - - name - - actions + - auditLogEnabled + - auditLogEnabledCurrent + - auditLogEnabledTimestamp + - hasSupportAccess + - useAIBIUserInterface + - maintenanceWindowEnabled + - maintenanceWindowDay + - maintenanceWindowTime + - oidcEnabled + type: object + TenantSettingsInput: + properties: + auditLogEnabled: + oneOf: + - type: boolean + - type: 'null' + hasSupportAccess: + oneOf: + - type: boolean + - type: 'null' + maintenanceWindowDay: + oneOf: + - type: string + - type: 'null' + maintenanceWindowEnabled: + oneOf: + - type: boolean + - type: 'null' + maintenanceWindowTime: + oneOf: + - type: string + - type: 'null' + oidcEnabled: + oneOf: + - type: boolean + - type: 'null' + useAIBIUserInterface: + oneOf: + - type: boolean + - type: 'null' type: object ThemeFont: properties: @@ -4014,6 +5227,65 @@ components: - url - format type: object + UpdateDeploymentInput: + properties: + cloudProvider: + oneOf: + - type: string + - type: 'null' + creationMethod: + oneOf: + - $ref: '#/components/schemas/UpdateDeploymentInputCreationMethod' + - type: 'null' + creationStep: + oneOf: + - $ref: '#/components/schemas/CreationStep' + - type: 'null' + customDomain: + oneOf: + - type: string + - type: 'null' + deployBranchMergeAllowed: + oneOf: + - type: boolean + - type: 'null' + deployBranchName: + oneOf: + - type: string + - type: 'null' + deployBranchReadOnly: + oneOf: + - type: boolean + - type: 'null' + deployMode: + oneOf: + - $ref: '#/components/schemas/UpdateDeploymentInputDeployMode' + - type: 'null' + deployProjectRoot: + oneOf: + - type: string + - type: 'null' + name: + oneOf: + - type: string + - type: 'null' + region: + oneOf: + - type: string + - type: 'null' + type: object + UpdateDeploymentInputCreationMethod: + enum: + - upload + - cubecloud + - github + - ssh + type: string + UpdateDeploymentInputDeployMode: + enum: + - git + - cli + type: string UpdateFolderInput: properties: name: @@ -4101,6 +5373,88 @@ components: - MONTHLY - CUSTOM type: string + UpdateOAuthIntegrationInput: + properties: + authUrl: + format: url + type: string + clientId: + type: string + clientSecret: + oneOf: + - type: string + - type: 'null' + config: + additionalProperties: true + type: object + name: + type: string + redirectUri: + format: url + type: string + scopes: + items: + type: string + type: array + tokenUrl: + format: url + type: string + type: + type: string + required: + - name + - type + - authUrl + - tokenUrl + - clientId + - redirectUri + type: object + UpdateOidcTokenConfigInput: + properties: + customAudience: + oneOf: + - type: string + - type: 'null' + customClaims: + oneOf: + - type: object + - type: 'null' + isEnabled: + oneOf: + - type: boolean + - type: 'null' + name: + oneOf: + - pattern: ^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$ + type: string + - type: 'null' + subFormat: + oneOf: + - pattern: ^(?:[A-Za-z0-9:_-]|\{(?:deployment_id|component|region)\})+$ + type: string + maxLength: 200 + - type: 'null' + targetEnvVar: + oneOf: + - pattern: ^[A-Z_][A-Z0-9_]*$ + type: string + maxLength: 128 + - type: 'null' + type: object + UpdatePublishedAiWidgetThreadInput: + properties: + checksum: + oneOf: + - type: string + - type: 'null' + threadId: + type: string + widgetId: + type: string + required: + - widgetId + - threadId + type: object UpdateReportInput: properties: endResultCell: @@ -4136,36 +5490,200 @@ components: oneOf: - type: string - type: 'null' - sqlQuery: + sqlQuery: + oneOf: + - type: string + - type: 'null' + title: + oneOf: + - type: string + - type: 'null' + workbookId: + oneOf: + - type: integer + - type: 'null' + type: object + UpdateWorkbookInput: + properties: + folderId: + oneOf: + - type: number + - type: 'null' + meta: + oneOf: + - type: object + additionalProperties: true + - type: 'null' + name: + oneOf: + - type: string + - type: 'null' + slug: + oneOf: + - type: string + - type: 'null' + slugTakeover: + oneOf: + - type: boolean + - type: 'null' + type: object + User: + properties: + activeRoleId: + oneOf: + - type: integer + - type: 'null' + aliases: + oneOf: + - items: + $ref: '#/components/schemas/UserAlias' + type: array + - type: 'null' + createdAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + defaultRoles: + oneOf: + - items: + type: string + type: array + - type: 'null' + email: + type: string + externalId: + oneOf: + - type: string + - type: 'null' + firstName: + oneOf: + - type: string + - type: 'null' + gitUser: + oneOf: + - type: string + - type: 'null' + id: + type: integer + impersonation: + oneOf: + - $ref: '#/components/schemas/UserImpersonationDTO' + - type: 'null' + isAdmin: + type: boolean + isDeactivated: + oneOf: + - type: boolean + - type: 'null' + lastLogin: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + notifications: + additionalProperties: true + type: object + picture: + oneOf: + - type: string + - type: 'null' + samlId: + oneOf: + - type: string + - type: 'null' + settings: + oneOf: + - $ref: '#/components/schemas/UserSettingsInput' + - type: 'null' + title: + oneOf: + - type: string + - type: 'null' + tosAccepted: + oneOf: + - type: object + - type: 'null' + updatedAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + username: + type: string + required: + - id + - email + - username + - isAdmin + - notifications + type: object + UserAlias: + properties: + createdAt: + pattern: \d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d.\d+Z? + type: string + id: + type: integer + source: + $ref: '#/components/schemas/UserAliasDTOSource' + updatedAt: + pattern: \d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d.\d+Z? + type: string + userId: + type: integer + username: + type: string + required: + - id + - userId + - username + - source + - createdAt + - updatedAt + type: object + UserAliasDTOSource: + enum: + - ldap + - saml + - scim + - manual + type: string + UserAttribute: + properties: + defaultValue: + oneOf: + - type: string + - type: 'null' + description: oneOf: - type: string - type: 'null' - title: + displayName: oneOf: - type: string - type: 'null' - workbookId: - oneOf: - - type: integer - - type: 'null' - type: object - UpdateWorkbookInput: - properties: - folderId: - oneOf: - - type: number - - type: 'null' - meta: - oneOf: - - type: object - additionalProperties: true - - type: 'null' + id: + type: integer name: - oneOf: - - type: string - - type: 'null' + type: string + type: + $ref: '#/components/schemas/UserAttributeDTOType' + required: + - id + - name + - type type: object - UserAttribute: + UserAttributeCreateInput: properties: defaultValue: oneOf: @@ -4179,21 +5697,28 @@ components: oneOf: - type: string - type: 'null' - id: - type: integer name: + minLength: 1 type: string type: - $ref: '#/components/schemas/UserAttributeDTOType' + $ref: '#/components/schemas/UserAttributeCreateInputType' required: - - id - name - type type: object + UserAttributeCreateInputType: + enum: + - string + - number + - boolean + - string_array + - number_array + type: string UserAttributeDTOType: enum: - string - number + - boolean - string_array - number_array type: string @@ -4223,6 +5748,7 @@ components: enum: - string - number + - boolean - string_array - number_array type: string @@ -4235,6 +5761,7 @@ components: - oneOf: - type: string - type: number + - type: boolean - type: array items: type: string @@ -4245,17 +5772,116 @@ components: required: - name type: object - UserAttributesConnectionResponse: + UserAttributeUpdateInput: properties: - items: + defaultValue: + oneOf: + - type: string + - type: 'null' + description: + oneOf: + - type: string + - type: 'null' + displayName: + oneOf: + - type: string + - type: 'null' + type: object + UserAttributeValueCreateInput: + properties: + userAttributeId: + minLength: 1 + type: string + userId: + minLength: 1 + type: string + required: + - userId + - userAttributeId + type: object + UserAttributeValueDTO: + properties: + description: + oneOf: + - type: string + - type: 'null' + displayName: + oneOf: + - type: string + - type: 'null' + formattedValue: + oneOf: + - {} + - type: 'null' + name: + type: string + type: + $ref: '#/components/schemas/UserAttributeValueDTOType' + userAttributeId: + type: integer + userId: + type: integer + value: + oneOf: + - {} + - type: 'null' + required: + - userId + - userAttributeId + - name + - type + type: object + UserAttributeValueDTOType: + enum: + - string + - number + - boolean + - string_array + - number_array + type: string + UserAttributesListResponse: + properties: + count: + type: integer + data: items: $ref: '#/components/schemas/UserAttribute' type: array - pageInfo: - $ref: '#/components/schemas/PageInfo' required: - - items - - pageInfo + - data + - count + type: object + UserCreateInput: + properties: + defaultRoles: + oneOf: + - items: + type: string + type: array + - type: 'null' + email: + type: string + externalId: + oneOf: + - type: string + - type: 'null' + firstName: + oneOf: + - type: string + - type: 'null' + isAdmin: + oneOf: + - type: boolean + - type: 'null' + password: + oneOf: + - type: string + - type: 'null' + username: + type: string + required: + - email + - username type: object UserGroupDTO: properties: @@ -4287,6 +5913,141 @@ components: - items - pageInfo type: object + UserImpersonationDTO: + properties: + expiresAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + impersonatedUser: + oneOf: + - $ref: '#/components/schemas/User' + - type: 'null' + startedAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + type: object + UserOAuthToken: + properties: + accessTokenExpiresAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + createdAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + id: + type: integer + integrationId: + type: integer + lastError: + oneOf: + - {} + - type: 'null' + refreshTokenExpiresAt: + oneOf: + - oneOf: + - format: date + type: string + - format: date-time + type: string + - type: 'null' + status: + type: string + updatedAt: + oneOf: + - format: date + type: string + - format: date-time + type: string + userId: + type: integer + required: + - id + - userId + - integrationId + - status + - createdAt + - updatedAt + type: object + UserSettingsInput: + properties: + alternatingRowColors: + oneOf: + - type: boolean + - type: 'null' + lastSeenChangelogId: + oneOf: + - type: integer + - type: 'null' + locale: + oneOf: + - type: string + - type: 'null' + theme: + oneOf: + - type: string + - type: 'null' + timezone: + oneOf: + - type: string + - type: 'null' + type: object + UserUpdateInput: + properties: + firstName: + oneOf: + - type: string + - type: 'null' + isAdmin: + oneOf: + - type: boolean + - type: 'null' + isDeactivated: + oneOf: + - type: boolean + - type: 'null' + notifications: + oneOf: + - items: + type: string + type: array + - type: 'null' + password: + oneOf: + - type: string + - type: 'null' + settings: + oneOf: + - type: object + additionalProperties: true + - type: 'null' + title: + oneOf: + - type: string + - type: 'null' + tosAccepted: + oneOf: + - type: object + additionalProperties: true + - type: 'null' + type: object WidgetThemeBorderSectionInput: properties: color: @@ -4412,6 +6173,10 @@ components: oneOf: - $ref: '#/components/schemas/Dashboard' - type: 'null' + slug: + oneOf: + - type: string + - type: 'null' type: $ref: '#/components/schemas/WorkbookDtoType' updatedAt: diff --git a/docs-mintlify/api-reference/changelog.mdx b/docs-mintlify/api-reference/changelog.mdx index 3c5922aa9ca92..8416c9ace70ff 100644 --- a/docs-mintlify/api-reference/changelog.mdx +++ b/docs-mintlify/api-reference/changelog.mdx @@ -7,6 +7,47 @@ rss: true {/* GENERATED FILE — do not edit by hand. */} {/* Run scripts/extract-changelog.js against the platform client CHANGELOG.md. */} + + ### Added + + - `POST /api/v1/deployments/{deploymentId}/dbt-sync` operation (`DbtSyncPublicController.startDbtSync`) — starts a dbt sync for a deployment, optionally on a specific branch. New schemas: `StartDbtSyncInput` (`branchName`), `DbtSyncResponse` (`branchName`, `syncJobId`, `workflowId`). + - `PUT /api/v1/deployments/{deploymentId}/reports/{reportId}/connect-workbook` operation (`ReportsPublicController.connectReportToWorkbook`) — links a report to the caller's own spreadsheet by recording its placement, and returns the updated `Report`. Requires only read access to the report. New schema: `ConnectReportToWorkbookInput` (`externalWorkbookId`, `resultLocation`, `endResultCell`). + - New `SecretsManage` and `DownloadData` RBAC actions, available across all policy action enums (`RoleWithAccess.actions`, the inherited/resource user & group policy actions, and the update-policy request bodies). + - `AIEngineerSettings` now includes `chatThreadPolicies`. New schemas: `ChatThreadPolicyDTO` (`actions`, `effect`, `resourceType`, `resources`), `ChatThreadPolicyDTOEffect` (`allow` | `deny`), `ChatThreadPolicyDTOResourceType`. + - `GET /api/v1/deployments/{deploymentId}/reports` accepts a new `search` query filter. + - `Report` now includes `canEdit`, `canManage`, and `externalWorkbookPlacements` — a report can be placed in multiple spreadsheets. New schema: `ReportPlacement` (`workbookId`, `resultLocation`, `endResultCell`). + - `ReportSnapshot` now includes `fillMissingRows`. New schema: `FillMissingRows` (`member`, `excludedDimensions`). + - `CreatorMode` now includes `showGeneratedSql` and `showSemanticSql` toggles. + - `Dashboard` now includes `restrictDataDownload` and `useBoardDashboards`. + - `UpdatePublishedAiWidgetThreadInput` accepts an optional `checksum`. + + ### Changed + + - `DashboardConfigInput.widgets` is now optional. + + + + ### Added + + - `POST /api/v1/deployments/{deploymentId}/workbooks/{workbookId}/dashboard/ai-widget-thread` operation (`WorkbooksPublicController.updatePublishedDashboardAiWidgetThread`) — attaches an AI widget thread to a published dashboard and returns the updated `Dashboard`. New schema: `UpdatePublishedAiWidgetThreadInput` (`threadId`, `widgetId`). + - `DELETE /api/v1/embed-tenants/{embedTenantName}/` operation (`EmbedTenantAdminPublicController.deleteEmbedTenant`) for deleting an embed tenant. + - `CubePlatformClientConfig.credentials` option, forwarded to the underlying `fetch` to control the credentials mode. Defaults to the browser's `same-origin` behaviour; set `'omit'` for token-only clients (e.g. signed embedding) that must never authenticate via a cookie. + - Embed session requests (`GenerateSession`) accept `tenantGroups` and `tenantGroupDefinitions` — per-embed-tenant groups for sharing and organizing content within a single embed tenant (Creator Mode), namespaced separately from the global `groups`. New `embedTenantProfile` field with a `displayName`. New schema: `EmbedTenantProfile`. + - `AppConfigResponse` now includes an `embedding` field. New schema: `EmbedSettings` (`locale`). + - `CreatorMode` now includes `localizedTitles` for per-locale workspace titles. New schema: `LocalizedValue` (`locale`, `value`). + - `GET /api/v1/deployments/{deploymentId}/reports` accepts new `folderId` and `externalWorkbookId` query filters. + - `UpdateWorkbookInput` accepts `slug` and `slugTakeover`, and the `Workbook` response now includes `slug`. + + ### Changed + + - `UserAttributeDefinitionDTOType` now includes a `"boolean"` value, and `UserAttributeInput.value` accepts `boolean`. + + ### Removed + + - **BREAKING:** removed the `GET /api/v1/embed-tenants/{embedTenantName}/user-attributes` and `DELETE /api/v1/embed-tenants/{embedTenantName}/user-attributes/{id}` operations, along with the `UserAttribute`, `UserAttributesConnectionResponse`, and `UserAttributeDTOType` schemas. + - **BREAKING:** removed the `ephemeralTtlSeconds` field from the embed session request (`GenerateSession`). + + ### Added diff --git a/docs-mintlify/api-reference/introduction.mdx b/docs-mintlify/api-reference/introduction.mdx index 270f62cf52809..366606fe16371 100644 --- a/docs-mintlify/api-reference/introduction.mdx +++ b/docs-mintlify/api-reference/introduction.mdx @@ -59,15 +59,28 @@ https://{tenant}.cubecloud.dev/api | Entity | Resource | Version | | --- | --- | --- | +{/* AUTOGEN:platform-endpoints START — generated by scripts/extract-api.mjs; do not edit by hand */} | [Deployments](/api-reference/deployments/get-deployments) | `/v1/deployments` | v1 | | [Environments](/api-reference/environments/get-deployment-environments) | `/v1/deployments/{deploymentId}/environments` | v1 | | [Folders](/api-reference/folders/list-folders) | `/v1/deployments/{deploymentId}/folders` | v1 | | [Reports](/api-reference/reports/list-reports) | `/v1/deployments/{deploymentId}/reports` | v1 | | [Workbooks](/api-reference/workbooks/get-workbooks) | `/v1/deployments/{deploymentId}/workbooks` | v1 | | [Notifications](/api-reference/notifications/list-scheduled-notifications) | `/v1/deployments/{deploymentId}/notifications` | v1 | -| [Workspace](/api-reference/workspace/list-shared-workspace-items) | `/v1/deployments/{deploymentId}/workspace` | v1 | +| [Workspace](/api-reference/workspace/list-shared-workspace-items) | `/v1/deployments/{deploymentId}` | v1 | +| [User Attributes](/api-reference/user-attributes/get-user-attributes) | `/v1/user-attributes` | v1 | +| [App Theme](/api-reference/app-theme/get-app-config) | `/v1/app-config` | v1 | | [Embed](/api-reference/embed/get-an-embeddable-dashboard) | `/v1/embed` | v1 | -| [Embed Tenants](/api-reference/embed-tenants/get-groups) | `/v1/embed-tenants` | v1 | +| [Embed Tenants](/api-reference/embed-tenants/delete-embed-tenant) | `/v1/embed-tenants/{embedTenantName}` | v1 | +| [Dbt Sync](/api-reference/dbt-sync/start-a-dbt-sync-for-a-deployment) | `/v1/deployments/{deploymentId}/dbt-sync` | v1 | +| [Env Variables](/api-reference/env-variables/get-env-variables) | `/v1/deployments/{deploymentId}/env-vars` | v1 | +| [OAuth Integrations](/api-reference/oauth-integrations/list-oauth-integrations) | `/v1/oauth-integrations` | v1 | +| [OIDC Token Configs](/api-reference/oidc-token-configs/list-oidc-token-configs) | `/v1/oidc-token-configs` | v1 | +| [Regions](/api-reference/regions/list-regions) | `/v1/regions` | v1 | +| [Tenant Settings](/api-reference/tenant-settings/get-tenant-settings) | `/v1/tenant/settings` | v1 | +| [User Attribute Values](/api-reference/user-attribute-values/upsert-user-attribute-value) | `/v1/user-attribute-values` | v1 | +| [User OAuth Tokens](/api-reference/user-oauth-tokens/list-user-oauth-tokens) | `/v1/user-oauth-tokens` | v1 | +| [Users Admin](/api-reference/users-admin/create-user) | `/v1/users` | v1 | +{/* AUTOGEN:platform-endpoints END */} | [Users (SCIM)](/api-reference/scim-users/list-users) | `/scim/v2/Users` | SCIM 2.0 | | [Groups (SCIM)](/api-reference/scim-groups/list-groups) | `/scim/v2/Groups` | SCIM 2.0 | diff --git a/docs-mintlify/docs.json b/docs-mintlify/docs.json index 4ab9ef7aad3fc..ae6eb071143e8 100644 --- a/docs-mintlify/docs.json +++ b/docs-mintlify/docs.json @@ -741,6 +741,8 @@ "pages": [ "GET /v1/deployments", "GET /v1/deployments/{deploymentId}", + "PUT /v1/deployments/{deploymentId}", + "DELETE /v1/deployments/{deploymentId}", "POST /v1/deployments/{deploymentId}/token" ] }, @@ -774,6 +776,7 @@ "GET /v1/deployments/{deploymentId}/reports/{reportId}", "PUT /v1/deployments/{deploymentId}/reports/{reportId}", "DELETE /v1/deployments/{deploymentId}/reports/{reportId}", + "PUT /v1/deployments/{deploymentId}/reports/{reportId}/connect-workbook", "PUT /v1/deployments/{deploymentId}/reports/{reportId}/refresh" ] }, @@ -787,6 +790,7 @@ "PUT /v1/deployments/{deploymentId}/workbooks/{workbookId}", "DELETE /v1/deployments/{deploymentId}/workbooks/{workbookId}", "PUT /v1/deployments/{deploymentId}/workbooks/{workbookId}/dashboard", + "POST /v1/deployments/{deploymentId}/workbooks/{workbookId}/dashboard/ai-widget-thread", "POST /v1/deployments/{deploymentId}/workbooks/{workbookId}/duplicate", "POST /v1/deployments/{deploymentId}/workbooks/{workbookId}/publish" ] @@ -814,6 +818,15 @@ "POST /v1/deployments/{deploymentId}/workspace/move" ] }, + { + "group": "User Attributes", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/user-attributes", + "POST /v1/user-attributes", + "PUT /v1/user-attributes/{id}" + ] + }, { "group": "App Theme", "openapi": "/api-reference/api.yaml", @@ -834,10 +847,88 @@ "group": "Embed Tenants", "openapi": "/api-reference/api.yaml", "pages": [ + "DELETE /v1/embed-tenants/{embedTenantName}", "GET /v1/embed-tenants/{embedTenantName}/groups", - "DELETE /v1/embed-tenants/{embedTenantName}/groups/{id}", - "GET /v1/embed-tenants/{embedTenantName}/user-attributes", - "DELETE /v1/embed-tenants/{embedTenantName}/user-attributes/{id}" + "DELETE /v1/embed-tenants/{embedTenantName}/groups/{id}" + ] + }, + { + "group": "Dbt Sync", + "openapi": "/api-reference/api.yaml", + "pages": [ + "POST /v1/deployments/{deploymentId}/dbt-sync" + ] + }, + { + "group": "Env Variables", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/deployments/{deploymentId}/env-vars", + "PUT /v1/deployments/{deploymentId}/env-vars" + ] + }, + { + "group": "OAuth Integrations", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/oauth-integrations", + "POST /v1/oauth-integrations", + "GET /v1/oauth-integrations/{id}", + "PUT /v1/oauth-integrations/{id}", + "DELETE /v1/oauth-integrations/{id}" + ] + }, + { + "group": "OIDC Token Configs", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/oidc-token-configs", + "POST /v1/oidc-token-configs", + "GET /v1/oidc-token-configs/{id}", + "PUT /v1/oidc-token-configs/{id}", + "DELETE /v1/oidc-token-configs/{id}" + ] + }, + { + "group": "Regions", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/regions" + ] + }, + { + "group": "Tenant Settings", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/tenant/settings", + "PUT /v1/tenant/settings" + ] + }, + { + "group": "User Attribute Values", + "openapi": "/api-reference/api.yaml", + "pages": [ + "POST /v1/user-attribute-values", + "GET /v1/user-attribute-values/{userId}" + ] + }, + { + "group": "User OAuth Tokens", + "openapi": "/api-reference/api.yaml", + "pages": [ + "GET /v1/user-oauth-tokens", + "GET /v1/user-oauth-tokens/{integrationId}", + "DELETE /v1/user-oauth-tokens/{integrationId}", + "POST /v1/user-oauth-tokens/{integrationId}/initiate" + ] + }, + { + "group": "Users Admin", + "openapi": "/api-reference/api.yaml", + "pages": [ + "POST /v1/users", + "PUT /v1/users/{id}", + "DELETE /v1/users/{id}" ] }, { diff --git a/docs-mintlify/docs/explore-analyze/dashboards/index.mdx b/docs-mintlify/docs/explore-analyze/dashboards/index.mdx index b79b1ffa37d00..bbfa7a64d494b 100644 --- a/docs-mintlify/docs/explore-analyze/dashboards/index.mdx +++ b/docs-mintlify/docs/explore-analyze/dashboards/index.mdx @@ -18,6 +18,10 @@ Dashboards enable you to: In the dashboard builder inside your [workbook][ref-workbooks], select the reports you want to include and arrange them on the canvas alongside other [widgets][ref-widgets] to tell your data story, then publish the dashboard. This gives stakeholders direct access to the insights that matter most, without the complexity of the underlying analysis. +## Data freshness + +Each widget shows a [freshness](/docs/explore-analyze/workbooks/querying-data#result-freshness-and-provenance) leaf indicating how recently its data was refreshed. The dashboard's own leaf reflects its least-recently-refreshed widget, so you can see at a glance whether everything on the dashboard is up to date. + ## Linking between dashboards Dashboards can link to one another. When a table widget shows a dimension that diff --git a/docs-mintlify/docs/explore-analyze/explore.mdx b/docs-mintlify/docs/explore-analyze/explore.mdx index 78b08eb72fa6d..44e29e7216cc6 100644 --- a/docs-mintlify/docs/explore-analyze/explore.mdx +++ b/docs-mintlify/docs/explore-analyze/explore.mdx @@ -45,6 +45,8 @@ Explore state is also saved in the URL, making it easy to share your exploration If you have developer or admin access, you can [apply a security context](/docs/explore-analyze/workbooks/querying-data#applying-a-security-context) to an exploration to verify what a specific end user—or an AI agent querying on their behalf—would see. +Explore results carry the same [freshness and pre-aggregation indicators](/docs/explore-analyze/workbooks/querying-data#result-freshness-and-provenance) as workbook reports. + ## Saving explorations You can save an exploration so you can return to it later without diff --git a/docs-mintlify/docs/explore-analyze/notifications.mdx b/docs-mintlify/docs/explore-analyze/notifications.mdx index 76395482414e5..26b2c6b6115ed 100644 --- a/docs-mintlify/docs/explore-analyze/notifications.mdx +++ b/docs-mintlify/docs/explore-analyze/notifications.mdx @@ -21,12 +21,22 @@ Notifications are configured as part of a [scheduled refresh][ref-scheduled-refr When creating or editing a schedule, click **Add notification** to expand the notification configuration card. + + +To send the same dashboard to another channel or recipient list, [duplicate an +existing schedule][ref-duplicate] instead of configuring the notification from +scratch — the copy carries over the source schedule's notification settings. + + + ### Delivery channel Toggle between two delivery channels: -- **Email** — select multiple recipients from workspace users using a searchable - picker with checkboxes. +- **Email** — select recipients using a searchable picker with checkboxes. The + picker is grouped into **Users** and **User groups** sections, so you can add + individual workspace users, entire [user groups][ref-user-groups], or a mix of + both. - **Slack** — select a Slack channel to post to. Requires connecting your Slack workspace first (one-time OAuth flow via **Connect to Slack**). Once connected, select a channel from a searchable picker. @@ -51,10 +61,31 @@ notification card header before saving. ## Email notifications -Email notifications are sent to selected workspace users after a scheduled +Email notifications are sent to the selected recipients after a scheduled refresh completes. Each recipient receives a message with a link to the dashboard and the attached screenshot. + + +Recipients don't have to be added by an editor. Anyone who can view the dashboard +can [subscribe themselves][ref-subscribe] to a schedule's email notifications +from the published dashboard. + + + +### Recipients + +The recipient picker is split into two sections: + +- **Users** — individual workspace users. +- **User groups** — [user groups][ref-user-groups]. A group is expanded to its + current members each time the notification is sent, so adding or removing + members takes effect on the next run without editing the schedule. The **User + groups** section only appears when your workspace has at least one user group. + +A recipient who is selected individually and also belongs to a selected group is +emailed only once. + ## Slack notifications Slack notifications post a message with the dashboard screenshot to a single @@ -73,4 +104,7 @@ This is a one-time setup. Once connected, you can select any channel your Slack workspace has access to. [ref-roles]: /admin/users-and-permissions/roles-and-permissions -[ref-scheduled-refreshes]: /docs/explore-analyze/scheduled-refreshes \ No newline at end of file +[ref-scheduled-refreshes]: /docs/explore-analyze/scheduled-refreshes +[ref-duplicate]: /docs/explore-analyze/scheduled-refreshes#duplicating-a-schedule +[ref-user-groups]: /admin/users-and-permissions/user-groups +[ref-subscribe]: /docs/explore-analyze/scheduled-refreshes#subscribing-to-notifications diff --git a/docs-mintlify/docs/explore-analyze/scheduled-refreshes.mdx b/docs-mintlify/docs/explore-analyze/scheduled-refreshes.mdx index c002922279073..16ee5a2c2bd77 100644 --- a/docs-mintlify/docs/explore-analyze/scheduled-refreshes.mdx +++ b/docs-mintlify/docs/explore-analyze/scheduled-refreshes.mdx @@ -7,7 +7,9 @@ description: Scheduled refreshes are available on Premium and Enterprise plans. Scheduled refreshes are available on [Premium and Enterprise plans](https://cube.dev/pricing).
Users need at least the [Explorer][ref-roles] role and Edit or Manage permission -on the workbook to set up scheduled refreshes. +on the workbook to set up scheduled refreshes. Anyone who can view the dashboard +can open the sidebar to see its schedules and [subscribe to +notifications](#subscribing-to-notifications). @@ -25,6 +27,16 @@ From the Dashboard Builder, click the calendar icon in the toolbar to open the scheduled refreshes sidebar. From this sidebar, you can create new schedules, modify existing ones, or remove schedules you no longer need. +The same calendar icon is also available on a published dashboard, so you can +reach the sidebar without opening the builder. What you can do there depends on +your permission on the workbook: + +- **Edit** or **Manage** — the full sidebar: create, edit, run, and delete + schedules. +- **View only** — a read-only list of the configured schedules, where you can + [subscribe or unsubscribe](#subscribing-to-notifications) yourself from each + schedule's email notifications. + The dashboard must be published before schedules can be created. @@ -73,6 +85,9 @@ The sidebar lists all existing schedules with the following information: - **Enable/disable toggle** — turn schedules on or off without deleting them. - **Next run time** — shows the next scheduled execution, or "Disabled" if the schedule is off. +- **Notification channel** — when a [notification][ref-notifications] is + configured, the card shows its delivery channel, for example "Notification: + Email" or "Notification: Slack (#channel)". ### Actions @@ -80,8 +95,43 @@ The sidebar lists all existing schedules with the following information: | --- | --- | | Run now | Manually trigger the schedule immediately | | Edit | Open the form dialog to modify the schedule | +| Duplicate | Create a new schedule pre-filled from this one (see [Duplicating a schedule](#duplicating-a-schedule)) | | Delete | Remove the schedule permanently | +## Duplicating a schedule + +To create a new schedule based on an existing one — for example, to send the +same dashboard to another Slack channel or a different set of recipients without +re-entering the frequency, timezone, and notification settings — you can +duplicate it. There are two ways: + +- **From the sidebar** — click the duplicate (copy) icon on a schedule card. The + form dialog opens pre-filled with that schedule's frequency, timezone, and + notification configuration. Adjust anything you need, then click **Save** to + create the new schedule. +- **From the edit dialog** — while editing a schedule, click **Save as copy** in + the dialog footer. This creates a new schedule from the current form values, + including any unsaved changes, instead of updating the original. + +Duplicates are independent schedules: the original is left unchanged, and the +copy keeps the source schedule's enabled or disabled state. + +## Subscribing to notifications + +If a schedule sends [email notifications][ref-notifications], anyone who can view +the dashboard can subscribe to it themselves — an editor does not have to add +them as a recipient. Open the scheduled refreshes sidebar from the published +dashboard and use the toggle on a schedule to subscribe or unsubscribe. +Subscribing adds you to the schedule's email recipients; unsubscribing removes +you. The change takes effect on the next run. + +The toggle appears only for schedules that email individual recipients — those +with notifications enabled and the email delivery channel. A schedule delivers to +either individual inboxes or a Slack channel, not both, so a schedule that posts +to Slack (or that has notifications turned off) has nothing to subscribe to. If an +editor switches a schedule to Slack, its existing subscriptions are cancelled; +you can subscribe again if it is later switched back to email. + ## Run phases When a scheduled refresh runs, it progresses through these phases: diff --git a/docs-mintlify/docs/explore-analyze/workbooks/querying-data.mdx b/docs-mintlify/docs/explore-analyze/workbooks/querying-data.mdx index c62981ff267ce..7f033e55b9529 100644 --- a/docs-mintlify/docs/explore-analyze/workbooks/querying-data.mdx +++ b/docs-mintlify/docs/explore-analyze/workbooks/querying-data.mdx @@ -213,6 +213,13 @@ The **Semantic SQL** tab also has a dropdown for viewing the same query in the f These views are convenient when you want to take a query you've built interactively in a workbook and reuse it directly through one of the Core Data APIs from your own application. +## Result freshness and provenance + +Two indicators next to each result describe the data behind it. + +- **Freshness** — a leaf icon shows how recently the underlying data was refreshed. Hover over it to see the last refresh time (for example, "Refreshed 5 minutes ago"); its color shifts as the data ages, so you can tell fresh from stale results at a glance. If the refresh time can't be determined, the leaf turns gray with a "Data age unknown" label. When you have access to [Query History](/admin/monitoring/query-history), clicking the leaf opens the underlying request for that result. +- **Pre-aggregation** — for those same users, a lightning-bolt icon appears next to the leaf with a "Served from a pre-aggregation" label whenever the result was served from a [pre-aggregation](/docs/pre-aggregations). If no icon is shown, the query ran directly against your data source. + ## Editing Semantic SQL by hand You can edit the Semantic SQL directly in the editor to refine a query — for example, to add a `WHERE` clause, change `GROUP BY` order, or apply a different sort. After making changes, click **Save and Run** to apply them to the query and refresh the results, or **Discard** to revert. diff --git a/docs-mintlify/embedding/iframe/dashboards.mdx b/docs-mintlify/embedding/iframe/dashboards.mdx index e8430459fda47..52fe69fcfb912 100644 --- a/docs-mintlify/embedding/iframe/dashboards.mdx +++ b/docs-mintlify/embedding/iframe/dashboards.mdx @@ -83,6 +83,36 @@ CSV is generated client-side from the data already loaded into the widget, so no additional query is issued. The parameter is opt-in — omit it (the default) to keep the download action hidden. +## Show or hide the AI chat + +Embedded dashboards show an AI chat (the agent panel and its launcher bubble) by +default. You can control it in two ways: + +- **Globally, in the UI.** Toggle **Show dashboard chat** under + **Embed → Settings** in the Cube Cloud console. This switch is + account-wide — it turns the AI chat on or off for the entire embedded surface, + i.e. every embedded dashboard. +- **Per session, via the API.** Pass `settings.showDashboardChat` when you + [generate the session](/reference/embed-apis/generate-session#session-settings). A per-session + value takes precedence over the global toggle — `false` hides the chat for that + session even when it is enabled account-wide, and `true` shows it even when it + is disabled: + +```javascript +body: JSON.stringify({ + deploymentId: DEPLOYMENT_ID, + externalId: "user@example.com", + settings: { + // Hide the AI chat for this viewer only + showDashboardChat: false, + }, +}), +``` + +This applies to embedded published dashboards; it does not affect the standalone +[Analytics Chat](/embedding/iframe/analytics-chat) surface, where embedding the +chat is itself the opt-in. + ## Set the language Embedded dashboards render their UI in the account's default language, which you can diff --git a/docs-mintlify/package.json b/docs-mintlify/package.json index 29b6482bce858..16ef386fe635a 100644 --- a/docs-mintlify/package.json +++ b/docs-mintlify/package.json @@ -6,7 +6,11 @@ "scripts": { "dev": "mintlify dev --port 3002", "build": "mintlify build", - "broken-links": "mintlify broken-links --check-anchors --check-redirects --check-snippets" + "broken-links": "mintlify broken-links --check-anchors --check-redirects --check-snippets", + "api:extract": "node scripts/extract-api.mjs", + "api:changelog": "node scripts/extract-changelog.mjs", + "api:sync": "yarn api:extract && yarn api:changelog", + "api:check": "node scripts/extract-api.mjs --check" }, "devDependencies": { "mintlify": "^4" diff --git a/docs-mintlify/reference/embed-apis/generate-session.mdx b/docs-mintlify/reference/embed-apis/generate-session.mdx index 67030bd9a2b7b..e0f1bc0c73782 100644 --- a/docs-mintlify/reference/embed-apis/generate-session.mdx +++ b/docs-mintlify/reference/embed-apis/generate-session.mdx @@ -46,6 +46,7 @@ POST https://{accountName}.cubecloud.dev/api/v1/embed/generate-session | `userAttributeDefinitions` | array | No | Idempotently upsert attribute definitions before applying values. Requires `creatorMode: true`. See [Creator mode bootstrap](#creator-mode-bootstrapping-groups-and-user-attributes). | | `groupDefinitions` | array | No | Idempotently upsert group definitions before assigning memberships. Requires `creatorMode: true`. See [Creator mode bootstrap](#creator-mode-bootstrapping-groups-and-user-attributes). | | `securityContext` | object | No | Custom security context object passed to Cube queries. Not allowed with `internalId`. | +| `settings` | object | No | Per-session overrides for embed behavior, applied to every embed viewed with this session. See [Session settings](#session-settings). | @@ -59,6 +60,25 @@ Accounts are limited to 10,000 external users. To increase this limit, please co +## Session settings + +`settings` is an object of per-session overrides for embed behavior. Each key is +**tri-state**: omit it to inherit the account-wide setting, or set `true`/`false` +to force the behavior for every embed viewed with this session, taking precedence +over the account-wide setting. + +```json +{ + "settings": { + "showDashboardChat": false + } +} +``` + +| Property | Type | Required | Notes | +|----------|------|----------|-------| +| `showDashboardChat` | boolean | No | Show or hide the AI chat (agent panel and launcher bubble) on embedded dashboards for this session. Omit to inherit the account-wide **Show dashboard chat** toggle (**Embed → Settings**; shown by default); `false` hides it even if enabled account-wide, `true` shows it even if disabled. Affects embedded **published dashboards** only — not the standalone [embedded chat][ref-analytics-chat] surface. See [Hiding the AI chat][ref-signed-embedding]. | + ## User profile `userProfile` lets you attach a human-readable display name and avatar to an external user so they render with a recognizable identity inside embedded surfaces (workbook owners, dashboard headers, etc.) instead of the raw `externalId`. @@ -349,3 +369,4 @@ Use session ID in [signed embedding][ref-signed-embedding]. [ref-api-keys]: /admin/account-billing/api-keys [ref-signed-embedding]: /embedding/iframe/auth/signed [ref-creator-mode]: /embedding/iframe/creator-mode +[ref-analytics-chat]: /embedding/iframe/analytics-chat diff --git a/docs-mintlify/scripts/extract-api.mjs b/docs-mintlify/scripts/extract-api.mjs index 995b1086fe8e3..f525f01ac4d7e 100644 --- a/docs-mintlify/scripts/extract-api.mjs +++ b/docs-mintlify/scripts/extract-api.mjs @@ -10,6 +10,13 @@ // in api-reference/scim.yaml. (Both the REST API and SCIM authenticate with a // Bearer token; see the securityScheme override below.) // +// One run regenerates ALL derived artifacts from the spec, so they can't drift: +// 1. api-reference/api.yaml — the standalone OpenAPI spec +// 2. docs.json — the Platform API nav groups (in place) +// 3. api-reference/introduction.mdx — the endpoint table (between AUTOGEN markers) +// Pass --check to verify these are up to date WITHOUT writing (exits non-zero on +// drift) — run it in CI so a spec change can never leave the committed docs stale. +// // The source spec lives in the (private) cubejs-enterprise repo and is NOT in // this repo, so there is no hardcoded path — point the script at the spec via // the SRC_SPEC env var (a CLI path arg is also accepted), or set it in @@ -29,7 +36,12 @@ import yaml from 'js-yaml'; const envPath = path.join(import.meta.dirname, '..', '.env'); if (fs.existsSync(envPath)) process.loadEnvFile(envPath); -const srcArg = process.argv[2] || process.env.SRC_SPEC; +// --check verifies the committed artifacts are up to date WITHOUT writing them, +// exiting non-zero if any drifted — wire it into CI so a spec change can never +// silently leave docs.json / the intro table stale. The spec path is the first +// non-flag arg (or SRC_SPEC). +const CHECK = process.argv.slice(2).includes('--check'); +const srcArg = process.argv.slice(2).find((a) => !a.startsWith('--')) || process.env.SRC_SPEC; if (!srcArg) { console.error( 'No source spec provided. Set the SRC_SPEC env var (or pass a path arg):\n' + @@ -45,7 +57,45 @@ if (!fs.existsSync(SRC)) { console.error(`Source spec not found: ${SRC}`); process.exit(1); } -const OUT = path.join(import.meta.dirname, '..', 'api-reference', 'api.yaml'); +const ROOT = path.join(import.meta.dirname, '..'); +const OUT = path.join(ROOT, 'api-reference', 'api.yaml'); +const DOCS_JSON = path.join(ROOT, 'docs.json'); +const INTRO_MDX = path.join(ROOT, 'api-reference', 'introduction.mdx'); +const API_REF = '/api-reference/api.yaml'; +// Markers delimiting the auto-generated Platform API rows in the intro table. +const INTRO_START = + '{/* AUTOGEN:platform-endpoints START — generated by scripts/extract-api.mjs; do not edit by hand */}'; +const INTRO_END = '{/* AUTOGEN:platform-endpoints END */}'; + +// Written-file tracker: writes on a normal run, records drift under --check. +const staleFiles = []; +function writeOrCheck(file, content) { + const current = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null; + if (current === content) return; + if (CHECK) { + staleFiles.push(path.relative(ROOT, file)); + return; + } + fs.writeFileSync(file, content); + console.log('Wrote', path.relative(ROOT, file)); +} + +// kebab-case a summary/tag the same way Mintlify slugs OpenAPI pages, so the +// intro-table links resolve to the generated pages. +function kebab(s) { + return String(s).trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); +} +// Longest shared path prefix (by segment) across a tag's paths — the "Resource" +// column value. Falls back to the single path when a tag has just one. +function commonPathPrefix(pathList) { + const split = pathList.map((p) => p.split('/')); + const first = split[0]; + let i = 0; + for (; i < first.length; i++) { + if (!split.every((s) => s[i] === first[i])) break; + } + return split.length === 1 ? first.join('/') : first.slice(0, i).join('/') || '/'; +} // Only the v1 REST API. SCIM lives in the hand-curated scim.yaml (different auth). const INCLUDE_PREFIX = '/api/v1/'; @@ -128,14 +178,27 @@ function applyDescription(op) { delete op.description; } -// Strip the " Public" suffix the source appends to every tag and normalize a few -// acronyms; TAG_MAP overrides win. +// Acronym fixups applied to auto-cleaned tags so casing stays consistent in the +// nav. Ordered longest-match-first: multi-token forms (e.g. the source's spaced +// "O Auth") must collapse before single-token rules run. Add a rule here whenever +// a new tag's title-cased form mangles an acronym. +// Case-insensitive for acronyms the upstream humanizer cases inconsistently: +// the tag comes through as "O Auth"/"Oidc" but the operation summary as +// "o auth"/"oidc". Matching either way normalizes both to one canonical form. +const ACRONYMS = [ + [/\bo\s*auth\b/gi, 'OAuth'], + [/\boidc\b/gi, 'OIDC'], + [/\bscim\b/gi, 'SCIM'], + [/\bai\b/gi, 'AI'], +]; + +// Strip the " Public" suffix the source appends to every tag and normalize +// acronyms via ACRONYMS; TAG_MAP overrides win. function cleanTag(raw) { if (TAG_MAP[raw]) return TAG_MAP[raw]; - return raw - .replace(/\s+Public$/, '') - .replace(/\bScim\b/g, 'SCIM') - .replace(/\bAi\b/g, 'AI'); + let tag = raw.replace(/\s+Public$/, ''); + for (const [re, repl] of ACRONYMS) tag = tag.replace(re, repl); + return tag; } const src = yaml.load(fs.readFileSync(SRC, 'utf8')); @@ -162,6 +225,11 @@ for (const [key, val] of Object.entries(src.paths)) { // Mintlify shows the operation description as plain text, so move the prose // into x-mint.content (rendered as MDX) and keep a plain copy for SEO. applyDescription(val[m]); + // Normalize acronyms in the summary too (it drives the page title AND slug), + // so "O Auth" reads "OAuth" everywhere, not just in the nav tag. + if (typeof val[m].summary === 'string') { + for (const [re, repl] of ACRONYMS) val[m].summary = val[m].summary.replace(re, repl); + } // strip "XxxController." prefix from operationId for clean page slugs if (typeof val[m].operationId === 'string') { val[m].operationId = val[m].operationId.replace(/^[^.]*\./, ''); @@ -269,30 +337,83 @@ const out = { }, }; -fs.writeFileSync(OUT, yaml.dump(out, { lineWidth: 100, noRefs: true })); -console.log('Wrote', OUT); +writeOrCheck(OUT, yaml.dump(out, { lineWidth: 100, noRefs: true })); console.log('paths:', Object.keys(paths).length, '| schemas:', Object.keys(schemas).length, '| tags:', orderedTags.length); -// 5. Emit nav fragment (groups in tag order; pages in source order within a tag) -// + index rows for the introduction table. +// 5. Group operations by tag (pages in source order within a tag) and capture, +// per tag, its paths + the first operation's summary — used to build both the +// docs.json nav and the intro-table rows. const byTag = {}; -const firstPathForTag = {}; +const pathsForTag = {}; +const firstSummaryForTag = {}; for (const [p, val] of Object.entries(paths)) { for (const m of METHODS) { if (!val[m]) continue; const tag = (val[m].tags && val[m].tags[0]) || 'Other'; (byTag[tag] = byTag[tag] || []).push(`${m.toUpperCase()} ${p}`); - if (!firstPathForTag[tag]) firstPathForTag[tag] = p; + if (!(pathsForTag[tag] || []).includes(p)) (pathsForTag[tag] = pathsForTag[tag] || []).push(p); + if (!(tag in firstSummaryForTag)) firstSummaryForTag[tag] = val[m].summary || ''; } } -const groups = orderedTags.filter((t) => byTag[t]).map((t) => ({ - group: t, - openapi: '/api-reference/api.yaml', - pages: byTag[t], -})); -console.log('\n=== NAV GROUPS (paste into docs.json API tab) ==='); -console.log(JSON.stringify(groups, null, 2)); -console.log('\n=== INDEX ROWS (entity | resource | version) ==='); -for (const t of orderedTags) { - if (firstPathForTag[t]) console.log(`${t} | ${firstPathForTag[t]} | v1`); +const groups = orderedTags + .filter((t) => byTag[t]) + .map((t) => ({ group: t, openapi: API_REF, pages: byTag[t] })); + +// 6. Rewrite the api.yaml-backed nav groups in docs.json in place, preserving all +// other nav (including the SCIM groups, which come from scim.yaml) and their order. +const docs = JSON.parse(fs.readFileSync(DOCS_JSON, 'utf8')); +let platformGroup = null; +(function find(node) { + if (platformGroup || !node || typeof node !== 'object') return; + if (Array.isArray(node)) return node.forEach(find); + if (Array.isArray(node.pages) && node.pages.some((g) => g && g.openapi === API_REF)) { + platformGroup = node; + return; + } + Object.values(node).forEach(find); +})(docs); +if (!platformGroup) { + console.error(`Aborting: no nav group backed by ${API_REF} found in docs.json.`); + process.exit(1); +} +const nonApi = platformGroup.pages.filter((g) => !(g && g.openapi === API_REF)); +platformGroup.pages = [...groups, ...nonApi]; +writeOrCheck(DOCS_JSON, JSON.stringify(docs, null, 2) + '\n'); + +// 7. Regenerate the Platform API rows in the introduction table between the +// AUTOGEN markers. The entity link mirrors Mintlify's page slug +// (/api-reference/{kebab tag}/{kebab summary}); the resource is the tag's +// common path prefix. SCIM rows live outside the markers (hand-maintained). +const intro = fs.readFileSync(INTRO_MDX, 'utf8'); +const s = intro.indexOf(INTRO_START); +const e = intro.indexOf(INTRO_END); +if (s < 0 || e < 0 || e < s) { + console.error( + `Aborting: AUTOGEN markers not found in ${path.relative(ROOT, INTRO_MDX)}.\n` + + `Add these two lines around the Platform API rows of the endpoint table:\n` + + ` ${INTRO_START}\n ${INTRO_END}` + ); + process.exit(1); +} +const rows = groups + .map( + (g) => + `| [${g.group}](/api-reference/${kebab(g.group)}/${kebab(firstSummaryForTag[g.group])}) ` + + `| \`${commonPathPrefix(pathsForTag[g.group])}\` | v1 |` + ) + .join('\n'); +const newIntro = intro.slice(0, s + INTRO_START.length) + '\n' + rows + '\n' + intro.slice(e); +writeOrCheck(INTRO_MDX, newIntro); + +// 8. Under --check, fail loudly if anything drifted from the committed files. +if (CHECK) { + if (staleFiles.length) { + console.error( + '\nOut of date with the source spec:\n ' + + staleFiles.join('\n ') + + '\n\nRun `node scripts/extract-api.mjs` (with SRC_SPEC set) and commit the result.' + ); + process.exit(1); + } + console.log('\nAPI reference is up to date. ✓'); } diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index cf1526fdf2754..cee55593de93f 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -2090,6 +2090,13 @@ class ApiGateway { }) ); + // A request may consist of several queries; the oldest refresh time + // defines the freshness of the whole result + const lastRefreshTimestamps = results + .map((r: any) => r.getRootResultObject()[0].lastRefreshTime) + .filter(Boolean) + .map((t: string) => new Date(t).getTime()); + this.log( { type: 'Load Request Success', @@ -2109,6 +2116,9 @@ class ApiGateway { // queriesWithData: // results.filter((r: any) => r.data?.length).length, dbType: results.map(r => r.getRootResultObject()[0].dbType), + lastRefreshTime: lastRefreshTimestamps.length + ? new Date(Math.min(...lastRefreshTimestamps)).toISOString() + : undefined, }, context, ); diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index a8553b617f305..496f7d8bedd3f 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -3297,9 +3297,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -3307,35 +3307,36 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 3.0.2", ] [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap 2.14.0", "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -3592,6 +3593,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn-mid" version = "0.6.0" @@ -4633,3 +4645,9 @@ dependencies = [ "quote", "syn 2.0.98", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/cubejs-backend-native/src/node_export.rs b/packages/cubejs-backend-native/src/node_export.rs index c5bdc1cc687d4..d45c386ce3aac 100644 --- a/packages/cubejs-backend-native/src/node_export.rs +++ b/packages/cubejs-backend-native/src/node_export.rs @@ -463,6 +463,7 @@ async fn handle_sql_query( "apiType": "sql", "duration": span_id.as_ref().unwrap().duration(), "isDataQuery": span_id.as_ref().unwrap().is_data_query().await, + "lastRefreshTime": span_id.as_ref().unwrap().last_refresh_time().await, }), ) .await?; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 05bccb946b3c3..e77f848b3fa94 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -513,6 +513,27 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface }, cubeDefinition); + if (cubeDefinition.isView) { + // Without this accessor, `Object.assign(cubeObject, cubeDefinition)` above would shadow + // `cubes` with the view's own un-merged, un-camelized `cubeDefinition.cubes`, bypassing + // `rawCubes()`. That breaks callers (e.g. `camelizeCube`) that read `cube.cubes` directly: + // for a view that `extends` another, the parent's cube-include entries would only get + // camelized (join_path -> joinPath) once the parent itself is transformed, making + // compilation depend on schema file processing order. `cubes` is only defined for views + // (see `viewSchema` in CubeValidator.ts), so this is scoped to `isView` to avoid adding + // an unexpected `cubes` key to plain cubes. + Object.defineProperty(cubeObject, 'cubes', { + enumerable: true, + configurable: true, + get(this: CubeDefinitionExtended) { + return this.rawCubes(); + }, + set(_v) { + // Dont allow to modify + }, + }); + } + if (cubeDefinition.extends) { const superCube = this.resolveSymbolsCall(cubeDefinition.extends, (name: string) => this.cubeReferenceProxy(name)); // eslint-disable-next-line no-underscore-dangle diff --git a/packages/cubejs-schema-compiler/test/unit/repro-11260.test.ts b/packages/cubejs-schema-compiler/test/unit/repro-11260.test.ts new file mode 100644 index 0000000000000..82d07b051f1f9 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/repro-11260.test.ts @@ -0,0 +1,76 @@ +import { prepareCompiler } from './PrepareCompiler'; + +describe('Repro #11260 - view extends order dependency', () => { + const testCubeContent = ` +cube(\`test\`, { + public: false, + sql: \`select 1 as id, 'test' as name\`, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\` + }, + name: { + sql: \`name\`, + type: \`string\` + } + } +}); +`; + + const viewAContent = ` +view(\`a_view\`, { + public: true, + cubes: [ + { + join_path: \`test\`, + includes: [ + "id", + "name" + ] + } + ] +}); +`; + + const viewBContent = ` +view(\`b_view\`, { + public: true, + extends: a_view, + cubes: [] +}); +`; + + const expectBViewInheritsDimensions = (metaTransformer: any) => { + const bViewMeta = metaTransformer.cubes.map((def: any) => def.config).find((def: any) => def.name === 'b_view'); + const dimensionNames = (bViewMeta.dimensions || []).map((d: any) => d.name).sort(); + expect(dimensionNames).toEqual(['b_view.id', 'b_view.name']); + }; + + it('compiles when base view (a_view) file comes before subview (b_view) file', async () => { + const { compiler, metaTransformer } = prepareCompiler([ + { content: testCubeContent, fileName: 'test.js' }, + { content: viewAContent, fileName: 'view_a.js' }, + { content: viewBContent, fileName: 'view_b.js' }, + ]); + + await compiler.compile(); + compiler.throwIfAnyErrors(); + + expectBViewInheritsDimensions(metaTransformer); + }); + + it('compiles when subview (b_view) file comes before base view (a_view) file', async () => { + const { compiler, metaTransformer } = prepareCompiler([ + { content: testCubeContent, fileName: 'test.js' }, + { content: viewBContent, fileName: 'view_b.js' }, + { content: viewAContent, fileName: 'view_a.js' }, + ]); + + await compiler.compile(); + compiler.throwIfAnyErrors(); + + expectBViewInheritsDimensions(metaTransformer); + }); +}); diff --git a/rust/cube/Cargo.lock b/rust/cube/Cargo.lock index b73647474e044..b61daabf1a4cd 100644 --- a/rust/cube/Cargo.lock +++ b/rust/cube/Cargo.lock @@ -455,6 +455,15 @@ dependencies = [ "serde_with", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "btoi" version = "0.4.3" @@ -3389,9 +3398,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -3425,11 +3434,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.18.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -3444,9 +3454,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.18.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock index 53bbbfce452ef..0116c719c5b52 100644 --- a/rust/cubesql/Cargo.lock +++ b/rust/cubesql/Cargo.lock @@ -2389,9 +2389,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -2820,35 +2820,46 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 3.0.2", ] [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap 2.4.0", "itoa 1.0.10", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -3102,6 +3113,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.1" @@ -4042,3 +4064,9 @@ dependencies = [ "quote", "syn 2.0.87", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs index 6d0790b00d640..929ab9409929c 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs @@ -10,7 +10,7 @@ use crate::{ CubeError, CubeErrorCauseType, }; use async_trait::async_trait; -use chrono::{Datelike, NaiveDate}; +use chrono::{DateTime, Datelike, NaiveDate, Utc}; use cubeclient::models::{ V1LoadRequestQuery, V1LoadResponse, V1LoadResult, V1LoadResultDataColumnar, }; @@ -793,7 +793,7 @@ async fn load_data( } else { let result = transport .load( - span_id, + span_id.clone(), request, sql_query, auth_context, @@ -822,6 +822,16 @@ async fn load_data( })?; if let Some(data) = result.into_iter().next() { + if let Some(span_id) = &span_id { + if let Some(last_refresh_time) = data.schema().metadata().get("lastRefreshTime") { + if let Ok(last_refresh_time) = DateTime::parse_from_rfc3339(last_refresh_time) { + span_id + .set_last_refresh_time(last_refresh_time.with_timezone(&Utc)) + .await; + } + } + } + match (options.max_records, data.num_rows()) { (Some(max_records), len) if len >= max_records => { return Err(ArrowError::ExternalError(Box::new(CubeError::user( diff --git a/rust/cubesql/cubesql/src/sql/postgres/shim.rs b/rust/cubesql/cubesql/src/sql/postgres/shim.rs index d487ff93b55df..1af7ba9a86950 100644 --- a/rust/cubesql/cubesql/src/sql/postgres/shim.rs +++ b/rust/cubesql/cubesql/src/sql/postgres/shim.rs @@ -382,7 +382,8 @@ impl AsyncPostgresShim { "query": span_id.query_key.clone(), "apiType": "sql", "duration": span_id.duration(), - "isDataQuery": span_id.is_data_query().await + "isDataQuery": span_id.is_data_query().await, + "lastRefreshTime": span_id.last_refresh_time().await }), ) .await?; @@ -1869,6 +1870,7 @@ impl AsyncPostgresShim { "apiType": "sql", "duration": start_time.elapsed().unwrap().as_millis() as u64, "isDataQuery": span_id.is_data_query().await, + "lastRefreshTime": span_id.last_refresh_time().await, }), ) .await?; diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index 509290d6223f5..1de989e4b2237 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use chrono::{DateTime, SecondsFormat, Utc}; use cubeclient::apis::{ configuration::Configuration as ClientConfiguration, default_api as cube_api, }; @@ -84,6 +85,7 @@ pub struct SpanId { pub query_key: serde_json::Value, span_start: SystemTime, is_data_query: RWLockAsync, + last_refresh_time: RWLockAsync>>, } impl SpanId { @@ -93,6 +95,7 @@ impl SpanId { query_key, span_start: SystemTime::now(), is_data_query: tokio::sync::RwLock::new(false), + last_refresh_time: tokio::sync::RwLock::new(None), } } @@ -106,6 +109,22 @@ impl SpanId { *read } + /// Records the refresh time of a load result contributing to this span. + /// A span may cover several load requests (e.g. a join of cube scans); + /// the oldest value wins, matching `lastRefreshTime` semantics elsewhere. + pub async fn set_last_refresh_time(&self, last_refresh_time: DateTime) { + let mut write = self.last_refresh_time.write().await; + *write = Some(match *write { + Some(current) => current.min(last_refresh_time), + None => last_refresh_time, + }); + } + + pub async fn last_refresh_time(&self) -> Option { + let read = self.last_refresh_time.read().await; + read.map(|t| t.to_rfc3339_opts(SecondsFormat::Millis, true)) + } + pub fn duration(&self) -> u64 { self.span_start .elapsed() @@ -1041,3 +1060,37 @@ impl SqlTemplates { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[tokio::test] + async fn span_id_last_refresh_time_keeps_oldest() { + let span_id = SpanId::new("test".to_string(), serde_json::json!({})); + assert_eq!(span_id.last_refresh_time().await, None); + + let newer = Utc.with_ymd_and_hms(2024, 6, 1, 12, 0, 0).unwrap(); + let older = Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(); + + span_id.set_last_refresh_time(newer).await; + assert_eq!( + span_id.last_refresh_time().await, + Some("2024-06-01T12:00:00.000Z".to_string()) + ); + + span_id.set_last_refresh_time(older).await; + assert_eq!( + span_id.last_refresh_time().await, + Some("2024-01-01T00:00:00.000Z".to_string()) + ); + + // A newer value must not override the recorded oldest one + span_id.set_last_refresh_time(newer).await; + assert_eq!( + span_id.last_refresh_time().await, + Some("2024-01-01T00:00:00.000Z".to_string()) + ); + } +} diff --git a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs index c5af7672b2674..7b8b4cc4856ff 100644 --- a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs +++ b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs @@ -488,6 +488,27 @@ impl RocksCacheStore { pub async fn check_all_indexes(&self) -> Result<(), CubeError> { RocksStore::check_all_indexes(&self.store).await } + + /// Schedules a no-op on both RW loops (default + queue) and awaits them, so that any + /// operation previously enqueued on those loops has finished and released its transient + /// Arc clone. NOTE: this only flushes the two RW loops; it does NOT wait for out-of-queue + /// readers (read_operation_out_of_queue), which run on detached spawn_blocking tasks with + /// their own Arc clone. LazyRocksCacheStore::wipe additionally waits on db_strong_count() + /// to cover those before closing the DB. + pub async fn drain_rw_loops(&self) -> Result<(), CubeError> { + self.store + .read_operation("wipe_barrier", |_| Ok(())) + .await?; + self.read_operation_queue("wipe_barrier", |_| Ok(())) + .await?; + + Ok(()) + } + + /// Number of strong references to the underlying RocksDB handle. + pub fn db_strong_count(&self) -> usize { + Arc::strong_count(&self.store.db) + } } impl RocksCacheStore { @@ -928,6 +949,9 @@ pub trait CacheStore: DIService + Send + Sync { async fn persist(&self) -> Result<(), CubeError>; async fn healthcheck(&self) -> Result<(), CubeError>; async fn rocksdb_properties(&self) -> Result, CubeError>; + // Wipe all cachestore state (cache + queue) and persist a fresh snapshot, updating the + // remote cachestore-current pointer so a poisoned snapshot is not re-hydrated on reload + async fn wipe(&self) -> Result<(), CubeError>; } #[async_trait] @@ -1683,6 +1707,15 @@ impl CacheStore for RocksCacheStore { async fn rocksdb_properties(&self) -> Result, CubeError> { self.store.rocksdb_properties() } + + async fn wipe(&self) -> Result<(), CubeError> { + // Wiping requires dropping and reopening the RocksDB from scratch, which a bare inner + // store cannot do to itself (it cannot rebuild its own Arc / swap the state). The + // registered production impl is always LazyRocksCacheStore, which owns the teardown. + Err(CubeError::internal( + "cachestore wipe is only supported through LazyRocksCacheStore".to_string(), + )) + } } crate::di_service!(RocksCacheStore, [CacheStore]); @@ -1835,6 +1868,10 @@ impl CacheStore for ClusterCacheStoreClient { async fn rocksdb_properties(&self) -> Result, CubeError> { panic!("CacheStore cannot be used on the worker node! rocksdb_properties was used.") } + + async fn wipe(&self) -> Result<(), CubeError> { + panic!("CacheStore cannot be used on the worker node! wipe was used.") + } } crate::di_service!(ClusterCacheStoreClient, [CacheStore]); diff --git a/rust/cubestore/cubestore/src/cachestore/lazy.rs b/rust/cubestore/cubestore/src/cachestore/lazy.rs index a5062cc55e47f..c4884563cad93 100644 --- a/rust/cubestore/cubestore/src/cachestore/lazy.rs +++ b/rust/cubestore/cubestore/src/cachestore/lazy.rs @@ -12,22 +12,34 @@ use crate::config::ConfigObj; use crate::metastore::{IdRow, MetaStoreEvent, MetaStoreFs, RocksPropertyRow}; use crate::CubeError; use async_trait::async_trait; -use log::trace; +use datafusion::cube_ext; use std::path::Path; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::sync::watch::{Receiver, Sender}; +use tokio::sync::Mutex; use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +const WIPE_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); +const WIPE_CLOSE_TIMEOUT: Duration = Duration::from_secs(30); +const WIPE_CLOSE_POLL_INTERVAL: Duration = Duration::from_millis(20); +const WIPE_REOPEN_TIMEOUT: Duration = Duration::from_secs(10); + +fn is_rocksdb_lock_error(err: &CubeError) -> bool { + let msg = err.to_string(); + msg.contains("lock hold by current process") + || msg.contains("While lock file") + || msg.contains("Resource temporarily unavailable") +} pub enum LazyRocksCacheStoreState { - FromRemote { - path: String, - metastore_fs: Arc, - config: Arc, - listeners: Vec>, - #[allow(dead_code)] // Receiver closed on drop + Initializing { + #[allow(dead_code)] // Receiver notified when this sender is dropped on transition init_flag: Sender, }, Closed {}, + Wiping {}, Initialized { store: Arc, }, @@ -36,6 +48,16 @@ pub enum LazyRocksCacheStoreState { pub struct LazyRocksCacheStore { init_signal: Option>, state: tokio::sync::RwLock, + // Kept on the wrapper (not only inside the state enum) so that `wipe` can rebuild the store + // from scratch regardless of the current state. + path: String, + metastore_fs: Arc, + config: Arc, + listeners: Vec>, + // Handles of the currently running processing loops. Owned here (rather than by CubeServices) + // so that `wipe` can stop and join them before dropping/reopening the RocksDB. + running_loops: Mutex>>>, + shutdown_token: CancellationToken, } impl LazyRocksCacheStore { @@ -46,15 +68,23 @@ impl LazyRocksCacheStore { config: Arc, listeners: Vec>, ) -> Result, CubeError> { - let store = RocksCacheStore::load_from_dump(path, dump_path, metastore_fs, config).await?; + let store = + RocksCacheStore::load_from_dump(path, dump_path, metastore_fs.clone(), config.clone()) + .await?; - for listener in listeners { - store.add_listener(listener).await; + for listener in &listeners { + store.add_listener(listener.clone()).await; } Ok(Arc::new(Self { init_signal: None, state: tokio::sync::RwLock::new(LazyRocksCacheStoreState::Initialized { store }), + path: path.to_string_lossy().to_string(), + metastore_fs, + config, + listeners, + running_loops: Mutex::new(vec![]), + shutdown_token: CancellationToken::new(), })) } @@ -68,13 +98,13 @@ impl LazyRocksCacheStore { Ok(Arc::new(Self { init_signal: Some(init_signal), - state: tokio::sync::RwLock::new(LazyRocksCacheStoreState::FromRemote { - path: path.to_string(), - metastore_fs, - config, - listeners, - init_flag, - }), + state: tokio::sync::RwLock::new(LazyRocksCacheStoreState::Initializing { init_flag }), + path: path.to_string(), + metastore_fs, + config, + listeners, + running_loops: Mutex::new(vec![]), + shutdown_token: CancellationToken::new(), })) } @@ -82,12 +112,18 @@ impl LazyRocksCacheStore { { let guard = self.state.read().await; match &*guard { - LazyRocksCacheStoreState::FromRemote { .. } => {} + LazyRocksCacheStoreState::Initializing { .. } => {} LazyRocksCacheStoreState::Closed { .. } => { return Err(CubeError::internal( "Unable to initialize Cache Store on lazy call, it was closed".to_string(), )); } + LazyRocksCacheStoreState::Wiping { .. } => { + return Err(CubeError::internal( + "Unable to initialize Cache Store on lazy call, a wipe is in progress" + .to_string(), + )); + } LazyRocksCacheStoreState::Initialized { store } => { return Ok(store.clone()); } @@ -96,78 +132,105 @@ impl LazyRocksCacheStore { let mut guard = self.state.write().await; match &*guard { - LazyRocksCacheStoreState::FromRemote { - path, - metastore_fs, - config, - listeners, - // receiver will be closed on drop - init_flag: _, - } => { - let store = - RocksCacheStore::load_from_remote(&path, metastore_fs.clone(), config.clone()) - .await?; - - for listener in listeners { + LazyRocksCacheStoreState::Initializing { .. } => { + let store = RocksCacheStore::load_from_remote( + &self.path, + self.metastore_fs.clone(), + self.config.clone(), + ) + .await?; + + for listener in &self.listeners { store.add_listener(listener.clone()).await; } + // Dropping the previous Initializing (via replace) drops init_flag, which + // notifies run_processing_loops to spawn the processing loops. *guard = LazyRocksCacheStoreState::Initialized { store: store.clone(), }; Ok(store) } - _ => Err(CubeError::internal( - "Unable to initialize Cache Store on lazy call, unexpected state".to_string(), + // Another caller initialized it between our read and write lock. + LazyRocksCacheStoreState::Initialized { store } => Ok(store.clone()), + LazyRocksCacheStoreState::Closed { .. } => Err(CubeError::internal( + "Unable to initialize Cache Store on lazy call, it was closed".to_string(), + )), + LazyRocksCacheStoreState::Wiping { .. } => Err(CubeError::internal( + "Unable to initialize Cache Store on lazy call, a wipe is in progress".to_string(), )), } } - pub async fn spawn_processing_loops(self: Arc) -> Vec>> { + /// Owns the processing-loop lifecycle for the cachestore. Called once by CubeServices; it + /// waits until the store is initialized, spawns the processing loops (stored in + /// `running_loops` so `wipe` can manage them), then blocks until shutdown. + pub async fn run_processing_loops(self: Arc) -> Result<(), CubeError> { if let Some(init_signal) = &self.init_signal { let _ = init_signal.clone().changed().await; } - let store = { - let guard = self.state.read().await; - if let LazyRocksCacheStoreState::Initialized { store } = &*guard { - store.clone() - } else { - return vec![]; + { + let store = { + let guard = self.state.read().await; + if let LazyRocksCacheStoreState::Initialized { store } = &*guard { + Some(store.clone()) + } else { + None + } + }; + + if let Some(store) = store { + let mut loops = self.running_loops.lock().await; + if loops.is_empty() { + *loops = store.spawn_processing_loops(); + } + // `store` clone is dropped at the end of this block so it does not keep the + // RocksDB alive while we are parked on the shutdown token. } - }; + } - trace!("start_processing_loops unblocked, Cache Store was initialized"); + self.shutdown_token.cancelled().await; - store.spawn_processing_loops() + Ok(()) } pub async fn stop_processing_loops(&self) { let store = { let mut guard = self.state.write().await; match &*guard { - LazyRocksCacheStoreState::Closed { .. } => { - return (); + LazyRocksCacheStoreState::Closed { .. } => None, + LazyRocksCacheStoreState::Initializing { .. } => { + *guard = LazyRocksCacheStoreState::Closed {}; + None } - LazyRocksCacheStoreState::FromRemote { .. } => { + LazyRocksCacheStoreState::Wiping { .. } => { + // A wipe owns the store; marking Closed makes the wipe abort its re-install + // (see wipe) instead of resurrecting the store after shutdown was requested. *guard = LazyRocksCacheStoreState::Closed {}; - - return (); + None } LazyRocksCacheStoreState::Initialized { store } => { let store_to_move = store.clone(); - *guard = LazyRocksCacheStoreState::Closed {}; - - store_to_move + Some(store_to_move) } } }; - trace!("stop_processing_loops unblocked, Cache Store was initialized"); + if let Some(store) = store { + store.stop_processing_loops().await; + } + + { + let mut loops = self.running_loops.lock().await; + for handle in loops.drain(..) { + let _ = handle.await; + } + } - store.stop_processing_loops().await + self.shutdown_token.cancel(); } } @@ -334,6 +397,371 @@ impl CacheStore for LazyRocksCacheStore { async fn rocksdb_properties(&self) -> Result, CubeError> { self.init().await?.rocksdb_properties().await } + + async fn wipe(&self) -> Result<(), CubeError> { + // Make sure the store is initialized. init() fires init_signal so run_processing_loops + // spawns the initial loops; the returned Arc is dropped immediately so it does not + // inflate the strong count the teardown relies on below. + self.init().await?; + + { + let mut guard = self.state.write().await; + match std::mem::replace(&mut *guard, LazyRocksCacheStoreState::Wiping {}) { + LazyRocksCacheStoreState::Initialized { store } => { + // Drop the guard before the teardown so the lock is not held across it. + drop(guard); + + // Everything from here is the point of no return: the loops are stopped and + // the store is destroyed, so it cannot be restored. + if let Err(err) = self.wipe_teardown(store).await { + let mut guard = self.state.write().await; + *guard = LazyRocksCacheStoreState::Closed {}; + drop(guard); + + if self.shutdown_token.is_cancelled() { + log::error!( + "SYSTEM CACHESTORE WIPE aborted during shutdown; cachestore left \ + closed: {}", + err + ); + } else { + log::error!( + "SYSTEM CACHESTORE WIPE failed after the teardown point of no \ + return; cachestore is now CLOSED and will reject every operation \ + until the node is restarted: {}", + err + ); + } + + return Err(err); + } + + Ok(()) + } + LazyRocksCacheStoreState::Wiping {} => { + *guard = LazyRocksCacheStoreState::Wiping {}; + Err(CubeError::internal( + "Unable to wipe Cache Store: a wipe is already in progress".to_string(), + )) + } + LazyRocksCacheStoreState::Closed {} => { + *guard = LazyRocksCacheStoreState::Closed {}; + Err(CubeError::internal( + "Unable to wipe Cache Store, it was closed".to_string(), + )) + } + LazyRocksCacheStoreState::Initializing { init_flag } => { + // init() above guarantees Initialized; restore defensively. + *guard = LazyRocksCacheStoreState::Initializing { init_flag }; + Err(CubeError::internal( + "Unable to wipe Cache Store, unexpected state".to_string(), + )) + } + } + } + } +} + +impl LazyRocksCacheStore { + async fn wipe_teardown(&self, store: Arc) -> Result<(), CubeError> { + // Stop the worker loops and JOIN them so their Arc clones are released. + // WorkerLoop/IntervalLoop tokens are permanently cancelled, so the old `store` can no + // longer run loops and there is no restore path. + store.stop_processing_loops().await; + { + let mut loops = self.running_loops.lock().await; + for handle in loops.drain(..) { + let _ = handle.await; + } + } + + // Flush both RW loops so in-flight serialized ops release their transient Arc clones. + // Bounded so a wedged RW loop cannot hang wipe forever. + if let Err(err) = tokio::time::timeout(WIPE_DRAIN_TIMEOUT, store.drain_rw_loops()).await { + log::warn!( + "Wiping cachestore: draining RW loops timed out ({:?}), continuing: {}", + WIPE_DRAIN_TIMEOUT, + err + ); + } + + // Wait until nothing else references the store or the underlying RocksDB, so that + // drop(store) closes the DB (releasing the directory LOCK) before we reopen. We watch + // the real Arc via db_strong_count() to catch detached out-of-queue spawn_blocking + // readers that the RW-loop drain above does not cover. Sleep-backoff, not a busy spin. + let deadline = Instant::now() + WIPE_CLOSE_TIMEOUT; + + while Arc::strong_count(&store) > 1 || store.db_strong_count() > 1 { + if Instant::now() >= deadline { + // We already committed (loops stopped); the store cannot be restored cleanly. + // Bail rather than risk removing/reopening the directory while the old DB is still + // open (which could corrupt the new DB); the caller marks the state Closed. + return Err(CubeError::internal( + "Unable to wipe Cache Store: still in use after draining; restart the node" + .to_string(), + )); + } + + log::warn!( + "Wiping cachestore: waiting for outstanding Cache Store references to be released before closing the DB (store strong count: {}, DB strong count: {})", + Arc::strong_count(&store), + store.db_strong_count() + ); + + tokio::time::sleep(WIPE_CLOSE_POLL_INTERVAL).await; + } + + // Close the DB (drops the last Arc) synchronously. + drop(store); + + // Remove the folder and reopen from scratch off the async executor (blocking syscalls + + // RocksDB open). Retry the open on a held directory LOCK to cover any residual detached + // Arc that outlived the wait above. + let path = self.path.clone(); + let metastore_fs = self.metastore_fs.clone(); + let config = self.config.clone(); + + let fresh = cube_ext::spawn_blocking(move || -> Result, CubeError> { + if let Err(err) = std::fs::remove_dir_all(&path) { + if err.kind() != std::io::ErrorKind::NotFound { + return Err(CubeError::internal(format!( + "Unable to wipe Cache Store: failed to remove {}: {}", + path, err + ))); + } + } + + let reopen_deadline = Instant::now() + WIPE_REOPEN_TIMEOUT; + + loop { + match RocksCacheStore::new(Path::new(&path), metastore_fs.clone(), config.clone()) { + Ok(store) => return Ok(store), + Err(err) if is_rocksdb_lock_error(&err) && Instant::now() < reopen_deadline => { + std::thread::sleep(Duration::from_millis(50)); + } + Err(err) => return Err(err), + } + } + }) + .await??; + + fresh.check_all_indexes().await?; + + for listener in &self.listeners { + fresh.add_listener(listener.clone()).await; + } + + { + let mut guard = self.state.write().await; + + if matches!(&*guard, LazyRocksCacheStoreState::Closed { .. }) { + return Err(CubeError::internal( + "WIPE aborted: shutdown requested during teardown".to_string(), + )); + } + + // Respawn the worker loops against the fresh store, then install it and release the + // state lock BEFORE the (remote, slow, fallible) snapshot upload. + { + let mut loops = self.running_loops.lock().await; + *loops = fresh.clone().spawn_processing_loops(); + } + + *guard = LazyRocksCacheStoreState::Initialized { + store: fresh.clone(), + }; + } + + // Persist a fresh full snapshot and move the remote cachestore-current pointer onto it. + // Best-effort: the store is already installed and usable, and the upload loop retries; + // a transient remote error must not brick the cachestore. + if let Err(err) = fresh.upload_check_point().await { + log::warn!( + "Wiped cachestore reopened, but persisting the fresh snapshot failed (upload loop will retry): {}", + err + ); + } + + Ok(()) + } } crate::di_service!(LazyRocksCacheStore, [CacheStore]); + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{init_test_logger, Config}; + use crate::metastore::BaseRocksStoreFs; + use crate::remotefs::LocalDirRemoteFs; + use std::env; + use std::path::PathBuf; + + fn test_dirs(test_name: &str) -> (PathBuf, PathBuf) { + let base = env::current_dir().unwrap().join("db-tmp").join("tests"); + ( + base.join(format!("{}-local", test_name)), + base.join(format!("{}-remote", test_name)), + ) + } + + async fn build_lazy( + config: Config, + local_path: PathBuf, + remote_path: PathBuf, + ) -> Arc { + let config_obj = config.config_obj(); + let remote_fs = LocalDirRemoteFs::new(Some(remote_path), local_path.clone()); + let cachestore_path = local_path.join("cachestore"); + let metastore_fs = BaseRocksStoreFs::new_for_cachestore(remote_fs, config_obj.clone()); + + LazyRocksCacheStore::load_from_remote( + cachestore_path.to_str().unwrap(), + metastore_fs, + config_obj, + vec![], + ) + .await + .unwrap() + } + + #[tokio::test] + async fn test_lazy_wipe_clears_state_and_stays_usable() -> Result<(), CubeError> { + init_test_logger().await; + + let (local, remote) = test_dirs("lazy_wipe_clears"); + let _ = std::fs::remove_dir_all(&local); + let _ = std::fs::remove_dir_all(&remote); + + let cachestore = build_lazy( + Config::test("lazy_wipe_clears"), + local.clone(), + remote.clone(), + ) + .await; + + cachestore + .cache_set( + CacheItem::new("prefix:key1".to_string(), Some(60), "v1".to_string()), + false, + ) + .await?; + cachestore + .cache_set( + CacheItem::new("prefix:key2".to_string(), Some(60), "v2".to_string()), + false, + ) + .await?; + cachestore + .queue_add(QueueAddPayload { + path: "queue:job1".to_string(), + value: "payload".to_string(), + priority: 0, + orphaned: None, + process_id: None, + exclusive: false, + external_id: None, + }) + .await?; + + assert_eq!(cachestore.cache_all(None).await?.len(), 2); + assert_eq!(cachestore.queue_all(None).await?.len(), 1); + + cachestore.wipe().await?; + + assert_eq!(cachestore.cache_all(None).await?.len(), 0); + assert_eq!(cachestore.queue_all(None).await?.len(), 0); + + // The reopened store must still be usable. + cachestore + .cache_set( + CacheItem::new("prefix:after".to_string(), Some(60), "fresh".to_string()), + false, + ) + .await?; + let row = cachestore + .cache_get("prefix:after".to_string()) + .await? + .expect("must return row after wipe"); + assert_eq!(row.into_row().value, "fresh".to_string()); + + // The local cachestore folder must exist again after the reopen. + assert!(local.join("cachestore").exists()); + + // Stop the loops respawned by wipe so the test does not leak background tasks. + cachestore.stop_processing_loops().await; + + let _ = std::fs::remove_dir_all(&local); + let _ = std::fs::remove_dir_all(&remote); + + Ok(()) + } + + #[tokio::test] + async fn test_lazy_wipe_publishes_clean_remote_snapshot() -> Result<(), CubeError> { + init_test_logger().await; + + let (local, remote) = test_dirs("lazy_wipe_remote"); + let local_pre = local.parent().unwrap().join("lazy_wipe_remote-local-pre"); + let local2 = local.parent().unwrap().join("lazy_wipe_remote-local2"); + for dir in [&local, &remote, &local_pre, &local2] { + let _ = std::fs::remove_dir_all(dir); + } + + let cachestore = build_lazy( + Config::test("lazy_wipe_remote"), + local.clone(), + remote.clone(), + ) + .await; + + cachestore + .cache_set( + CacheItem::new("prefix:key1".to_string(), Some(60), "v1".to_string()), + false, + ) + .await?; + + // Upload the seed to the remote so the test actually proves that wipe RESETS the remote + // (rather than that an empty remote yields an empty store). init() returns the inner + // store; the returned Arc is scoped so it is dropped before wipe (else it would inflate + // the strong count wipe waits on). + { + let store = cachestore.init().await?; + store.upload_check_point().await?; + } + + // A fresh instance over the same remote must now see the seeded row (pre-wipe state). + { + let reloaded = build_lazy( + Config::test("lazy_wipe_remote"), + local_pre.clone(), + remote.clone(), + ) + .await; + assert_eq!(reloaded.cache_all(None).await?.len(), 1); + reloaded.stop_processing_loops().await; + } + + cachestore.wipe().await?; + + // After wipe, a fresh instance over the same remote must download the post-wipe snapshot + // via cachestore-current and come back empty. + let reloaded = build_lazy( + Config::test("lazy_wipe_remote"), + local2.clone(), + remote.clone(), + ) + .await; + assert_eq!(reloaded.cache_all(None).await?.len(), 0); + reloaded.stop_processing_loops().await; + + cachestore.stop_processing_loops().await; + + for dir in [&local, &remote, &local_pre, &local2] { + let _ = std::fs::remove_dir_all(dir); + } + + Ok(()) + } +} diff --git a/rust/cubestore/cubestore/src/config/mod.rs b/rust/cubestore/cubestore/src/config/mod.rs index f8cc36a0c92de..c33f66b2666b2 100644 --- a/rust/cubestore/cubestore/src/config/mod.rs +++ b/rust/cubestore/cubestore/src/config/mod.rs @@ -135,11 +135,7 @@ impl CubeServices { let rocks_cache_store = self.rocks_cache_store.clone().unwrap(); futures.push(cube_ext::spawn(async move { - let loops = rocks_cache_store.spawn_processing_loops().await; - - Self::wait_loops(loops).await?; - - Ok(()) + rocks_cache_store.run_processing_loops().await })); let cluster = self.cluster.clone(); diff --git a/rust/cubestore/cubestore/src/queryplanner/test_utils.rs b/rust/cubestore/cubestore/src/queryplanner/test_utils.rs index dc14d74226d3a..1765bef2feacf 100644 --- a/rust/cubestore/cubestore/src/queryplanner/test_utils.rs +++ b/rust/cubestore/cubestore/src/queryplanner/test_utils.rs @@ -923,6 +923,10 @@ impl CacheStore for CacheStoreMock { async fn rocksdb_properties(&self) -> Result, CubeError> { panic!("CacheStore mock!") } + + async fn wipe(&self) -> Result<(), CubeError> { + panic!("CacheStore mock!") + } } crate::di_service!(CacheStoreMock, [CacheStore]); diff --git a/rust/cubestore/cubestore/src/sql/cachestore.rs b/rust/cubestore/cubestore/src/sql/cachestore.rs index af78d45e5aeb7..d1848be69f864 100644 --- a/rust/cubestore/cubestore/src/sql/cachestore.rs +++ b/rust/cubestore/cubestore/src/sql/cachestore.rs @@ -175,6 +175,13 @@ impl CacheStoreSqlService { self.cachestore.healthcheck().await?; Ok(Arc::new(DataFrame::new(vec![], vec![]))) } + CacheStoreCommand::Wipe => { + log::warn!( + "Wiping cachestore state (SYSTEM CACHESTORE WIPE): truncating all tables and persisting a fresh snapshot" + ); + self.cachestore.wipe().await?; + Ok(Arc::new(DataFrame::new(vec![], vec![]))) + } } } diff --git a/rust/cubestore/cubestore/src/sql/parser.rs b/rust/cubestore/cubestore/src/sql/parser.rs index 3ebc4ef1651a3..156813e6029e3 100644 --- a/rust/cubestore/cubestore/src/sql/parser.rs +++ b/rust/cubestore/cubestore/src/sql/parser.rs @@ -218,6 +218,7 @@ pub enum CacheStoreCommand { Eviction, Info, Persist, + Wipe, } type QueryParameterHolder = Option; @@ -605,6 +606,8 @@ impl<'a> CubeStoreParser<'a> { CacheStoreCommand::Info } else if self.parse_custom_token("healthcheck") { CacheStoreCommand::Healthcheck + } else if self.parse_custom_token("wipe") { + CacheStoreCommand::Wipe } else { return Err(ParserError::ParserError( "Unknown cachestore command".to_string(),