From a91fa5a5e451bf2b645713139b4f8f618c6e76ee Mon Sep 17 00:00:00 2001 From: Jon Langevin Date: Tue, 30 Jun 2026 07:06:02 -0400 Subject: [PATCH] refactor(orchestrator): remove 7 bespoke auth/secret steps (Phase 3 consolidation) Removes step.bcrypt_check/hash, jwt_generate/decode, oauth_exchange, secret_manage, vault_config from RatchetPlugin (impls + StepFactories + Manifest.StepTypes). Their functionality is consolidated onto the canonical surface: bcrypt/jwt -> workflow-plugin-auth (existing steps, in-process entry PR1); secret_manage get/set -> engine secret_fetch/secret_set, list/delete -> workflow-plugin-infra (PR3); vault declare -> engine secrets.vault module, status/test -> infra. jwt_generate/jwt_decode dropped (0 ratchet refs, YAGNI). Remediation text in security_audit.go updated to point at the new canonical steps (no dangling refs). Served gRPC union unchanged (7 steps); in-process RatchetPlugin now 19 steps. ratchet on old pin v0.9.1 unaffected until PR7 rewires to released v0.11.0. Shared helpers preserved: vaultConfigDir() relocated to vault_config.go (still used by secretsGuardHook P13 token redaction); extractBody() inlined into its single remaining caller step_provider_models.go. SecretGuardService LoadSecrets/LoadAllSecrets/Provider retained (lazy-resolve redaction path + optional webhook/human_request consumers). Co-Authored-By: Claude --- orchestrator/plugin.go | 9 +- orchestrator/ratchetplugin_test.go | 5 +- orchestrator/secret_guard_lazy_test.go | 2 +- orchestrator/security_audit.go | 6 +- orchestrator/services.go | 7 +- orchestrator/step_bcrypt.go | 66 ---- orchestrator/step_jwt_decode.go | 107 ------ orchestrator/step_jwt_generate.go | 77 ---- orchestrator/step_oauth_exchange.go | 165 --------- orchestrator/step_provider_models.go | 7 +- orchestrator/step_secret_manage.go | 183 ---------- orchestrator/step_vault_config.go | 464 ------------------------- orchestrator/vault_config.go | 13 + 13 files changed, 31 insertions(+), 1080 deletions(-) delete mode 100644 orchestrator/step_bcrypt.go delete mode 100644 orchestrator/step_jwt_decode.go delete mode 100644 orchestrator/step_jwt_generate.go delete mode 100644 orchestrator/step_oauth_exchange.go delete mode 100644 orchestrator/step_secret_manage.go delete mode 100644 orchestrator/step_vault_config.go diff --git a/orchestrator/plugin.go b/orchestrator/plugin.go index c40315f..5cb18c2 100644 --- a/orchestrator/plugin.go +++ b/orchestrator/plugin.go @@ -41,7 +41,7 @@ func New() *RatchetPlugin { Author: "GoCodeAlone", Description: "Ratchet autonomous agent orchestration plugin", ModuleTypes: []string{"agent.provider", "ratchet.sse_hub", "ratchet.scheduler", "ratchet.mcp_client", "ratchet.mcp_server", "authz.casbin", "agent.guardrails"}, - StepTypes: []string{"step.agent_execute", "step.provider_test", "step.provider_models", "step.model_pull", "step.workspace_init", "step.container_control", "step.secret_manage", "step.vault_config", "step.mcp_reload", "step.oauth_exchange", "step.approval_resolve", "step.webhook_process", "step.security_audit", "step.test_interact", "step.human_request_resolve", "step.memory_extract", "step.bcrypt_check", "step.bcrypt_hash", "step.jwt_generate", "step.jwt_decode", "step.blackboard_post", "step.blackboard_read", "step.self_improve_validate", "step.self_improve_diff", "step.self_improve_deploy", "step.lsp_diagnose"}, + StepTypes: []string{"step.agent_execute", "step.provider_test", "step.provider_models", "step.model_pull", "step.workspace_init", "step.container_control", "step.mcp_reload", "step.approval_resolve", "step.webhook_process", "step.security_audit", "step.test_interact", "step.human_request_resolve", "step.memory_extract", "step.blackboard_post", "step.blackboard_read", "step.self_improve_validate", "step.self_improve_diff", "step.self_improve_deploy", "step.lsp_diagnose"}, WiringHooks: []string{"agent.provider_registry", "ratchet.sse_route_registration", "ratchet.mcp_server_route_registration", "ratchet.db_init", "ratchet.auth_token", "ratchet.secrets_guard", "ratchet.provider_registry", "ratchet.tool_policy_engine", "ratchet.sub_agent_manager", "ratchet.tool_registry", "ratchet.container_manager", "ratchet.transcript_recorder", "ratchet.skill_manager", "ratchet.approval_manager", "ratchet.human_request_manager", "ratchet.webhook_manager", "ratchet.security_auditor", "ratchet.browser_manager", "ratchet.test_interaction", "ratchet.blackboard", "ratchet.mcp_tools_wiring"}, }, }, @@ -83,20 +83,13 @@ func (p *RatchetPlugin) StepFactories() map[string]plugin.StepFactory { "step.model_pull": agentplugin.NewModelPullStepFactory(), "step.workspace_init": newWorkspaceInitFactory(), "step.container_control": newContainerControlFactory(), - "step.secret_manage": newSecretManageFactory(), - "step.vault_config": newVaultConfigFactory(), "step.mcp_reload": newMCPReloadFactory(), - "step.oauth_exchange": newOAuthExchangeFactory(), "step.approval_resolve": newApprovalResolveFactory(), "step.webhook_process": newWebhookProcessStepFactory(), "step.security_audit": newSecurityAuditFactory(), "step.test_interact": newTestInteractFactory(), "step.human_request_resolve": newHumanRequestResolveFactory(), "step.memory_extract": newMemoryExtractFactory(), - "step.bcrypt_check": newBcryptCheckFactory(), - "step.bcrypt_hash": newBcryptHashFactory(), - "step.jwt_generate": newJWTGenerateFactory(), - "step.jwt_decode": newJWTDecodeFactory(), "step.blackboard_post": newBlackboardPostFactory(), "step.blackboard_read": newBlackboardReadFactory(), "step.self_improve_validate": newSelfImproveValidateFactory(), diff --git a/orchestrator/ratchetplugin_test.go b/orchestrator/ratchetplugin_test.go index 79ff441..1a958f4 100644 --- a/orchestrator/ratchetplugin_test.go +++ b/orchestrator/ratchetplugin_test.go @@ -1113,13 +1113,10 @@ func TestPlugin_StepFactories(t *testing.T) { expected := []string{ "step.agent_execute", "step.provider_test", "step.provider_models", "step.model_pull", "step.workspace_init", "step.container_control", - "step.secret_manage", "step.vault_config", - "step.mcp_reload", "step.oauth_exchange", + "step.mcp_reload", "step.approval_resolve", "step.webhook_process", "step.security_audit", "step.test_interact", "step.human_request_resolve", "step.memory_extract", - "step.bcrypt_check", "step.bcrypt_hash", - "step.jwt_generate", "step.jwt_decode", "step.authz_check_casbin", "step.authz_add_policy", "step.authz_remove_policy", "step.authz_role_assign", "step.blackboard_post", "step.blackboard_read", diff --git a/orchestrator/secret_guard_lazy_test.go b/orchestrator/secret_guard_lazy_test.go index 3693a28..7db46c8 100644 --- a/orchestrator/secret_guard_lazy_test.go +++ b/orchestrator/secret_guard_lazy_test.go @@ -277,7 +277,7 @@ func TestProviderRegistryNilAccessorNoPanic(t *testing.T) { } // TestProviderRegistryUpdateSecretsProviderOverridesLazy proves the runtime -// hot-swap (UpdateSecretsProvider, used by step.vault_config) takes precedence +// hot-swap (UpdateSecretsProvider on the ProviderRegistry) takes precedence // over the lazy guard accessor: after the swap, resolution uses the new provider // and the lazy accessor is no longer consulted. func TestProviderRegistryUpdateSecretsProviderOverridesLazy(t *testing.T) { diff --git a/orchestrator/security_audit.go b/orchestrator/security_audit.go index 38cbfd7..96c065e 100644 --- a/orchestrator/security_audit.go +++ b/orchestrator/security_audit.go @@ -212,7 +212,7 @@ func (c *ProviderSecurityCheck) Run(ctx context.Context) []AuditFinding { Severity: SeverityHigh, Title: fmt.Sprintf("Provider %q has no secret configured", alias), Description: fmt.Sprintf("AI provider %q (type: %s) does not reference a vault secret for its API key.", alias, provType), - Remediation: "Configure the provider's API key via the vault using step.secret_manage.", + Remediation: "Configure the provider's API key via the vault using step.secret_set.", }) } } @@ -235,7 +235,7 @@ func (c *ProviderSecurityCheck) Run(ctx context.Context) []AuditFinding { Severity: SeverityMedium, Title: fmt.Sprintf("Sensitive credential in environment variable %q", key), Description: "Credentials should be stored in the vault, not in environment variables.", - Remediation: "Move this credential to the vault using step.secret_manage and remove the env var.", + Remediation: "Move this credential to the vault using step.secret_set and remove the env var.", }) break } @@ -329,7 +329,7 @@ func (c *VaultCheck) Run(_ context.Context) []AuditFinding { Severity: SeverityCritical, Title: "Vault-dev backend in use in production", Description: "The development vault backend is not suitable for production use. It stores secrets in memory and is not persistent.", - Remediation: "Configure a remote HashiCorp Vault instance using step.vault_config with action: configure.", + Remediation: "Configure a remote HashiCorp Vault instance by declaring the engine secrets.vault module in config (and verify with step.secret_vault_status).", }) } else if hasVaultDev { findings = append(findings, AuditFinding{ diff --git a/orchestrator/services.go b/orchestrator/services.go index 6dd38e6..35c1c1f 100644 --- a/orchestrator/services.go +++ b/orchestrator/services.go @@ -152,10 +152,15 @@ func (a toolRegistryAdapter) SetPaginator(rp *ResponsePaginator) { // SecretGuardService is the orchestrator-scoped secret redaction / provider. // // Consumers: -// - step_secret_manage (REQUIRED-STATEFUL — Provider/LoadSecrets/LoadAllSecrets). // - step_human_request (TRULY-OPTIONAL — autoStoreSecret helper only). // - step_webhook (TRULY-OPTIONAL — signature-verification secret lookup). // - step_agent_execute (TRULY-OPTIONAL — CheckAndRedact/Redact on messages). +// +// Phase 3 (PR5): the bespoke step_secret_manage/vault_config consumers were +// removed; secret mutations now flow through the engine step.secret_set / +// step.secret_fetch + workflow-plugin-infra secret-admin steps. The Provider / +// LoadSecrets / LoadAllSecrets methods are retained for the lazy-resolve +// redaction path and the optional consumers above. type SecretGuardService interface { LoadSecrets(ctx context.Context, names []string) error LoadAllSecrets(ctx context.Context) error diff --git a/orchestrator/step_bcrypt.go b/orchestrator/step_bcrypt.go deleted file mode 100644 index e4f09d1..0000000 --- a/orchestrator/step_bcrypt.go +++ /dev/null @@ -1,66 +0,0 @@ -package orchestrator - -import ( - "context" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" - "golang.org/x/crypto/bcrypt" -) - -// BcryptCheckStep compares a password against a bcrypt hash. -// Reads "password" and "password_hash" from pc.Current. -// Returns {match: true/false}. -type BcryptCheckStep struct { - name string -} - -func (s *BcryptCheckStep) Name() string { return s.name } - -func (s *BcryptCheckStep) Execute(_ context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - password, _ := pc.Current["password"].(string) - hash, _ := pc.Current["password_hash"].(string) - - match := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) == nil - - return &module.StepResult{ - Output: map[string]any{"match": match}, - }, nil -} - -func newBcryptCheckFactory() plugin.StepFactory { - return func(name string, _ map[string]any, _ modular.Application) (any, error) { - return &BcryptCheckStep{name: name}, nil - } -} - -// BcryptHashStep hashes a password with bcrypt. -// Reads "password" from pc.Current. -// Returns {hash: "$2a$..."}. -type BcryptHashStep struct { - name string -} - -func (s *BcryptHashStep) Name() string { return s.name } - -func (s *BcryptHashStep) Execute(_ context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - password, _ := pc.Current["password"].(string) - - hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - return &module.StepResult{ - Output: map[string]any{"error": err.Error()}, - }, nil - } - - return &module.StepResult{ - Output: map[string]any{"hash": string(hash)}, - }, nil -} - -func newBcryptHashFactory() plugin.StepFactory { - return func(name string, _ map[string]any, _ modular.Application) (any, error) { - return &BcryptHashStep{name: name}, nil - } -} diff --git a/orchestrator/step_jwt_decode.go b/orchestrator/step_jwt_decode.go deleted file mode 100644 index 073d188..0000000 --- a/orchestrator/step_jwt_decode.go +++ /dev/null @@ -1,107 +0,0 @@ -package orchestrator - -import ( - "context" - "net/http" - "strings" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" - "github.com/golang-jwt/jwt/v5" -) - -// JWTDecodeStep decodes a JWT token from the Authorization header. -// Tries multiple sources: HTTP request metadata, Current, trigger data. -// Returns decoded claims: {sub, username, role, authenticated}. -type JWTDecodeStep struct { - name string - secret []byte -} - -func (s *JWTDecodeStep) Name() string { return s.name } - -func (s *JWTDecodeStep) Execute(_ context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - authHeader := "" - - // Try HTTP request from metadata first (most reliable) - if req, ok := pc.Metadata["_http_request"].(*http.Request); ok && req != nil { - authHeader = req.Header.Get("Authorization") - } - - // Fall back to Current (set by step.set from request_parse) - if authHeader == "" { - if auth, ok := pc.Current["authorization"].(string); ok { - authHeader = auth - } - } - - // Fall back to trigger data - if authHeader == "" { - if headers, ok := pc.TriggerData["headers"].(map[string]any); ok { - if auth, ok := headers["Authorization"].(string); ok { - authHeader = auth - } - } - } - - tokenStr := strings.TrimPrefix(authHeader, "Bearer ") - if tokenStr == "" || tokenStr == authHeader { - return &module.StepResult{ - Output: map[string]any{"error": "no bearer token", "authenticated": false}, - }, nil - } - - token, err := jwt.Parse(tokenStr, func(_ *jwt.Token) (any, error) { - return s.secret, nil - }) - if err != nil || !token.Valid { - return &module.StepResult{ - Output: map[string]any{"error": "invalid token", "authenticated": false}, - }, nil - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - return &module.StepResult{ - Output: map[string]any{"error": "invalid claims", "authenticated": false}, - }, nil - } - - output := map[string]any{"authenticated": true} - if sub, ok := claims["sub"].(string); ok { - output["sub"] = sub - } - if username, ok := claims["username"].(string); ok { - output["username"] = username - } - if role, ok := claims["role"].(string); ok { - output["role"] = role - } - - return &module.StepResult{Output: output}, nil -} - -func newJWTDecodeFactory() plugin.StepFactory { - return func(name string, config map[string]any, app modular.Application) (any, error) { - secret := "" - if s, ok := config["secret"].(string); ok && s != "" { - secret = s - } - if secret == "" { - if cp, ok := app.SvcRegistry()["config-provider"]; ok { - if getter, ok := cp.(interface{ Get(string) string }); ok { - secret = getter.Get("auth_secret") - } - } - } - if secret == "" { - secret = "ratchet-dev-secret-change-me" - } - - return &JWTDecodeStep{ - name: name, - secret: []byte(secret), - }, nil - } -} diff --git a/orchestrator/step_jwt_generate.go b/orchestrator/step_jwt_generate.go deleted file mode 100644 index 2ae6367..0000000 --- a/orchestrator/step_jwt_generate.go +++ /dev/null @@ -1,77 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "time" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" - "github.com/golang-jwt/jwt/v5" -) - -// JWTGenerateStep creates a signed JWT token. -// Reads "user_id", "username", "role" from pc.Current. -// Returns {token: "eyJ..."}. -type JWTGenerateStep struct { - name string - secret []byte -} - -func (s *JWTGenerateStep) Name() string { return s.name } - -func (s *JWTGenerateStep) Execute(_ context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - userID, _ := pc.Current["user_id"].(string) - username, _ := pc.Current["username"].(string) - role, _ := pc.Current["role"].(string) - - now := time.Now() - claims := jwt.MapClaims{ - "sub": userID, - "username": username, - "role": role, - "iat": now.Unix(), - "exp": now.Add(24 * time.Hour).Unix(), - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - tokenString, err := token.SignedString(s.secret) - if err != nil { - return &module.StepResult{ - Output: map[string]any{"error": fmt.Sprintf("jwt sign: %v", err)}, - }, nil - } - - return &module.StepResult{ - Output: map[string]any{"token": tokenString}, - }, nil -} - -func newJWTGenerateFactory() plugin.StepFactory { - return func(name string, config map[string]any, app modular.Application) (any, error) { - // Try to get secret from step config first, then from config provider - secret := "" - if s, ok := config["secret"].(string); ok && s != "" { - secret = s - } - - // Fall back to config provider - if secret == "" { - if cp, ok := app.SvcRegistry()["config-provider"]; ok { - if getter, ok := cp.(interface{ Get(string) string }); ok { - secret = getter.Get("auth_secret") - } - } - } - - if secret == "" { - secret = "ratchet-dev-secret-change-me" - } - - return &JWTGenerateStep{ - name: name, - secret: []byte(secret), - }, nil - } -} diff --git a/orchestrator/step_oauth_exchange.go b/orchestrator/step_oauth_exchange.go deleted file mode 100644 index c93d838..0000000 --- a/orchestrator/step_oauth_exchange.go +++ /dev/null @@ -1,165 +0,0 @@ -package orchestrator - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" -) - -const ( - anthropicOAuthTokenURL = "https://console.anthropic.com/v1/oauth/token" - anthropicCreateAPIKeyURL = "https://api.anthropic.com/api/oauth/claude_cli/create_api_key" - anthropicOAuthClientID = "9d1c250a-e61b-44d9-88ed-5944d1962f5e" - oauthHTTPTimeout = 15 * time.Second -) - -// OAuthExchangeStep handles OAuth authorization code exchange server-side, -// proxying to provider OAuth endpoints to avoid CORS issues in the browser. -type OAuthExchangeStep struct { - name string - app modular.Application -} - -func (s *OAuthExchangeStep) Name() string { return s.name } - -func (s *OAuthExchangeStep) Execute(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - body := extractBody(pc) - providerType := extractString(body, "provider", "") - code := extractString(body, "code", "") - codeVerifier := extractString(body, "code_verifier", "") - redirectURI := extractString(body, "redirect_uri", "") - - switch providerType { - case "anthropic": - result, err := exchangeAnthropicOAuth(ctx, code, codeVerifier, redirectURI) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": err.Error(), - }, - }, nil - } - return &module.StepResult{Output: result}, nil - default: - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "unsupported provider", - }, - }, nil - } -} - -// exchangeAnthropicOAuth performs the two-step Anthropic OAuth exchange: -// 1. Exchange auth code for access token -// 2. Use access token to create a permanent API key -func exchangeAnthropicOAuth(ctx context.Context, code, codeVerifier, redirectURI string) (map[string]any, error) { - client := &http.Client{Timeout: oauthHTTPTimeout} - - // Step 1: Exchange code for access token - tokenReqBody, err := json.Marshal(map[string]any{ - "code": code, - "state": codeVerifier, - "grant_type": "authorization_code", - "client_id": anthropicOAuthClientID, - "redirect_uri": redirectURI, - "code_verifier": codeVerifier, - }) - if err != nil { - return nil, fmt.Errorf("marshal token request: %w", err) - } - - tokenReq, err := http.NewRequestWithContext(ctx, http.MethodPost, anthropicOAuthTokenURL, bytes.NewReader(tokenReqBody)) - if err != nil { - return nil, fmt.Errorf("create token request: %w", err) - } - tokenReq.Header.Set("Content-Type", "application/json") - - tokenResp, err := client.Do(tokenReq) - if err != nil { - return nil, fmt.Errorf("token request failed: %w", err) - } - defer func() { _ = tokenResp.Body.Close() }() - - tokenBody, err := io.ReadAll(tokenResp.Body) - if err != nil { - return nil, fmt.Errorf("read token response: %w", err) - } - - if tokenResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("token endpoint error (status %d): %s", tokenResp.StatusCode, truncateOAuth(string(tokenBody), 200)) - } - - var tokenResult map[string]any - if err := json.Unmarshal(tokenBody, &tokenResult); err != nil { - return nil, fmt.Errorf("parse token response: %w", err) - } - - accessToken, _ := tokenResult["access_token"].(string) - if accessToken == "" { - return nil, fmt.Errorf("no access_token in response") - } - - // Step 2: Create a permanent API key using the access token - apiKeyReq, err := http.NewRequestWithContext(ctx, http.MethodPost, anthropicCreateAPIKeyURL, http.NoBody) - if err != nil { - return nil, fmt.Errorf("create api key request: %w", err) - } - apiKeyReq.Header.Set("Authorization", "Bearer "+accessToken) - apiKeyReq.Header.Set("Content-Type", "application/json") - - apiKeyResp, err := client.Do(apiKeyReq) - if err != nil { - return nil, fmt.Errorf("api key request failed: %w", err) - } - defer func() { _ = apiKeyResp.Body.Close() }() - - apiKeyBody, err := io.ReadAll(apiKeyResp.Body) - if err != nil { - return nil, fmt.Errorf("read api key response: %w", err) - } - - if apiKeyResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("api key endpoint error (status %d): %s", apiKeyResp.StatusCode, truncateOAuth(string(apiKeyBody), 200)) - } - - var apiKeyResult map[string]any - if err := json.Unmarshal(apiKeyBody, &apiKeyResult); err != nil { - return nil, fmt.Errorf("parse api key response: %w", err) - } - - rawKey, _ := apiKeyResult["raw_key"].(string) - if rawKey == "" { - return nil, fmt.Errorf("no raw_key in api key response") - } - - return map[string]any{ - "success": true, - "api_key": rawKey, - }, nil -} - -func truncateOAuth(s string, maxLen int) string { - if len(s) <= maxLen { - return s - } - return s[:maxLen] + "..." -} - -func newOAuthExchangeFactory() plugin.StepFactory { - return func(name string, _ map[string]any, app modular.Application) (any, error) { - return &OAuthExchangeStep{ - name: name, - app: app, - }, nil - } -} diff --git a/orchestrator/step_provider_models.go b/orchestrator/step_provider_models.go index 0c132f7..4202f0f 100644 --- a/orchestrator/step_provider_models.go +++ b/orchestrator/step_provider_models.go @@ -17,7 +17,12 @@ type ProviderModelsStep struct { func (s *ProviderModelsStep) Name() string { return s.name } func (s *ProviderModelsStep) Execute(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - body := extractBody(pc) + // Extract the request body from the pipeline context. After step.request_parse + // the body is in pc.Current["body"]; fall back to the whole Current map. + body, _ := pc.Current["body"].(map[string]any) + if body == nil { + body = pc.Current + } providerType := extractString(body, "type", "") apiKey := extractString(body, "api_key", "") baseURL := extractString(body, "base_url", "") diff --git a/orchestrator/step_secret_manage.go b/orchestrator/step_secret_manage.go deleted file mode 100644 index 28c2453..0000000 --- a/orchestrator/step_secret_manage.go +++ /dev/null @@ -1,183 +0,0 @@ -package orchestrator - -import ( - "context" - "errors" - "fmt" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" - "github.com/GoCodeAlone/workflow/secrets" -) - -// SecretManageStep manages secrets (set, delete, list) as a pipeline step. -type SecretManageStep struct { - name string - action string - keyExpr string // template expression for the secret key - valExpr string // template expression for the secret value - app modular.Application - tmpl *module.TemplateEngine -} - -func (s *SecretManageStep) Name() string { return s.name } - -func (s *SecretManageStep) Execute(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - // Lazy-lookup SecretGuard (registered by wiring hook after step factories) - var guard *SecretGuard - if svc, ok := s.app.SvcRegistry()["ratchet-secret-guard"]; ok { - guard, _ = svc.(*SecretGuard) - } - if guard == nil { - return nil, fmt.Errorf("secret_manage step %q: secret guard not available", s.name) - } - - sp := guard.Provider() - if sp == nil { - return nil, fmt.Errorf("secret_manage step %q: secrets provider not available", s.name) - } - - switch s.action { - case "set": - key, err := s.resolveKey(pc) - if err != nil || key == "" { - return nil, fmt.Errorf("secret_manage step %q: key is required", s.name) - } - value, err := s.resolveValue(pc) - if err != nil || value == "" { - return nil, fmt.Errorf("secret_manage step %q: value is required", s.name) - } - - if err := sp.Set(ctx, key, value); err != nil { - return nil, fmt.Errorf("secret_manage step %q: set secret: %w", s.name, err) - } - - // Reload the secret into SecretGuard so it can be redacted - _ = guard.LoadSecrets(ctx, []string{key}) - - // Invalidate ProviderRegistry cache for providers using this secret - s.invalidateProviderCache(key) - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "key": key, - "action": "set", - }, - }, nil - - case "delete": - key, err := s.resolveKey(pc) - if err != nil || key == "" { - return nil, fmt.Errorf("secret_manage step %q: key is required", s.name) - } - - if err := sp.Delete(ctx, key); err != nil { - if isNotFound(err) { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "key": key, - "action": "delete", - "error": "secret not found", - }, - }, nil - } - return nil, fmt.Errorf("secret_manage step %q: delete secret: %w", s.name, err) - } - - // Reload all secrets to update SecretGuard - _ = guard.LoadAllSecrets(ctx) - - // Invalidate ProviderRegistry cache for providers using this secret - s.invalidateProviderCache(key) - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "key": key, - "action": "delete", - }, - }, nil - - case "list": - keys, err := sp.List(ctx) - if err != nil { - return nil, fmt.Errorf("secret_manage step %q: list secrets: %w", s.name, err) - } - if keys == nil { - keys = []string{} - } - - return &module.StepResult{ - Output: map[string]any{ - "keys": keys, - "action": "list", - }, - }, nil - - default: - return nil, fmt.Errorf("secret_manage step %q: unknown action %q (want set|delete|list)", s.name, s.action) - } -} - -// resolveKey resolves the key expression from config or falls back to pc.Current. -func (s *SecretManageStep) resolveKey(pc *module.PipelineContext) (string, error) { - if s.keyExpr != "" { - raw, err := s.tmpl.Resolve(s.keyExpr, pc) - if err != nil { - return "", err - } - return fmt.Sprintf("%v", raw), nil - } - return extractString(pc.Current, "key", ""), nil -} - -// resolveValue resolves the value expression from config or falls back to pc.Current. -func (s *SecretManageStep) resolveValue(pc *module.PipelineContext) (string, error) { - if s.valExpr != "" { - raw, err := s.tmpl.Resolve(s.valExpr, pc) - if err != nil { - return "", err - } - return fmt.Sprintf("%v", raw), nil - } - return extractString(pc.Current, "value", ""), nil -} - -// invalidateProviderCache clears the ProviderRegistry cache for the given secret key. -func (s *SecretManageStep) invalidateProviderCache(secretName string) { - if svc, ok := s.app.SvcRegistry()["ratchet-provider-registry"]; ok { - if registry, ok := svc.(*ProviderRegistry); ok { - registry.InvalidateCacheBySecret(secretName) - } - } -} - -// isNotFound checks if an error is a secrets.ErrNotFound. -func isNotFound(err error) bool { - return errors.Is(err, secrets.ErrNotFound) -} - -// newSecretManageFactory returns a plugin.StepFactory for "step.secret_manage". -func newSecretManageFactory() plugin.StepFactory { - return func(name string, cfg map[string]any, app modular.Application) (any, error) { - action, _ := cfg["action"].(string) - if action == "" { - action = "list" - } - - keyExpr, _ := cfg["key"].(string) - valExpr, _ := cfg["value"].(string) - - return &SecretManageStep{ - name: name, - action: action, - keyExpr: keyExpr, - valExpr: valExpr, - app: app, - tmpl: module.NewTemplateEngine(), - }, nil - } -} diff --git a/orchestrator/step_vault_config.go b/orchestrator/step_vault_config.go deleted file mode 100644 index 38c26e8..0000000 --- a/orchestrator/step_vault_config.go +++ /dev/null @@ -1,464 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "net/url" - "os" - "regexp" - - "github.com/GoCodeAlone/modular" - "github.com/GoCodeAlone/workflow/module" - "github.com/GoCodeAlone/workflow/plugin" - "github.com/GoCodeAlone/workflow/secrets" -) - -// validPathSegment matches alphanumeric, hyphens, underscores, and forward slashes. -var validPathSegment = regexp.MustCompile(`^[a-zA-Z0-9_/.-]+$`) - -// VaultConfigStep manages vault backend configuration as a pipeline step. -// Actions: get_status, test, configure, migrate, reset -type VaultConfigStep struct { - name string - action string - app modular.Application - tmpl *module.TemplateEngine -} - -func (s *VaultConfigStep) Name() string { return s.name } - -func (s *VaultConfigStep) Execute(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - switch s.action { - case "get_status": - return s.getStatus() - case "test": - return s.testConnection(ctx, pc) - case "configure": - return s.configure(ctx, pc) - case "migrate": - return s.migrate(ctx) - case "reset": - return s.reset(ctx) - default: - return nil, fmt.Errorf("vault_config step %q: unknown action %q", s.name, s.action) - } -} - -func (s *VaultConfigStep) getStatus() (*module.StepResult, error) { - guard := s.lookupGuard() - if guard == nil { - return nil, fmt.Errorf("vault_config step %q: secret guard not available", s.name) - } - - result := map[string]any{ - "backend": guard.BackendName(), - } - - // Load saved config for address/mount info - cfg, _ := LoadVaultConfig(vaultConfigDir()) - if cfg != nil { - result["address"] = cfg.Address - result["mount_path"] = cfg.MountPath - result["namespace"] = cfg.Namespace - } else { - result["address"] = "" - result["mount_path"] = "" - result["namespace"] = "" - } - - return &module.StepResult{Output: result}, nil -} - -func (s *VaultConfigStep) testConnection(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - body := extractBody(pc) - address := extractString(body, "address", "") - token := extractString(body, "token", "") - mountPath := extractString(body, "mount_path", "secret") - namespace := extractString(body, "namespace", "") - - if address == "" { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "address is required", - }, - }, nil - } - if token == "" { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "token is required", - }, - }, nil - } - - if msg, ok := validateVaultInputs(address, mountPath, namespace); !ok { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": msg, - }, - }, nil - } - - vp, err := secrets.NewVaultProvider(secrets.VaultConfig{ - Address: address, - Token: token, - MountPath: mountPath, - Namespace: namespace, - }) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "failed to create vault client - check address and credentials", - }, - }, nil - } - - // Test connectivity by listing secrets - _, err = vp.List(ctx) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "connection test failed - verify vault is reachable and token is valid", - }, - }, nil - } - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "message": "connection successful", - }, - }, nil -} - -func (s *VaultConfigStep) configure(ctx context.Context, pc *module.PipelineContext) (*module.StepResult, error) { - guard := s.lookupGuard() - if guard == nil { - return nil, fmt.Errorf("vault_config step %q: secret guard not available", s.name) - } - - body := extractBody(pc) - address := extractString(body, "address", "") - token := extractString(body, "token", "") - mountPath := extractString(body, "mount_path", "secret") - namespace := extractString(body, "namespace", "") - - if address == "" || token == "" { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "address and token are required", - }, - }, nil - } - - if msg, ok := validateVaultInputs(address, mountPath, namespace); !ok { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": msg, - }, - }, nil - } - - // Create new vault provider to verify it works - vp, err := secrets.NewVaultProvider(secrets.VaultConfig{ - Address: address, - Token: token, - MountPath: mountPath, - Namespace: namespace, - }) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "failed to create vault client - check address and credentials", - }, - }, nil - } - - // Test connectivity - if _, err := vp.List(ctx); err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "connection test failed - verify vault is reachable and token is valid", - }, - }, nil - } - - // Optionally migrate secrets - migrateStr := extractString(body, "migrate_secrets", "") - if migrateStr == "true" { - if err := s.migrateSecrets(ctx, guard.Provider(), vp); err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "migration failed - some secrets may not have been copied", - }, - }, nil - } - } - - // Save config file - cfg := &VaultConfigFile{ - Backend: "vault-remote", - Address: address, - Token: token, - MountPath: mountPath, - Namespace: namespace, - } - if err := SaveVaultConfig(vaultConfigDir(), cfg); err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "failed to save vault configuration", - }, - }, nil - } - - // Hot-swap the provider - guard.SetProvider(vp, "vault-remote") - - // Register vault token for redaction so it never appears in logs/transcripts - guard.AddKnownSecret("VAULT_TOKEN", token) - - // Update ProviderRegistry's secrets provider reference - s.updateProviderRegistry(vp) - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "message": "configured remote vault backend", - "backend": "vault-remote", - }, - }, nil -} - -func (s *VaultConfigStep) migrate(ctx context.Context) (*module.StepResult, error) { - guard := s.lookupGuard() - if guard == nil { - return nil, fmt.Errorf("vault_config step %q: secret guard not available", s.name) - } - - // Load saved config to create destination provider - cfg, err := LoadVaultConfig(vaultConfigDir()) - if err != nil || cfg == nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "no vault config found - configure first", - }, - }, nil - } - - if cfg.Backend != "vault-remote" { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "migration only supported when remote vault is configured", - }, - }, nil - } - - dst, err := secrets.NewVaultProvider(secrets.VaultConfig{ - Address: cfg.Address, - Token: cfg.Token, - MountPath: cfg.MountPath, - Namespace: cfg.Namespace, - }) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "failed to connect to destination vault - check saved configuration", - }, - }, nil - } - - migrated, err := s.doMigrate(ctx, guard.Provider(), dst) - if err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "migration failed - some secrets may not have been copied", - }, - }, nil - } - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "message": fmt.Sprintf("migrated %d secrets", migrated), - "migrated": migrated, - }, - }, nil -} - -func (s *VaultConfigStep) reset(ctx context.Context) (*module.StepResult, error) { - guard := s.lookupGuard() - if guard == nil { - return nil, fmt.Errorf("vault_config step %q: secret guard not available", s.name) - } - - // Delete config file - if err := DeleteVaultConfig(vaultConfigDir()); err != nil { - return &module.StepResult{ - Output: map[string]any{ - "success": false, - "error": "failed to reset vault configuration", - }, - }, nil - } - - // Try to start vault-dev - dp, err := secrets.NewDevVaultProvider(secrets.DevVaultConfig{}) - if err != nil { - // Fall back to file provider - secretsDir := os.Getenv("RATCHET_SECRETS_DIR") - if secretsDir == "" { - secretsDir = "data/secrets" - } - _ = os.MkdirAll(secretsDir, 0700) - fp := secrets.NewFileProvider(secretsDir) - guard.SetProvider(fp, "file") - s.updateProviderRegistry(fp) - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "message": "reset to file backend (vault binary not available)", - "backend": "file", - }, - }, nil - } - - // Register dev vault for cleanup - _ = s.app.RegisterService("ratchet-vault-dev", dp) - guard.SetProvider(dp, "vault-dev") - s.updateProviderRegistry(dp) - - return &module.StepResult{ - Output: map[string]any{ - "success": true, - "message": "reset to vault-dev backend", - "backend": "vault-dev", - }, - }, nil -} - -// migrateSecrets copies all secrets from src to dst. -func (s *VaultConfigStep) migrateSecrets(ctx context.Context, src, dst secrets.Provider) error { - _, err := s.doMigrate(ctx, src, dst) - return err -} - -func (s *VaultConfigStep) doMigrate(ctx context.Context, src, dst secrets.Provider) (int, error) { - keys, err := src.List(ctx) - if err != nil { - return 0, fmt.Errorf("list source secrets: %w", err) - } - - count := 0 - for _, key := range keys { - val, err := src.Get(ctx, key) - if err != nil { - continue // skip secrets that can't be read - } - if err := dst.Set(ctx, key, val); err != nil { - return count, fmt.Errorf("set %q in destination: %w", key, err) - } - count++ - } - return count, nil -} - -func (s *VaultConfigStep) lookupGuard() *SecretGuard { - if svc, ok := s.app.SvcRegistry()["ratchet-secret-guard"]; ok { - if guard, ok := svc.(*SecretGuard); ok { - return guard - } - } - return nil -} - -func (s *VaultConfigStep) updateProviderRegistry(p secrets.Provider) { - if svc, ok := s.app.SvcRegistry()["ratchet-provider-registry"]; ok { - if registry, ok := svc.(*ProviderRegistry); ok { - registry.UpdateSecretsProvider(p) - } - } -} - -// validateVaultInputs validates address, mount_path, and namespace fields. -func validateVaultInputs(address, mountPath, namespace string) (string, bool) { - // Validate address is a well-formed URL with http(s) scheme - u, err := url.Parse(address) - if err != nil || u.Host == "" { - return "address must be a valid URL (e.g. https://vault.example.com:8200)", false - } - if u.Scheme != "http" && u.Scheme != "https" { - return "address must use http or https scheme", false - } - if u.Path != "" && u.Path != "/" { - return "address should not include a path — use mount_path instead", false - } - - // Validate mount_path: alphanumeric, hyphens, underscores, dots, slashes only - if mountPath != "" && !validPathSegment.MatchString(mountPath) { - return "mount_path contains invalid characters (allowed: alphanumeric, hyphens, underscores, dots, slashes)", false - } - if len(mountPath) > 128 { - return "mount_path is too long (max 128 characters)", false - } - - // Validate namespace: same rules as mount_path - if namespace != "" && !validPathSegment.MatchString(namespace) { - return "namespace contains invalid characters (allowed: alphanumeric, hyphens, underscores, dots, slashes)", false - } - if len(namespace) > 128 { - return "namespace is too long (max 128 characters)", false - } - - return "", true -} - -// extractBody extracts the request body from the pipeline context. -// After step.request_parse, the body is in pc.Current["body"]. -func extractBody(pc *module.PipelineContext) map[string]any { - if body, ok := pc.Current["body"].(map[string]any); ok { - return body - } - return pc.Current -} - -// vaultConfigDir returns the directory for vault config storage. -func vaultConfigDir() string { - dir := os.Getenv("RATCHET_DATA_DIR") - if dir == "" { - dir = "data" - } - return dir -} - -// newVaultConfigFactory returns a plugin.StepFactory for "step.vault_config". -func newVaultConfigFactory() plugin.StepFactory { - return func(name string, cfg map[string]any, app modular.Application) (any, error) { - action, _ := cfg["action"].(string) - if action == "" { - action = "get_status" - } - - return &VaultConfigStep{ - name: name, - action: action, - app: app, - tmpl: module.NewTemplateEngine(), - }, nil - } -} diff --git a/orchestrator/vault_config.go b/orchestrator/vault_config.go index 5d510ca..7e80e8e 100644 --- a/orchestrator/vault_config.go +++ b/orchestrator/vault_config.go @@ -91,6 +91,19 @@ func DeleteVaultConfig(dir string) error { return err } +// vaultConfigDir returns the directory used for vault config storage. The +// directory also backs the at-rest encryption keyfile (see deriveKey). The +// secretsGuardHook (P13) reads a saved remote-vault token from here so the +// token is added to the guard's redaction set before the engine vault module +// owns the live connection. +func vaultConfigDir() string { + dir := os.Getenv("RATCHET_DATA_DIR") + if dir == "" { + dir = "data" + } + return dir +} + // deriveKey derives a 32-byte AES key from the config directory path and a // machine-local keyfile. The keyfile is created on first use and stored at // dir/.vault-key with 0600 permissions. This ensures the token can only be