diff --git a/.env.example b/.env.example index 126b8a8..2a3e535 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,9 @@ OBJECT_STORE_DRIVER=memory # SecretStore against the running workflow's environment, e.g. # LLM_OPENAI_API_KEY={{secret:openai_prod_key}} # Store the value per environment with: fuse secrets set --env openai_prod_key sk-... +# Or reference a credential id; its apiKey field supplies the key (per environment): +# LLM_OPENAI_CREDENTIAL=openai-prod +# fuse credentials set openai-prod apiKey sk-... --env # LLM_DEFAULT_PROVIDER=ollama # # Ollama (local dev, no API key needed) — OpenAI-compatible endpoint: diff --git a/internal/app/cli/secrets.go b/internal/app/cli/secrets.go index ddef53e..8dbd803 100644 --- a/internal/app/cli/secrets.go +++ b/internal/app/cli/secrets.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/open-source-cloud/fuse/internal/app/config" "github.com/open-source-cloud/fuse/internal/app/di" @@ -59,11 +60,19 @@ func newSecretsListCommand() *cobra.Command { if err != nil { return err } + // Hide credential-backed secrets (cred//); they are managed via + // `fuse credentials`, not exposed in the plain secret surface (ADR-0031). + visible := make([]string, 0, len(names)) + for _, n := range names { + if !strings.HasPrefix(n, "cred/") { + visible = append(visible, n) + } + } fmt.Printf("secrets in environment %q:\n", env) - if len(names) == 0 { + if len(visible) == 0 { fmt.Println(" (none)") } - for _, n := range names { + for _, n := range visible { fmt.Printf(" - %s\n", n) } return nil diff --git a/internal/app/config/config.go b/internal/app/config/config.go index fa71ba5..c9313ad 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -63,10 +63,13 @@ type ( // LLMProviderConfig configures a single LLM provider connection. LLMProviderConfig struct { - Enabled bool `env:"ENABLED" envDefault:"false"` - APIKey string `env:"API_KEY"` - BaseURL string `env:"BASE_URL"` - Model string `env:"MODEL"` + Enabled bool `env:"ENABLED" envDefault:"false"` + APIKey string `env:"API_KEY"` + BaseURL string `env:"BASE_URL"` + Model string `env:"MODEL"` + // Credential references a credential id (ADR-0031); when set and API_KEY is not given, the + // provider's apiKey is taken from that credential's apiKey field (resolved per environment). + Credential string `env:"CREDENTIAL"` Temperature float32 `env:"TEMPERATURE" envDefault:"0.7"` } diff --git a/internal/app/di/llm.go b/internal/app/di/llm.go index d19e834..d403e2b 100644 --- a/internal/app/di/llm.go +++ b/internal/app/di/llm.go @@ -80,12 +80,21 @@ func provideLLMRegistry(cfg *config.Config, secretStore secrets.SecretStore) llm return llm.NewRegistry(factories, cfg.LLM.DefaultProvider) } -// newProviderFactory returns a factory for one provider. When the config has no {{secret:NAME}} -// references the provider is built once and the factory returns that singleton (preserving the -// previous static behavior). Otherwise the factory resolves the references per call against the -// given environment (falling back to defaultEnv when empty) and builds a fresh provider. +// newProviderFactory returns a factory for one provider. When the config has no {{secret:NAME}} or +// {{credential:ID.FIELD}} references the provider is built once and the factory returns that +// singleton (preserving the previous static behavior). Otherwise the factory resolves the +// references per call against the given environment (falling back to defaultEnv when empty) and +// builds a fresh provider. func newProviderFactory(name string, conf config.LLMProviderConfig, build providerBuilder, store secrets.SecretStore, defaultEnv string) llm.ProviderFactory { - if !secrets.HasSecretRef(conf.APIKey) && !secrets.HasSecretRef(conf.BaseURL) { + // A configured credential id supplies the apiKey as a credential reference (ADR-0031) when the + // key is not set explicitly; it then resolves like any other reference, per environment. Only + // apiKey is mapped automatically — a base URL is provider-specific and rarely part of a + // credential, so set BASE_URL={{credential:.baseUrl}} explicitly if needed. + if conf.Credential != "" && conf.APIKey == "" { + conf.APIKey = secrets.CredentialRefToken(conf.Credential, "apiKey") + } + + if !hasAnyRef(conf.APIKey) && !hasAnyRef(conf.BaseURL) { p := build(name, conf.APIKey, conf.BaseURL, conf.Model) return func(_ context.Context, _ string) (llm.Provider, error) { return p, nil } } @@ -102,14 +111,29 @@ func newProviderFactory(name string, conf config.LLMProviderConfig, build provid } return v.Reveal(), nil } - apiKey, err := secrets.ReplaceSecretRefs(conf.APIKey, resolve) + apiKey, err := resolveRefs(conf.APIKey, resolve) if err != nil { return nil, fmt.Errorf("llm[%s]: resolve api key for environment %q: %w", name, env, err) } - baseURL, err := secrets.ReplaceSecretRefs(conf.BaseURL, resolve) + baseURL, err := resolveRefs(conf.BaseURL, resolve) if err != nil { return nil, fmt.Errorf("llm[%s]: resolve base url for environment %q: %w", name, env, err) } return build(name, apiKey, baseURL, conf.Model), nil } } + +// hasAnyRef reports whether s contains a secret or credential reference. +func hasAnyRef(s string) bool { + return secrets.HasSecretRef(s) || secrets.HasCredentialRef(s) +} + +// resolveRefs resolves both {{secret:NAME}} and {{credential:ID.FIELD}} references in s via the +// same per-environment resolve function (credential refs map to their reserved secret name). +func resolveRefs(s string, resolve func(string) (string, error)) (string, error) { + out, err := secrets.ReplaceSecretRefs(s, resolve) + if err != nil { + return "", err + } + return secrets.ReplaceCredentialRefs(out, resolve) +} diff --git a/internal/app/di/llm_test.go b/internal/app/di/llm_test.go index 5b3ace2..f93799a 100644 --- a/internal/app/di/llm_test.go +++ b/internal/app/di/llm_test.go @@ -40,6 +40,38 @@ func TestProvideLLMRegistry_ResolvesKeyPerEnvironment(t *testing.T) { assert.ErrorIs(t, err, secrets.ErrSecretNotFound) } +func TestProvideLLMRegistry_ResolvesCredential(t *testing.T) { + ctx := context.Background() + store := secrets.NewMemorySecretStore() + // A credential's apiKey field value lives at the reserved cred//apiKey secret name. + require.NoError(t, store.Set(ctx, secrets.Scope{Environment: "staging"}, + secrets.CredentialSecretName("openai-prod", "apiKey"), "sk-from-credential")) + + cfg := &config.Config{ + Environment: "default", + LLM: config.LLMConfig{ + DefaultProvider: providerOpenAI, + OpenAI: config.LLMProviderConfig{ + Enabled: true, + Credential: "openai-prod", + Model: "gpt-4o-mini", + }, + }, + } + + reg := provideLLMRegistry(cfg, store) + + // The credential's apiKey resolves in staging -> provider builds. + prov, err := reg.Get(ctx, "staging", providerOpenAI) + require.NoError(t, err) + assert.Equal(t, providerOpenAI, prov.Name()) + + // An environment lacking the credential value surfaces a resolution error, not a panic. + _, err = reg.Get(ctx, "prod", providerOpenAI) + require.Error(t, err) + assert.ErrorIs(t, err, secrets.ErrSecretNotFound) +} + func TestProvideLLMRegistry_StaticProviderIsSingleton(t *testing.T) { ctx := context.Background() cfg := &config.Config{ diff --git a/internal/workflow/edge_schema.go b/internal/workflow/edge_schema.go index 60c2c38..45b0bbc 100644 --- a/internal/workflow/edge_schema.go +++ b/internal/workflow/edge_schema.go @@ -21,6 +21,10 @@ const ( // scoped by the workflow's environment. The resolved value is redacted in every // engine sink (ADR-0031). SourceSecret InputMappingSource = "secret" + // SourceCredential resolves a credential field, with Variable holding ".". + // The value is read from the SecretStore at cred//, scoped by the workflow's + // environment, and redacted in every engine sink (ADR-0031). + SourceCredential InputMappingSource = "credential" ) type ( diff --git a/internal/workflow/workflow.go b/internal/workflow/workflow.go index 9200428..0fe9b8a 100644 --- a/internal/workflow/workflow.go +++ b/internal/workflow/workflow.go @@ -77,21 +77,28 @@ func (w *Workflow) SetSecretResolver(r secrets.Resolver) { w.secretResolver = r } -// resolveSchemaValue resolves any {{secret:NAME}} references embedded in a schema -// string value, wrapping the whole result as a SecretValue so it is redacted in -// every sink. Non-string or reference-free values pass through unchanged. +// resolveSchemaValue resolves any {{secret:NAME}} and {{credential:ID.FIELD}} references embedded +// in a schema string value, wrapping the whole result as a SecretValue so it is redacted in every +// sink. Non-string or reference-free values pass through unchanged. func (w *Workflow) resolveSchemaValue(value any) (any, error) { s, ok := value.(string) - if !ok || !secrets.HasSecretRef(s) { + if !ok || (!secrets.HasSecretRef(s) && !secrets.HasCredentialRef(s)) { return value, nil } - resolved, err := secrets.ReplaceSecretRefs(s, func(name string) (string, error) { + reveal := func(name string) (string, error) { sv, err := w.secretValueByName(name) if err != nil { return "", err } return sv.Reveal(), nil - }) + } + resolved, err := secrets.ReplaceSecretRefs(s, reveal) + if err != nil { + return nil, err + } + // Credential refs map to the reserved cred// secret name and resolve via the same + // environment-scoped resolver. + resolved, err = secrets.ReplaceCredentialRefs(resolved, reveal) if err != nil { return nil, err } @@ -103,6 +110,17 @@ func (w *Workflow) resolveSecret(name string) (secrets.SecretValue, error) { return w.secretValueByName(name) } +// resolveCredential resolves a SourceCredential mapping. variable holds "."; it is +// split on the LAST dot (so ids may contain dots) and read from the reserved cred// +// secret name, scoped to this workflow's environment (ADR-0031). +func (w *Workflow) resolveCredential(variable string) (secrets.SecretValue, error) { + dot := strings.LastIndex(variable, ".") + if dot <= 0 || dot == len(variable)-1 { + return secrets.SecretValue{}, fmt.Errorf("invalid credential reference %q: expected \".\"", variable) + } + return w.secretValueByName(secrets.CredentialSecretName(variable[:dot], variable[dot+1:])) +} + // secretValueByName resolves a secret by name, scoped to this workflow's environment. func (w *Workflow) secretValueByName(name string) (secrets.SecretValue, error) { if w.secretResolver == nil { @@ -685,118 +703,133 @@ func (w *Workflow) inputMapping(edge *Edge, mappings []InputMapping) map[string] continue } args.Set(mapping.MapTo, sv) + case SourceCredential: + sv, err := w.resolveCredential(mapping.Variable) + if err != nil { + log.Error().Err(err).Str("edge", edge.ID()).Str("param", mapping.MapTo). + Str("credential", mapping.Variable).Msg("failed to resolve credential") + continue + } + args.Set(mapping.MapTo, sv) case SourceFlow: - outputParamName := strutil.AfterFirstDot(mapping.Variable) + w.applyFlowMapping(args, edge, mapping, inputParamSchema, allowCustomInputParameters) + } + } - nodeFrom := edge.From() - if nodeFrom == nil { - log.Error().Str("edge", edge.ID()).Str("param", outputParamName). - Msg("Node from is nil") - break - } - nodeFromMetadata := nodeFrom.FunctionMetadata() - if nodeFromMetadata == nil { - log.Error().Str("edge", edge.ID()).Str("param", outputParamName). - Msg("Node from metadata is nil") - break - } + log.Debug().Msgf("Args: %+v", args.Raw()) - outputParamSchema, exists := nodeFromMetadata.Output.Parameters[outputParamName] - if !allowCustomInputParameters && !exists { - log.Error().Str("edge", edge.ID()).Str("param", outputParamName). - Msgf("Output ParamSchema not found for input mapping") - continue - } + return args.Raw() +} - isArray := strings.HasPrefix(inputParamSchema.Type, "[]") - rawValue := w.aggregatedOutput.Get(mapping.Variable) - var value any - switch { - case inputParamSchema.Type == "" && allowCustomInputParameters: - value = rawValue - case isArray: - // Parallel branches each contribute one element; validate against the upstream - // output type (e.g. int from rand), then coerce to the slice-typed destination. - var parsedScalar any - var err error - if outputParamSchema.Type == "" { - parsedScalar = rawValue - } else { - parsedScalar, err = typeschema.ParseValue(outputParamSchema.Type, rawValue) - if err != nil { - log.Error(). - Err(err). - Str("edge", edge.ID()). - Str("param", mapping.MapTo). - Any("value", rawValue). - Msg(logMsgErrorParsingValue) - continue - } - } - if outputParamSchema.Type != "" && !w.validateInputMapping(&outputParamSchema, parsedScalar) { - log.Error(). - Str("edge", edge.ID()). - Str("param", mapping.MapTo). - Any("value", parsedScalar). - Msg(logMsgFailedParamValidation) - continue - } - value, err = typeschema.ParseValue(inputParamSchema.Type, parsedScalar) - if err != nil { - log.Error(). - Err(err). - Str("edge", edge.ID()). - Str("param", mapping.MapTo). - Any("value", parsedScalar). - Msg(logMsgErrorParsingValue) - continue - } - default: - var err error - value, err = typeschema.ParseValue(inputParamSchema.Type, rawValue) - if err != nil { - log.Error(). - Err(err). - Str("edge", edge.ID()). - Str("param", mapping.MapTo). - Any("value", mapping.Value). - Msg(logMsgErrorParsingValue) - continue - } - if !w.validateInputMapping(&inputParamSchema, value) { - log.Error(). - Str("edge", edge.ID()). - Str("param", mapping.MapTo). - Any("value", value). - Msg(logMsgFailedParamValidation) - continue - } - } +// applyFlowMapping resolves a SourceFlow input mapping (a value forwarded from an upstream node's +// output) and writes it into args. It is split out of inputMapping to keep that method's +// cyclomatic complexity in check; the early returns here mirror the original per-mapping skips. +func (w *Workflow) applyFlowMapping(args *store.KV, edge *Edge, mapping InputMapping, inputParamSchema workflow.ParameterSchema, allowCustomInputParameters bool) { + outputParamName := strutil.AfterFirstDot(mapping.Variable) - if isArray { - currentArray := args.Get(mapping.MapTo) - if currentArray != nil { - currentArraySlice := reflect.ValueOf(currentArray) - valueSlice := reflect.ValueOf(value) - args.Set(mapping.MapTo, reflect.AppendSlice(currentArraySlice, valueSlice).Interface()) - } else { - args.Set(mapping.MapTo, value) - } - continue - } + nodeFrom := edge.From() + if nodeFrom == nil { + log.Error().Str("edge", edge.ID()).Str("param", outputParamName). + Msg("Node from is nil") + return + } + nodeFromMetadata := nodeFrom.FunctionMetadata() + if nodeFromMetadata == nil { + log.Error().Str("edge", edge.ID()).Str("param", outputParamName). + Msg("Node from metadata is nil") + return + } + + outputParamSchema, exists := nodeFromMetadata.Output.Parameters[outputParamName] + if !allowCustomInputParameters && !exists { + log.Error().Str("edge", edge.ID()).Str("param", outputParamName). + Msgf("Output ParamSchema not found for input mapping") + return + } - // if the value is nil, and there is a default value, use the default value - if value == nil && inputParamSchema.Default != nil { - value = inputParamSchema.Default + isArray := strings.HasPrefix(inputParamSchema.Type, "[]") + rawValue := w.aggregatedOutput.Get(mapping.Variable) + var value any + switch { + case inputParamSchema.Type == "" && allowCustomInputParameters: + value = rawValue + case isArray: + // Parallel branches each contribute one element; validate against the upstream + // output type (e.g. int from rand), then coerce to the slice-typed destination. + var parsedScalar any + var err error + if outputParamSchema.Type == "" { + parsedScalar = rawValue + } else { + parsedScalar, err = typeschema.ParseValue(outputParamSchema.Type, rawValue) + if err != nil { + log.Error(). + Err(err). + Str("edge", edge.ID()). + Str("param", mapping.MapTo). + Any("value", rawValue). + Msg(logMsgErrorParsingValue) + return } + } + if outputParamSchema.Type != "" && !w.validateInputMapping(&outputParamSchema, parsedScalar) { + log.Error(). + Str("edge", edge.ID()). + Str("param", mapping.MapTo). + Any("value", parsedScalar). + Msg(logMsgFailedParamValidation) + return + } + value, err = typeschema.ParseValue(inputParamSchema.Type, parsedScalar) + if err != nil { + log.Error(). + Err(err). + Str("edge", edge.ID()). + Str("param", mapping.MapTo). + Any("value", parsedScalar). + Msg(logMsgErrorParsingValue) + return + } + default: + var err error + value, err = typeschema.ParseValue(inputParamSchema.Type, rawValue) + if err != nil { + log.Error(). + Err(err). + Str("edge", edge.ID()). + Str("param", mapping.MapTo). + Any("value", mapping.Value). + Msg(logMsgErrorParsingValue) + return + } + if !w.validateInputMapping(&inputParamSchema, value) { + log.Error(). + Str("edge", edge.ID()). + Str("param", mapping.MapTo). + Any("value", value). + Msg(logMsgFailedParamValidation) + return + } + } + if isArray { + currentArray := args.Get(mapping.MapTo) + if currentArray != nil { + currentArraySlice := reflect.ValueOf(currentArray) + valueSlice := reflect.ValueOf(value) + args.Set(mapping.MapTo, reflect.AppendSlice(currentArraySlice, valueSlice).Interface()) + } else { args.Set(mapping.MapTo, value) } + return } - log.Debug().Msgf("Args: %+v", args.Raw()) + // if the value is nil, and there is a default value, use the default value + if value == nil && inputParamSchema.Default != nil { + value = inputParamSchema.Default + } - return args.Raw() + args.Set(mapping.MapTo, value) } func (w *Workflow) validateInputMapping(schema *workflow.ParameterSchema, value any) bool { diff --git a/internal/workflow/workflow_credential_test.go b/internal/workflow/workflow_credential_test.go new file mode 100644 index 0000000..2395ed3 --- /dev/null +++ b/internal/workflow/workflow_credential_test.go @@ -0,0 +1,44 @@ +package workflow + +import ( + "context" + "encoding/json" + "testing" + + "github.com/open-source-cloud/fuse/pkg/secrets" + "github.com/open-source-cloud/fuse/pkg/workflow" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWorkflow_ResolveCredentialReferences(t *testing.T) { + store := secrets.NewMemorySecretStore() + // Credential field values live at the reserved cred// secret name. + require.NoError(t, store.Set(context.Background(), secrets.Scope{Environment: "test"}, + secrets.CredentialSecretName("openai-prod", "apiKey"), "sk-PROD")) + + w := &Workflow{id: workflow.ID("wf-1")} + w.SetSecretResolver(secrets.NewResolver(store, "test")) + + // source:"credential" with "." resolves to a redacted SecretValue. + sv, err := w.resolveCredential("openai-prod.apiKey") + require.NoError(t, err) + assert.Equal(t, "sk-PROD", sv.Reveal()) + b, _ := json.Marshal(sv) + assert.JSONEq(t, `"***"`, string(b)) + + // {{credential:id.field}} embedded in a schema string -> a SecretValue wrapping the whole value. + v, err := w.resolveSchemaValue("Bearer {{credential:openai-prod.apiKey}}") + require.NoError(t, err) + wrapped, ok := v.(secrets.SecretValue) + require.True(t, ok) + assert.Equal(t, "Bearer sk-PROD", wrapped.Reveal()) + + // Malformed reference (no field) is an error. + _, err = w.resolveCredential("openai-prod") + require.Error(t, err) + + // Unknown credential field surfaces the not-found error. + _, err = w.resolveCredential("openai-prod.missing") + assert.ErrorIs(t, err, secrets.ErrSecretNotFound) +}