From ad8a89d3ca2cdc8aef621bd8e837e70b1f5756b2 Mon Sep 17 00:00:00 2001 From: Christopher Langton Date: Thu, 12 Nov 2026 21:22:52 +1100 Subject: [PATCH] test(aifirewall): property tests for Plan and ValidateGuardrail; plan server maps in key order Adds testing/quick properties over generated policy files and server states: Plan runs in kind order, is deterministic, and converges (applying its mutating changes leaves nothing to change); ValidateGuardrail accepts exactly the enforceable rule type and action pairs and answers for any input. The determinism property failed: Plan ranged over server.Guardrails and server.Providers directly, so guardrail drift/deletes and provider drift came out in map order, different on every apply. Those loops now go through sortedKeys, made generic, as the model loop already did. Closes #222 --- pkg/aifirewall/plan_property_test.go | 278 +++++++++++++++++++++++++++ pkg/aifirewall/policy.go | 15 +- 2 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 pkg/aifirewall/plan_property_test.go diff --git a/pkg/aifirewall/plan_property_test.go b/pkg/aifirewall/plan_property_test.go new file mode 100644 index 0000000000..0ae10f5109 --- /dev/null +++ b/pkg/aifirewall/plan_property_test.go @@ -0,0 +1,278 @@ +package aifirewall + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + "testing/quick" +) + +// Property tests for Plan and ValidateGuardrail. The example tests in +// policy_test.go pin particular plans; these check the invariants every plan +// must keep, over generated policies and server states. + +var planQuickConfig = &quick.Config{MaxCount: 2000} + +// Small vocabularies, so a generated file and a generated server overlap often +// enough to exercise create, update, unchanged, drift and delete together. +var ( + propGuardrailNames = []string{"g1", "g2", "g3", "g4", "g5"} + propRuleTypes = []string{"blocked_pattern", "pii_redact", "tool_deny", "mcp_allow"} + propActions = []string{"block", "flag", "redact", "strip"} + propPatterns = []string{"secret", "", "mcp__github__*"} + propProviders = []string{"openai", "anthropic", "groq", "mistral"} + propProviderActs = []string{"allow", "deny", "default"} + propModels = []string{"m1", "m2", "m3"} + propModelActs = []string{"allow", "deny"} +) + +func pick[T any](r *rand.Rand, xs []T) T { return xs[r.Intn(len(xs))] } + +// planCase is one generated (desired, server) pair. Names are unique within +// the file, as Validate requires. Provider keys and anyProvider models are not +// generated: both are re-sent on every apply by design (a key is write-only and +// an anyProvider entry is expanded server-side), so they are not expected to +// converge and are covered by the example tests instead. +type planCase struct { + Desired PolicyFile + Server ServerState +} + +func (planCase) Generate(r *rand.Rand, _ int) reflect.Value { + var c planCase + c.Desired.Spec.Prune = r.Intn(2) == 0 + + for _, name := range propGuardrailNames { + if r.Intn(2) == 0 { + g := GuardrailSpec{ + Name: name, RuleType: pick(r, propRuleTypes), Action: pick(r, propActions), + Pattern: pick(r, propPatterns), Priority: r.Intn(3), + } + if r.Intn(3) == 0 { + g.Enabled = boolPtr(r.Intn(2) == 0) + } + c.Desired.Spec.Guardrails = append(c.Desired.Spec.Guardrails, g) + } + } + for _, p := range propProviders { + for _, m := range propModels { + if r.Intn(3) == 0 { + c.Desired.Spec.Models = append(c.Desired.Spec.Models, + ModelSpec{Provider: p, Slug: m, Action: pick(r, propModelActs)}) + } + } + } + for _, p := range propProviders { + if r.Intn(2) == 0 { + c.Desired.Spec.Providers = append(c.Desired.Spec.Providers, + ProviderSpec{Slug: p, Action: pick(r, propProviderActs)}) + } + } + if r.Intn(2) == 0 { + c.Desired.Spec.Settings = &SettingsSpec{LogsEnabled: boolPtr(r.Intn(2) == 0)} + } + + c.Server = ServerState{ + Providers: map[string]string{}, + Models: map[string]string{}, + Guardrails: map[string]ServerGuardrail{}, + LogsEnabled: r.Intn(2) == 0, + } + for i, name := range propGuardrailNames { + if r.Intn(2) == 0 { + c.Server.Guardrails[name] = ServerGuardrail{ + UUID: fmt.Sprintf("uuid-%d", i), RuleType: pick(r, propRuleTypes), Action: pick(r, propActions), + Pattern: pick(r, propPatterns), Priority: r.Intn(3), Enabled: r.Intn(2) == 0, + } + } + } + for _, p := range propProviders { + if r.Intn(2) == 0 { + c.Server.Providers[p] = pick(r, []string{"", "allow", "deny"}) + } + for _, m := range propModels { + if r.Intn(3) == 0 { + c.Server.Models[p+"/"+m] = pick(r, propModelActs) + } + } + } + return reflect.ValueOf(c) +} + +var kindOrder = map[Kind]int{KindGuardrail: 0, KindModel: 1, KindProvider: 2, KindKey: 3, KindSettings: 4} + +// TestPlanRunsInKindOrder: guardrails, then models, then providers, then keys, +// then settings, whatever the input. Tightening must never pass through a state +// where a provider is open before the guardrails that constrain it exist. +func TestPlanRunsInKindOrder(t *testing.T) { + prop := func(c planCase) bool { + changes := Plan(c.Desired, c.Server) + for i := 1; i < len(changes); i++ { + if kindOrder[changes[i-1].Kind] > kindOrder[changes[i].Kind] { + t.Logf("%s before %s in %+v", changes[i-1].Kind, changes[i].Kind, changes) + return false + } + } + return true + } + if err := quick.Check(prop, planQuickConfig); err != nil { + t.Error(err) + } +} + +// TestPlanIsDeterministic: equal inputs give the same plan, in the same order, +// every time. The server state arrives as maps, whose iteration order changes +// from run to run; a plan that followed it would print and apply its drift and +// deletes in a different order on every apply. +func TestPlanIsDeterministic(t *testing.T) { + prop := func(c planCase) bool { + first := Plan(c.Desired, c.Server) + for range 5 { + if again := Plan(c.Desired, c.Server); !reflect.DeepEqual(first, again) { + t.Logf("plans differ:\n%+v\n%+v", first, again) + return false + } + } + return true + } + if err := quick.Check(prop, planQuickConfig); err != nil { + t.Error(err) + } +} + +// applyPlan does to a copy of the server what apply would do with the plan's +// mutating changes. +func applyPlan(server ServerState, changes []Change) ServerState { + out := ServerState{ + Providers: map[string]string{}, + Models: map[string]string{}, + Guardrails: map[string]ServerGuardrail{}, + LogsEnabled: server.LogsEnabled, + } + for k, v := range server.Providers { + out.Providers[k] = v + } + for k, v := range server.Models { + out.Models[k] = v + } + for k, v := range server.Guardrails { + out.Guardrails[k] = v + } + for _, c := range Mutating(changes) { + switch c.Kind { + case KindGuardrail: + if c.Op == OpDelete { + delete(out.Guardrails, c.Target) + continue + } + g := c.Guardrail + out.Guardrails[c.Target] = ServerGuardrail{ + UUID: c.UUID, RuleType: g.RuleType, Action: g.Action, + Pattern: g.Pattern, Priority: g.Priority, Enabled: g.IsEnabled(), + } + case KindModel: + if c.Op == OpDelete { + delete(out.Models, c.Target) + continue + } + out.Models[c.Model.Provider+"/"+c.Model.Slug] = c.Model.Action + case KindProvider: + action := c.Provider.Action + if action == "default" { + action = "" + } + out.Providers[c.Target] = action + case KindSettings: + out.LogsEnabled = *c.Enable + } + } + return out +} + +// TestPlanConverges: once a plan's mutating changes are applied, planning the +// same file again has nothing left to change. A plan that never converges would +// rewrite the same objects on every apply and could never be clean. +func TestPlanConverges(t *testing.T) { + prop := func(c planCase) bool { + after := applyPlan(c.Server, Plan(c.Desired, c.Server)) + if rest := Mutating(Plan(c.Desired, after)); len(rest) != 0 { + t.Logf("still mutating after apply: %+v", rest) + return false + } + return true + } + if err := quick.Check(prop, planQuickConfig); err != nil { + t.Error(err) + } +} + +// TestValidateGuardrailAcceptsExactlyTheEnforceablePairs walks every rule type +// and action the gateway knows, with a pattern valid for the rule type, and +// checks the verdict against the pairs the gateway can enforce. +func TestValidateGuardrailAcceptsExactlyTheEnforceablePairs(t *testing.T) { + patterns := map[string]string{ + "blocked_pattern": "secret", "max_messages": "50", "pii_redact": "", "delimiter_integrity": "", + "tool_allow": "Bash", "tool_deny": "Bash", "mcp_allow": "mcp__github__*", "mcp_deny": "mcp__github__*", + "skill_allow": "review", "skill_deny": "review", "client_allow": "cursor", "client_deny": "cursor", + } + allowed := map[string][]string{ + "blocked_pattern": {"block", "redact", "flag"}, + "max_messages": {"block", "redact", "flag"}, + "pii_redact": {"block", "redact", "flag"}, + "delimiter_integrity": {"block", "flag"}, + "tool_allow": {"block", "flag", "strip"}, + "tool_deny": {"block", "flag", "strip"}, + "mcp_allow": {"block", "flag", "strip"}, + "mcp_deny": {"block", "flag", "strip"}, + "skill_allow": {"block", "flag"}, + "skill_deny": {"block", "flag"}, + "client_allow": {"block", "flag"}, + "client_deny": {"block", "flag"}, + } + if len(allowed) != len(contentRuleTypes)+len(capabilityRuleTypes) { + t.Fatalf("the table covers %d rule types, the vocabulary has %d", len(allowed), len(contentRuleTypes)+len(capabilityRuleTypes)) + } + for ruleType, ok := range allowed { + for action := range validGuardrailActions { + want := false + for _, a := range ok { + want = want || a == action + } + err := ValidateGuardrail("g", ruleType, action, patterns[ruleType]) + if (err == nil) != want { + t.Errorf("ValidateGuardrail(%s, %s) = %v, want accepted=%v", ruleType, action, err, want) + } + } + } +} + +// TestValidateGuardrailIsTotal: any strings give an answer, never a panic, and +// a rule type or action outside the vocabulary is always refused. +func TestValidateGuardrailIsTotal(t *testing.T) { + prop := func(name, ruleType, action, pattern string) bool { + err := ValidateGuardrail(name, ruleType, action, pattern) + known := (contentRuleTypes[ruleType] || capabilityRuleTypes[ruleType]) && validGuardrailActions[action] + return known || err != nil + } + if err := quick.Check(prop, planQuickConfig); err != nil { + t.Error(err) + } + // Known vocabulary with arbitrary patterns: still an answer, never a panic. + var types []string + for k := range contentRuleTypes { + types = append(types, k) + } + for k := range capabilityRuleTypes { + types = append(types, k) + } + r := rand.New(rand.NewSource(1)) + known := func(pattern string) bool { + action := []string{"block", "redact", "flag", "strip"}[r.Intn(4)] + _ = ValidateGuardrail("g", types[r.Intn(len(types))], action, pattern) + return true + } + if err := quick.Check(known, planQuickConfig); err != nil { + t.Error(err) + } +} diff --git a/pkg/aifirewall/policy.go b/pkg/aifirewall/policy.go index 443a303d7a..2db496e145 100644 --- a/pkg/aifirewall/policy.go +++ b/pkg/aifirewall/policy.go @@ -427,15 +427,15 @@ func Plan(desired PolicyFile, server ServerState) []Change { } } if desired.Spec.Prune { - for name, cur := range server.Guardrails { + for _, name := range sortedKeys(server.Guardrails) { if !hasGuardrail(desired.Spec.Guardrails, name) { changes = append(changes, Change{ - Kind: KindGuardrail, Op: OpDelete, Target: name, UUID: cur.UUID, + Kind: KindGuardrail, Op: OpDelete, Target: name, UUID: server.Guardrails[name].UUID, }) } } } else { - for name := range server.Guardrails { + for _, name := range sortedKeys(server.Guardrails) { if !hasGuardrail(desired.Spec.Guardrails, name) { changes = append(changes, Change{ Kind: KindGuardrail, Op: OpDrift, Target: name, @@ -526,7 +526,8 @@ func Plan(desired PolicyFile, server ServerState) []Change { // which *opens* access — the opposite of what every other prune does — so it // stays a deliberate, explicit act. Reporting it costs nothing and stops the // row from being silently invisible, which is how it behaved before. - for slug, action := range server.Providers { + for _, slug := range sortedKeys(server.Providers) { + action := server.Providers[slug] if action == "" || hasProvider(desired.Spec.Providers, slug) { continue } @@ -604,8 +605,10 @@ func mentionsModel(models []ModelSpec, key string) bool { return false } -// sortedKeys keeps the plan deterministic; map iteration order is not. -func sortedKeys(m map[string]string) []string { +// sortedKeys keeps the plan deterministic; map iteration order is not. Every +// loop in Plan over a server map goes through it, or the drift and deletes it +// reports come out in a different order on every apply. +func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k)