diff --git a/compilers/compile/naming.go b/compilers/compile/naming.go index 018fcae78..2fa1a22de 100644 --- a/compilers/compile/naming.go +++ b/compilers/compile/naming.go @@ -1,6 +1,10 @@ package compile -import "github.com/dexpace/morphic/ir" +import ( + "strings" + + "github.com/dexpace/morphic/ir" +) // emptyNameHint is the hint minted for an entity the source left unnamed: an // enum member whose value is "", a property or component keyed by it, a tag @@ -59,6 +63,24 @@ func SubHint(parent, suffix string) string { return neutralHint(parent) + "_" + neutralHint(suffix) } +// SubHints is SubHint along a chain, each suffix composed onto the hint +// before it: SubHints(p, a, b) is SubHint(SubHint(p, a), b), and SubHints(p) +// is p. Since a composed hint is neutral already, the chain is each part +// neutralized once and joined, so a hint composed of n parts costs n parts, +// not the n² a fold of SubHint spends re-reading its growing parent. +func SubHints(parent string, suffixes ...string) string { + if len(suffixes) == 0 { + return parent + } + var b strings.Builder + b.WriteString(neutralHint(parent)) + for _, s := range suffixes { + b.WriteByte('_') + b.WriteString(neutralHint(s)) + } + return b.String() +} + // neutralHint returns the neutral word sequence of hint, or the minted name when // the position it was derived from names nothing a word can be read out of. func neutralHint(hint string) string { diff --git a/compilers/compile/naming_test.go b/compilers/compile/naming_test.go index 0bf6143ac..c54a6b561 100644 --- a/compilers/compile/naming_test.go +++ b/compilers/compile/naming_test.go @@ -118,3 +118,34 @@ func TestSubHint_IsItselfANeutralHint(t *testing.T) { assert.Equal(t, "combo_a_2_item", nested) assert.Equal(t, nested, ir.CanonicalWords(nested), "the grammar leaves a composed hint alone") } + +// TestSubHints_IsAFoldOfSubHint pins SubHints to the SubHint chain it +// shortens, over parents and suffixes that neutralize to nothing, carry case, +// digits and separators, and are already composed, and to the parent alone +// for an empty chain. +func TestSubHints_IsAFoldOfSubHint(t *testing.T) { + t.Parallel() + parts := []string{"", "***", "Combo_A", "userID", "2", "item", "x-ext", "empty_item", "HTTPServer2Go"} + for _, parent := range parts { + assert.Equal(t, parent, compile.SubHints(parent), "an empty chain leaves the parent as it is") + for _, a := range parts { + for _, b := range parts { + assert.Equal(t, compile.SubHint(compile.SubHint(parent, a), b), compile.SubHints(parent, a, b), + "%q, %q, %q", parent, a, b) + } + } + } +} + +// FuzzSubHints_IsAFoldOfSubHint holds SubHints to the SubHint chain on any +// three parts. +func FuzzSubHints_IsAFoldOfSubHint(f *testing.F) { + f.Add("Combo_A", "2", "item") + f.Add("", "***", "éA") + f.Fuzz(func(t *testing.T, parent, a, b string) { + want := compile.SubHint(compile.SubHint(compile.SubHint(parent, a), b), a) + if got := compile.SubHints(parent, a, b, a); got != want { + t.Fatalf("SubHints(%q, %q, %q, %q) = %q, the chain %q", parent, a, b, a, got, want) + } + }) +} diff --git a/compilers/openapi/conformance_test.go b/compilers/openapi/conformance_test.go index 3fda0ea2e..1f710a89a 100644 --- a/compilers/openapi/conformance_test.go +++ b/compilers/openapi/conformance_test.go @@ -237,6 +237,7 @@ func conformanceCases() []conformanceCase { {"path-item-docs", assertPathItemDocs}, {"path-item-operations", assertPathItemOperations}, {"path-item-anchored-key", assertPathItemAnchoredKey}, + {"yaml-merge-keys", assertYAMLMergeKeys}, {"path-item-ref-siblings", assertPathItemRefSiblings}, {"deprecation", assertDeprecation}, {"extension-promotion", assertExtensionPromotion}, diff --git a/compilers/openapi/conformance_unmodeled_test.go b/compilers/openapi/conformance_unmodeled_test.go index e54f784a8..a0676ad59 100644 --- a/compilers/openapi/conformance_unmodeled_test.go +++ b/compilers/openapi/conformance_unmodeled_test.go @@ -1072,3 +1072,91 @@ func indexByParam(bindings []ir.HTTPParamBinding) map[string]ir.HTTPParamBinding } return out } + +// assertYAMLMergeKeys pins that every reader of the source reads a mapping as +// the parser does (GitHub #1089): a field, a stray key or a keyword merged in +// through `<<` is kept and announced as a written one is, by each kind of +// reader that keeps one (GitHub #384, #395), an extension keeps a key YAML +// types as a number or a boolean as its text (GitHub #617), a plain number +// past float64's range stays a number (GitHub #783), and an example's members +// come in the parser's order. The spec writes no alias, so the order oracle +// can permute it; alias keys are the readers' own tests'. +func assertYAMLMergeKeys(t *testing.T, doc *ir.Document, diags []ir.Diagnostic) { + t.Helper() + op, ok := opByName(doc, "getMerged") + require.True(t, ok) + assertKept(t, op.Unmodeled, []keptEntry{ + {"openapi:pathItemSummary", `"Shared summary"`, "/paths/~1merged/summary"}, + {"openapi:pathItemDescription", `"Shared description"`, "/paths/~1merged/description"}, + {"openapi:servers", `[{"url":"https://path.example"}]`, "/paths/~1merged/servers"}, + {"openapi:pathItem/mergedBogus", `{"responses":{"200":{"description":"MERGED"}}}`, "/paths/~1merged/mergedBogus"}, + {"openapi:operationServers", `[{"url":"https://operation.example"}]`, "/paths/~1merged/get/servers"}, + {"openapi:x-codes", `{"100":"Continue","200":"OK","true":"T"}`, "/paths/~1merged/get/x-codes"}, + {"openapi:x-big", `1e400`, "/paths/~1merged/get/x-big"}, + }) + assert.Equal(t, []ir.Severity{ir.SeverityWarning}, + diagsAt(diags, "openapi/unknown-object-key", "/paths/~1merged/mergedBogus"), "the merged stray key is announced") + for _, d := range diags { + assert.NotContains(t, d.Message, "could not be serialized", "every extension converts") + assert.NotEqual(t, "openapi/unknown-key-unreachable", d.Code, "every merged key is reached") + } + assertMergedReaders(t, doc, op) + + body, ok := doc.Types["t/anon/paths/~1merged/get/responses/200/content/application~1json/schema"] + require.True(t, ok) + assertKept(t, body.Common().Unmodeled, []keptEntry{ + {"openapi:not", `{"required":["forbidden"]}`, "/paths/~1merged/get/responses/200/content/application~1json/schema/not"}, + }) + examples := body.Common().Examples + require.Len(t, examples, 1) + require.NotNil(t, examples[0].Value) + members := make([]string, 0, len(examples[0].Value.Object)) + for _, f := range examples[0].Value.Object { + members = append(members, f.Name) + } + assert.Equal(t, []string{"a", "b"}, members, "the merged member first, as the parser reads it") +} + +// assertMergedReaders pins the merged constructs kept beside the operation: +// on its parameter, its response and that response's header, the other +// operation's encoding, and the security scheme. +func assertMergedReaders(t *testing.T, doc *ir.Document, op ir.Operation) { + t.Helper() + require.Len(t, op.Params, 1) + assertKept(t, op.Params[0].Unmodeled, []keptEntry{ + {"openapi:allowEmptyValue", `true`, "/paths/~1merged/get/parameters/0/allowEmptyValue"}, + }) + require.Len(t, op.Responses, 1) + assertKept(t, op.Responses[0].Unmodeled, []keptEntry{ + {"openapi:links", `{"self":{"operationId":"getMerged"}}`, "/paths/~1merged/get/responses/200/links"}, + }) + require.Len(t, op.Responses[0].Headers, 1) + assertKept(t, op.Responses[0].Headers[0].Unmodeled, []keptEntry{ + {"openapi:style", `"simple"`, "/paths/~1merged/get/responses/200/headers/X-Rate/style"}, + }) + post, ok := opByName(doc, "postMerged") + require.True(t, ok) + require.NotNil(t, post.Request) + require.Len(t, post.Request.Contents, 1) + assertKept(t, post.Request.Contents[0].Unmodeled, []keptEntry{{"openapi:encoding/f/allowReserved", `true`, + "/paths/~1merged/post/requestBody/content/multipart~1form-data/encoding/f/allowReserved"}}) + scheme, ok := doc.Auth["auth/openapi/components/securitySchemes/key"] + require.True(t, ok) + assertKept(t, scheme.Unmodeled, []keptEntry{ + {"openapi:bearerFormat", `"JWT"`, "/components/securitySchemes/key/bearerFormat"}, + }) +} + +// keptEntry is an Unmodeled entry expected under key, holding value as JSON, +// located at the pointer at. +type keptEntry struct{ key, value, at string } + +// assertKept requires each of want in got, with its value and location. +func assertKept(t *testing.T, got ir.Unmodeled, want []keptEntry) { + t.Helper() + for _, w := range want { + entry := unmodeledEntry(t, got, w.key) + assert.Equal(t, w.value, string(entry.Value), w.key) + assert.Equal(t, jsontext.Pointer(w.at), entry.Provenance.Pointer, w.key) + } +} diff --git a/compilers/openapi/helpers_test.go b/compilers/openapi/helpers_test.go index 549a31398..3336d3b3f 100644 --- a/compilers/openapi/helpers_test.go +++ b/compilers/openapi/helpers_test.go @@ -51,7 +51,7 @@ func loweredFor(t *testing.T, src string) (*lowerer, []ir.Diagnostic) { func lowerSpec(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { t.Helper() l, diags := loweredFor(t, src) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) // named components; the entry Compile's run() calls first + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) // named components; the entry Compile's run() calls first doc := l.document(t) return doc, append(diags, l.diags.List()...) } @@ -65,11 +65,11 @@ func lowerServiceSpec(t *testing.T, src string) (*ir.Document, ir.Service, []ir. require.NoError(t, err) require.NotNil(t, loadedDoc) l := newLowerer(loadedDoc, Options{}.withDefaults()) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) auth, authDiags := auth.LowerSecuritySchemes(l.ctx) l.out.Auth = auth l.diags.AppendAll(authDiags) - svc, tagDefs, svcDiags := operation.LowerService(t.Context(), l.ctx.WithAuth(l.out.Auth), l.types, &l.anchors) + svc, tagDefs, svcDiags := operation.LowerService(t.Context(), l.ctx.WithAuth(l.out.Auth), l.types) l.out.TagDefs = tagDefs l.diags.AppendAll(svcDiags) l.out.Services = []ir.Service{svc} @@ -81,16 +81,15 @@ func lowerServiceSpec(t *testing.T, src string) (*ir.Document, ir.Service, []ir. } // lowerer is test scaffolding, not a compiler type. The compiler has no such -// struct: every lowering is a function of the context, the registry and the -// memos, and run owns the document being built (#177). Tests that drive one -// lowering in isolation still want those five things in one place, and holding -// them here keeps that convenience out of the production call graph. +// struct: every lowering is a function of the context and the registry, and +// run owns the document being built (#177). Tests that drive one lowering in +// isolation still want those four things in one place, and holding them here +// keeps that convenience out of the production call graph. type lowerer struct { - ctx lowering.Ctx - out *ir.Document - types *compile.Types - diags compile.Diags - anchors schema.AnchorIndex + ctx lowering.Ctx + out *ir.Document + types *compile.Types + diags compile.Diags } // lowererOver is the only place the fixture's fields are initialised. Both diff --git a/compilers/openapi/internal/anchors/anchors.go b/compilers/openapi/internal/anchors/anchors.go new file mode 100644 index 000000000..9ecef534e --- /dev/null +++ b/compilers/openapi/internal/anchors/anchors.go @@ -0,0 +1,127 @@ +// Package anchors indexes the $dynamicAnchor declarations of a source +// document, which a $dynamicRef expands against. It reads the tree as the +// parser does, through aliases and `<<` merges: an anchor a merge or an alias +// pulls into a position is declared there as well, so "declared exactly once" +// counts what the document declares rather than what it spells out. +package anchors + +import ( + "encoding/json/jsontext" + "slices" + "strconv" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/ids" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" +) + +// MaxDepth bounds how deep the index walk reads. Document trees nest far +// shallower — a schema under a media type under a response is about ten +// levels — so it only ever stops a pathological structure. +const MaxDepth = 512 + +// MaxNodes bounds how many nodes the walk reads in all. Depth alone does not +// bound it: the walk follows aliases, and an alias graph is a DAG, so a shallow +// document can present far more paths than it has nodes. +const MaxNodes = 1 << 20 + +// Index is a document's $dynamicAnchor declarations: for each name, the +// pointers of the mappings declaring it, in document order. +type Index struct { + byName map[string][]jsontext.Pointer + complete bool +} + +// Sites returns the pointers declaring name. +func (x Index) Sites(name string) []jsontext.Pointer { return slices.Clone(x.byName[name]) } + +// Complete reports whether the walk read the whole tree. A partial index can +// only undercount, and an undercount reads as "declared exactly once", the one +// count that expands a $dynamicRef, so a reader of a partial one says so. +func (x Index) Complete() bool { return x.complete } + +// frame is one entry of the walk's stack: a node to read at a pointer and +// depth, or, where anchor is set, a mapping's $dynamicAnchor to record. +type frame struct { + node *yaml.Node + pointer jsontext.Pointer + depth int + anchor bool +} + +// Build indexes the document under root. It reads the raw tree, since the +// parser's schema model has no field for the keyword +// (docs/parser-library.md#unmodeled-keywords), position by position as the +// parser presents each mapping's pairs (nodeview.View.MappingPairs): a walk +// of Content (nodeview.Walk) would miss what a merge brings in. It is bounded +// by MaxDepth and MaxNodes, and by the view's merge bound, each reported by +// Complete. +func Build(root *yaml.Node) Index { + return build(root, MaxDepth, MaxNodes) +} + +// build is Build within maxDepth levels and maxNodes nodes. +func build(root *yaml.Node, maxDepth, maxNodes int) Index { + view := nodeview.New() + x := Index{byName: map[string][]jsontext.Pointer{}, complete: true} + stack := []frame{{node: root}} + for budget := maxNodes; len(stack) > 0; { + f := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if f.anchor { + x.record(f.node, f.pointer) + continue + } + n := nodeview.Deref(f.node) + if n == nil { + continue + } + if f.depth > maxDepth || budget == 0 { + x.complete = false + continue + } + budget-- + stack = append(stack, children(view, n, f)...) + } + x.complete = x.complete && !view.Exhausted() + return x +} + +// children returns what the walk reads below n, read at f, last first so the +// stack yields them in order: a document's content, a sequence's items, a +// mapping's values, and a mapping's $dynamicAnchor where it writes one. A +// scalar holds nothing. +func children(view *nodeview.View, n *yaml.Node, f frame) []frame { + var out []frame + switch n.Kind { + case yaml.DocumentNode: + for _, c := range n.Content { + out = append(out, frame{node: c, pointer: f.pointer, depth: f.depth + 1}) + } + case yaml.SequenceNode: + for i, c := range n.Content { + out = append(out, frame{node: c, pointer: f.pointer + ids.Ptr(strconv.Itoa(i)), depth: f.depth + 1}) + } + case yaml.MappingNode: + for _, p := range view.MappingPairs(n) { + if p.Key == "$dynamicAnchor" { + out = append(out, frame{node: p.Val, pointer: f.pointer, anchor: true}) + continue + } + out = append(out, frame{node: p.Val, pointer: f.pointer + ids.Ptr(p.Key), depth: f.depth + 1}) + } + default: + } + slices.Reverse(out) + return out +} + +// record indexes the anchor val names at pointer. A value that is not a +// non-empty scalar names nothing a $dynamicRef could spell. +func (x Index) record(val *yaml.Node, pointer jsontext.Pointer) { + if val == nil || val.Kind != yaml.ScalarNode || val.Value == "" { + return + } + x.byName[val.Value] = append(x.byName[val.Value], pointer) +} diff --git a/compilers/openapi/internal/anchors/anchors_bench_test.go b/compilers/openapi/internal/anchors/anchors_bench_test.go new file mode 100644 index 000000000..8cb2d1093 --- /dev/null +++ b/compilers/openapi/internal/anchors/anchors_bench_test.go @@ -0,0 +1,42 @@ +package anchors_test + +import ( + "os" + "testing" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/anchors" +) + +// BenchmarkBuild measures what deriving the $dynamicAnchor index at entry +// would cost a document that never asks for it. +// +// The index stays a lazy memo (lowering.Ctx.DynamicAnchorSites) rather than +// being built with the context: the walk is not free, almost no document +// writes $dynamicRef, and building it emits a diagnostic that at entry would +// reach documents that never use the keyword. +// +// Read the number against BenchmarkCompile_Petstore in compilers/openapi; the +// design's claim is a ratio. +func BenchmarkBuild(b *testing.B) { + // Four levels up: this package sits at compilers/openapi/internal/anchors, + // and the corpus is at the repo root. A missing or unparseable fixture + // stops the benchmark rather than skipping it, since a skip is silent + // without -v and exits 0. + data, err := os.ReadFile("../../../../testdata/conformance/openapi/allof-inline-merge.yaml") + if err != nil { + b.Fatalf("corpus fixture unavailable: %v", err) + } + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + b.Fatalf("fixture does not parse: %v", err) + } + + b.ReportAllocs() + for b.Loop() { + if !anchors.Build(&root).Complete() { + b.Fatal("the fixture should not exhaust the walk bounds") + } + } +} diff --git a/compilers/openapi/internal/anchors/anchors_internal_test.go b/compilers/openapi/internal/anchors/anchors_internal_test.go new file mode 100644 index 000000000..0397a30e7 --- /dev/null +++ b/compilers/openapi/internal/anchors/anchors_internal_test.go @@ -0,0 +1,199 @@ +package anchors + +import ( + "encoding/json/jsontext" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" + "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" +) + +// TestBuild_WalksEveryNodeShape drives the walk over the shapes a YAML +// document can present, rather than only the mappings a schema happens to be +// written as. It meets whatever the source wrote — a sequence, a bare scalar, +// a key that is no scalar — and must index anchors under a sequence and under +// the empty key the parser reads such a key as, while declining the rest. +func TestBuild_WalksEveryNodeShape(t *testing.T) { + t.Parallel() + cases := []struct { + name string + node *yaml.Node + want map[string][]jsontext.Pointer + }{ + {"a nil node yields nothing", nil, map[string][]jsontext.Pointer{}}, + { + "a bare scalar declares no anchor", + openapitest.YAMLNode(t, `just-a-string`), + map[string][]jsontext.Pointer{}, + }, + { + "a sequence indexes its elements by ordinal", + openapitest.YAMLNode(t, "- {$dynamicAnchor: first}\n- {other: 1}\n- {$dynamicAnchor: third}\n"), + map[string][]jsontext.Pointer{"first": {"/0"}, "third": {"/2"}}, + }, + { + "a sequence element standing in for a mapping is followed", + openapitest.YAMLNode(t, "- &a {$dynamicAnchor: first}\n- *a\n"), + map[string][]jsontext.Pointer{"first": {"/0", "/1"}}, + }, + { + "a key that is no scalar is read as the empty string", + openapitest.YAMLNode(t, "? [a, b]\n: {$dynamicAnchor: buried}\n$dynamicAnchor: reached\n"), + map[string][]jsontext.Pointer{"buried": {"/"}, "reached": {""}}, + }, + { + "an empty anchor name is not indexed", + openapitest.YAMLNode(t, `{$dynamicAnchor: ""}`), + map[string][]jsontext.Pointer{}, + }, + { + "a non-scalar anchor value is not indexed", + openapitest.YAMLNode(t, `{$dynamicAnchor: [a]}`), + map[string][]jsontext.Pointer{}, + }, + { + "an anchor written as an alias of a scalar is indexed under the scalar", + openapitest.YAMLNode(t, "names: [&n tail]\nA: {$dynamicAnchor: *n}\n"), + map[string][]jsontext.Pointer{"tail": {"/A"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + x := Build(tc.node) + assert.Empty(t, cmp.Diff(tc.want, x.byName)) + assert.True(t, x.Complete(), "nothing here reaches a walk bound") + }) + } +} + +// TestBuild_CountsWhatAnAliasBringsIn pins the count the whole expansion rests +// on. A YAML alias and a `<<` merge key each hand a second position the anchor +// the first one wrote, and the parser reads both positions as declaring it, so +// an index that walked only the spelled-out tree would report one declaration +// where the document has two, and "declared exactly once" would expand a +// reference that is genuinely ambiguous. +func TestBuild_CountsWhatAnAliasBringsIn(t *testing.T) { + t.Parallel() + cases := []struct { + name, source string + want []jsontext.Pointer + }{ + { + name: "an alias re-declares the anchor at its own position", + source: "A: &base {$dynamicAnchor: tail}\nB: *base\n", + want: []jsontext.Pointer{"/A", "/B"}, + }, + { + name: "a merge key does too", + source: "A: &base {$dynamicAnchor: tail}\nB: {<<: *base, description: merged}\n", + want: []jsontext.Pointer{"/A", "/B"}, + }, + { + name: "an explicit key still beats the merged one", + source: "A: &base {$dynamicAnchor: tail}\nB: {<<: *base, $dynamicAnchor: own}\n", + want: []jsontext.Pointer{"/A"}, + }, + { + name: "a key written twice declares the anchor its last copy names", + source: "A: {$dynamicAnchor: own, $dynamicAnchor: tail}\n", + want: []jsontext.Pointer{"/A"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + x := Build(openapitest.YAMLNode(t, tc.source)) + assert.True(t, x.Complete()) + assert.Equal(t, tc.want, x.Sites("tail")) + }) + } +} + +// TestBuild_DocumentNodeUnwraps pins that the walk accepts a whole document as +// well as a content node, since it is handed whatever the loader kept. +func TestBuild_DocumentNodeUnwraps(t *testing.T) { + t.Parallel() + var doc yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("$dynamicAnchor: top\n"), &doc)) + require.Equal(t, yaml.DocumentNode, doc.Kind) + + x := Build(&doc) + assert.Equal(t, map[string][]jsontext.Pointer{"top": {""}}, x.byName) + assert.True(t, x.Complete()) +} + +// TestBuild_StopsAtTheDepthCap pins the depth bound at its boundary. A raw tree +// deeper than the cap is a legal document, so the walk must stop rather than +// read on; an anchor within the cap is still indexed, and the truncation is +// reported rather than passed off as a complete index. +func TestBuild_StopsAtTheDepthCap(t *testing.T) { + t.Parallel() + within, past := Build(nestedAnchor(MaxDepth)), Build(nestedAnchor(MaxDepth+1)) + + assert.Equal(t, []jsontext.Pointer{jsontext.Pointer(strings.Repeat("/k", MaxDepth))}, within.Sites("deep"), + "the deepest anchor the cap admits is indexed") + assert.True(t, within.Complete()) + assert.Empty(t, past.Sites("deep"), "one level further is not read") + assert.False(t, past.Complete(), "and says so, since an unseen anchor undercounts every name") +} + +// TestBuild_StopsAtTheNodeBudget pins the other bound. Depth alone does not +// bound a walk that follows aliases, since an alias DAG presents many more +// paths than the tree has nodes; the budget is what caps the total. +func TestBuild_StopsAtTheNodeBudget(t *testing.T) { + t.Parallel() + source := openapitest.YAMLNode(t, "a: {$dynamicAnchor: first}\nb: {$dynamicAnchor: second}\n") + + cut := build(source, MaxDepth, 2) // the root mapping and its first value, and no more + assert.False(t, cut.Complete(), "the budget ran out mid-walk") + assert.Equal(t, []jsontext.Pointer{"/a"}, cut.Sites("first")) + assert.Empty(t, cut.Sites("second"), "so the anchors past it are missing") + + whole := build(source, MaxDepth, 3) + assert.True(t, whole.Complete(), "a budget of every node reads them all") + assert.Equal(t, []jsontext.Pointer{"/b"}, whole.Sites("second")) +} + +// TestBuild_ReportsAMergeChainPastTheBound pins the third bound: a mapping +// whose `<<` chain runs deeper than the view reads holds keys the walk never +// sees, which reads as an undercount just as a depth cut does. +func TestBuild_ReportsAMergeChainPastTheBound(t *testing.T) { + t.Parallel() + within := Build(openapitest.YAMLNode(t, openapitest.NestedMerges(nodeview.MergeDepthLimit, "{$dynamicAnchor: tail}"))) + past := Build(openapitest.YAMLNode(t, openapitest.NestedMerges(nodeview.MergeDepthLimit+1, "{$dynamicAnchor: tail}"))) + + assert.True(t, within.Complete()) + assert.NotEmpty(t, within.Sites("tail"), "a chain the view reads brings the anchor in") + assert.False(t, past.Complete(), "one past the bound is reported") +} + +// TestSites_ReturnsACopy pins that a reader cannot change the index through +// what Sites hands it. +func TestSites_ReturnsACopy(t *testing.T) { + t.Parallel() + x := Build(openapitest.YAMLNode(t, "A: {$dynamicAnchor: tail}\n")) + x.Sites("tail")[0] = "/B" + assert.Equal(t, []jsontext.Pointer{"/A"}, x.Sites("tail")) +} + +// nestedAnchor builds a chain of depth mappings, each holding the next under +// the key k, around one declaring $dynamicAnchor deep, so the declaring +// mapping sits depth levels below the root. It is built rather than +// parsed because nesting this deep in source text is indentation arithmetic. +func nestedAnchor(depth int) *yaml.Node { + n := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{openapitest.StrNode("$dynamicAnchor"), openapitest.StrNode("deep")}, + } + for range depth { + n = &yaml.Node{Kind: yaml.MappingNode, Content: []*yaml.Node{openapitest.StrNode("k"), n}} + } + return n +} diff --git a/compilers/openapi/internal/annotation/annotation_internal_test.go b/compilers/openapi/internal/annotation/annotation_internal_test.go index e8f7227e7..0c477a60e 100644 --- a/compilers/openapi/internal/annotation/annotation_internal_test.go +++ b/compilers/openapi/internal/annotation/annotation_internal_test.go @@ -15,6 +15,7 @@ import ( yaml "gopkg.in/yaml.v3" "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/ir" ) @@ -155,6 +156,13 @@ func TestDialectAt_AttributesTheKeywordRatherThanTheSchema(t *testing.T) { // the compiler's loader parses documents with, so the raw nodes the verbatim // readers read off are present. A schema built in Go carries none, which // TestValidationOnlyAt_NeedsTheRawNode covers separately. +// pastMergeBound writes body as a flow mapping merged in through a chain of +// `<<` one level deeper than nodeview.MergeDepthLimit, which the parser reads +// and the compiler's view does not reach. +func pastMergeBound(body string) string { + return openapitest.NestedMerges(nodeview.MergeDepthLimit+1, body) +} + func schemaFromYAML(t *testing.T, body string) *oas3.Schema { t.Helper() var js oas3.JSONSchema[oas3.Referenceable] @@ -209,8 +217,9 @@ func TestRead_NodeTypeWithNoXMLObjectHasNothingToFill(t *testing.T) { // TestRead_NodeTypeOnlyADefinedValueIsPlaced pins what reaches XMLHints.NodeType // on 3.2: one of the five node types the specification defines. Anything else is // kept verbatim under the xml object's scope with a warning, an attribute flag -// the nodeType contradicts is kept the same way rather than overwritten, and a -// nodeType merged in through `<<` keeps the census's report. +// the nodeType contradicts is kept the same way rather than overwritten, a +// nodeType merged in through `<<` is read as one written there is, and one +// merged in from past nodeview.MergeDepthLimit keeps the census's report. func TestRead_NodeTypeOnlyADefinedValueIsPlaced(t *testing.T) { t.Parallel() for _, tc := range []struct { @@ -228,7 +237,8 @@ func TestRead_NodeTypeOnlyADefinedValueIsPlaced(t *testing.T) { {"agrees with attribute", "{attribute: true, nodeType: attribute}", "attribute", "", ""}, {"contradicts attribute", "{attribute: true, nodeType: element}", "element", "openapi:xml/attribute", diag.DegradedConstruct}, - {"merged in", "{<<: {nodeType: cdata}, name: q}", "", "", diag.UnknownKeyUnreachable}, + {"merged in", "{<<: {nodeType: cdata}, name: q}", "cdata", "", ""}, + {"merged in past the bound", pastMergeBound("{nodeType: cdata, name: q}"), "", "", diag.UnknownKeyUnreachable}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/compilers/openapi/internal/annotation/nodes.go b/compilers/openapi/internal/annotation/nodes.go index 3dd078b23..a5148a130 100644 --- a/compilers/openapi/internal/annotation/nodes.go +++ b/compilers/openapi/internal/annotation/nodes.go @@ -6,120 +6,21 @@ import ( oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" "github.com/speakeasy-api/openapi/yml" yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" ) // RawPropertyNode returns the raw YAML value node of a top-level schema -// keyword, or nil when absent. The library's GetPropertyNode resolves Go core -// field names and returns key nodes; this scans the schema's root mapping for -// the on-wire keyword and returns its value node, which is where exact literals -// live. +// keyword, its alias followed, or nil when absent. The library's +// GetPropertyNode resolves Go core field names and returns key nodes; this +// reads the schema's root mapping for the on-wire keyword as the parser reads +// it (nodeview.Child), through its `<<` merges, and returns its value node, +// which is where exact literals live. func RawPropertyNode(s *oas3.Schema, key string) *yaml.Node { if s == nil { return nil } - return RawChildNode(s.GetRootNode(), key) -} - -// RawChildNode returns the raw YAML value node of a mapping child keyed by the -// on-wire name, unwrapping a document node first; nil when absent. It reads -// exact literals the high-level model does not preserve (links, servers, -// content maps). -// -// The last pair spelling the key wins, as for the parser, which skips every -// earlier occurrence (docs/parser-library.md#duplicate-key). An explicit pair -// and an aliased one are one key to the parser and two nodes here, so the first -// would describe a mapping by a value nothing else in the compiler uses. -func RawChildNode(root *yaml.Node, key string) *yaml.Node { - if root == nil { - return nil - } - if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { - root = root.Content[0] - } - if root.Kind != yaml.MappingNode { - return nil - } - var found *yaml.Node - for i := 0; i+1 < len(root.Content); i += 2 { - if keyName(root.Content[i]) == key { - found = root.Content[i+1] - } - } - return found -} - -// RawMappingKeys returns the on-wire names a raw mapping writes, each once, in -// first-written order, unwrapping a document node first; nil for a node that is -// not a mapping or writes no key. Each name resolves through RawChildNode. -// -// It exists for a Path Item Object, whose unmarshaller skips a key with an -// anchored value (GitHub #412). Anchors are cleared first (GitHub #459, #501, -// #538), so it is needed only past #538's hop bound. -// -// A `<<` merge key is left out and its merged pairs are not read (GitHub #395). -// A repeated key is returned once, so a census does not report a collision the -// document lacks. -func RawMappingKeys(root *yaml.Node) []string { - if root == nil { - return nil - } - if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { - root = root.Content[0] - } - if root.Kind != yaml.MappingNode { - return nil - } - var keys []string - seen := make(map[string]bool, len(root.Content)/2) - for i := 0; i+1 < len(root.Content); i += 2 { - k := root.Content[i] - if isMergeKey(k) { - continue - } - name := keyName(k) - if seen[name] { - continue - } - seen[name] = true - keys = append(keys, name) - } - return keys -} - -// keyName is the on-wire name a mapping key node spells, following an alias to -// the scalar it stands for. -// -// yaml.v3 leaves an alias node's own Value as the anchor name, so a key written -// as an alias matches nothing when read raw — while the parser reads that key -// under the name it resolves to, which is the name every caller here looks up. -// Comparing the two spellings is what let a key the model reported as -// undeclared reach no Unmodeled entry at all (GitHub #297). -func keyName(n *yaml.Node) string { - if n.Kind == yaml.AliasNode && n.Alias != nil { - return n.Alias.Value - } - return n.Value -} - -// RawChildNodes is RawChildNode for every key of a mapping at once: each name -// the mapping writes, to the node RawChildNode returns for it. A caller looking -// up many keys of one large mapping reads its pairs once rather than once per -// key. It is nil for a missing node and for one that is not a mapping. -func RawChildNodes(root *yaml.Node) map[string]*yaml.Node { - if root == nil { - return nil - } - if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { - root = root.Content[0] - } - if root.Kind != yaml.MappingNode { - return nil - } - children := make(map[string]*yaml.Node, len(root.Content)/2) - for i := 0; i+1 < len(root.Content); i += 2 { - children[keyName(root.Content[i])] = root.Content[i+1] - } - return children + return nodeview.Child(s.GetRootNode(), key) } // DeclaresAny reports whether s writes any of keywords. It reads the raw nodes diff --git a/compilers/openapi/internal/annotation/nodes_internal_test.go b/compilers/openapi/internal/annotation/nodes_internal_test.go index 6fa7ac5f8..bb592a375 100644 --- a/compilers/openapi/internal/annotation/nodes_internal_test.go +++ b/compilers/openapi/internal/annotation/nodes_internal_test.go @@ -1,201 +1,15 @@ package annotation import ( - "maps" - "slices" + "strings" "testing" + soa "github.com/speakeasy-api/openapi/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v3" - - "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" - "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" ) -// TestRawChildNode_ReadsOnlyAMappingChild pins the raw reader every verbatim -// keyword goes through. Reading pairs off a sequence would take one item for a -// key and the next for its value, and failing to step through a document node -// would report a keyword as absent that the source did write. -func TestRawChildNode_ReadsOnlyAMappingChild(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte("a: 1\n"), &doc)) - - require.Equal(t, yaml.DocumentNode, doc.Kind) - found := RawChildNode(&doc, "a") - require.NotNil(t, found, "a document node is stepped through to its mapping") - assert.Equal(t, "1", found.Value) - - assert.Nil(t, RawChildNode(nil, "a")) - assert.Nil(t, RawChildNode(&doc, "absent")) - assert.Nil(t, RawChildNode(openapitest.YAMLNode(t, "[a, 1]"), "a"), - "a sequence has no keyed children, even one whose items spell the key") - assert.Nil(t, RawChildNode(openapitest.YAMLNode(t, "plain"), "a"), "nor does a scalar") - assert.Nil(t, RawChildNode(&yaml.Node{Kind: yaml.DocumentNode}, "a"), "nor an empty document") -} - -// TestRawMappingKeys_SpellsWhatRawChildNodeCanFind pins the raw key reader to -// the raw child reader beside it: every name it returns is one RawChildNode -// finds a value for, and a name RawChildNode cannot find — one merged in through -// `<<` — is not returned. The census hands each name straight to the keeper, -// which reads the value with RawChildNode, so a name spelled any other way would -// be announced as unreachable for a key the document wrote plainly. -func TestRawMappingKeys_SpellsWhatRawChildNodeCanFind(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - src string - want []string - }{ - {"plain keys in written order", "b: 1\na: 2\n", []string{"b", "a"}}, - {"an anchored value is a key like any other", "a: &x {k: v}\nb: *x\n", []string{"a", "b"}}, - {"an aliased key is spelled by the scalar it stands for", - "names: [&n spelled]\n*n : 1\n", []string{"names", "spelled"}}, - {"a repeated key is returned once", "a: 1\nb: 2\na: 3\n", []string{"a", "b"}}, - {"a merge key is not a key the mapping writes", - "base: &b {merged: 1}\nuse:\n <<: *b\n own: 2\n", []string{"base", "use"}}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - root := openapitest.YAMLNode(t, tc.src) - got := RawMappingKeys(root) - assert.Equal(t, tc.want, got) - for _, key := range got { - assert.NotNil(t, RawChildNode(root, key), "RawChildNode finds a value for %q", key) - } - }) - } - - t.Run("a merge key's own pairs are read on the merging mapping", func(t *testing.T) { - t.Parallel() - use := RawChildNode(openapitest.YAMLNode(t, "base: &b {merged: 1}\nuse:\n <<: *b\n own: 2\n"), "use") - require.NotNil(t, use) - assert.Equal(t, []string{"own"}, RawMappingKeys(use), - "the merged-in key is the parser's to present, not the raw mapping's — GitHub #395") - }) -} - -// TestRawMappingKeys_ReadsOnlyAMapping pins the unwrap and refusal rules to -// RawChildNode's: a document node is stepped through, and anything that is not a -// mapping yields nothing rather than a partial read. -func TestRawMappingKeys_ReadsOnlyAMapping(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte("a: 1\n"), &doc)) - require.Equal(t, yaml.DocumentNode, doc.Kind) - assert.Equal(t, []string{"a"}, RawMappingKeys(&doc), "a document node is stepped through to its mapping") - - assert.Nil(t, RawMappingKeys(nil)) - assert.Nil(t, RawMappingKeys(openapitest.YAMLNode(t, "{}")), "a mapping writing no key yields none") - assert.Nil(t, RawMappingKeys(openapitest.YAMLNode(t, "[1, 2]")), "a sequence has no keys") - assert.Nil(t, RawMappingKeys(openapitest.YAMLNode(t, "plain")), "nor does a scalar") - assert.Nil(t, RawMappingKeys(&yaml.Node{Kind: yaml.DocumentNode}), "nor an empty document") -} - -// TestRawChildNode_IsNotTheMergeAwareView pins the difference between this -// reader and nodeview's, which is why both exist: what a keyword is preserved -// *as* is what the source spelled, while what a pointer or $ref *resolves to* -// is what the parser will see. -// -// Answering a raw read through the view would make a merged keyword appear at a -// schema that never wrote it and replace an alias by its target; the first two -// cases pin those divergences. The third pins an agreement: since GitHub #356 -// both take the last pair of a repeated key, as the parser does. -func TestRawChildNode_IsNotTheMergeAwareView(t *testing.T) { - t.Parallel() - - // use is the mapping under test in each case; the raw read of a top-level - // key is unambiguous, so it is safe to navigate with. - useOf := func(t *testing.T, src string) *yaml.Node { - t.Helper() - use := RawChildNode(openapitest.YAMLNode(t, src), "use") - require.NotNil(t, use, "the fixture must declare a `use` mapping") - return use - } - - t.Run("a merge key contributes nothing to the raw read", func(t *testing.T) { - t.Parallel() - use := useOf(t, "base: &b {title: merged}\nuse:\n <<: *b\n") - - assert.Nil(t, RawChildNode(use, "title"), - "the source wrote `<<`, not `title`, so nothing is preserved at title") - merged := nodeview.New().ChildByToken(use, "title") - require.NotNil(t, merged, "the parser, however, does see it") - assert.Equal(t, "merged", merged.Value) - }) - - t.Run("an aliased value is not dereferenced by the raw read", func(t *testing.T) { - t.Parallel() - use := useOf(t, "base: &b anchored\nuse: {title: *b}\n") - - raw := RawChildNode(use, "title") - require.NotNil(t, raw) - assert.Equal(t, yaml.AliasNode, raw.Kind, "the raw tree keeps the alias the source wrote") - viewed := nodeview.New().ChildByToken(use, "title") - require.NotNil(t, viewed, "the view resolves the key the raw tree kept aliased") - assert.Equal(t, "anchored", viewed.Value, "where the view stands the anchor in its place") - }) - - t.Run("a repeated key resolves alike on both sides", func(t *testing.T) { - t.Parallel() - use := useOf(t, "use: {title: first, title: last}\n") - - raw, viewed := RawChildNode(use, "title"), nodeview.New().ChildByToken(use, "title") - require.NotNil(t, raw, "the raw read finds the key") - require.NotNil(t, viewed, "and so does the view") - assert.Equal(t, "last", raw.Value, - "the raw read takes the pair the parser reads, not the first written") - assert.Equal(t, "last", viewed.Value, - "and the view takes the same one, so this is no longer a divergence") - }) -} - -// TestRawChildNode_FindsAKeyWrittenAsAnAlias pins the one spelling where the raw -// tree and the parsed model disagree about a key's name. yaml.v3 leaves an alias -// node's own Value as the anchor, so matching it raw looks for "k" while the -// parser has already read the pair under "aliasedKey" — and every caller here -// asks by the name the parser used. A key the census reported as undeclared then -// reached no Unmodeled entry and, before this, no diagnostic either. -func TestRawChildNode_FindsAKeyWrittenAsAnAlias(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte("anchor: &k aliasedKey\n*k : found\n"), &doc)) - - found := RawChildNode(&doc, "aliasedKey") - - require.NotNil(t, found, "the key is looked up by the name it resolves to") - assert.Equal(t, "found", found.Value) - assert.Nil(t, RawChildNode(&doc, "k"), "and not by the anchor it is written as") -} - -// TestRawChildNode_RepeatedKeyReadsTheLastPair holds this reader to the pair -// the parser reads, which is the last of a repeated key. -// -// The repeat is spelled with an alias because yaml.v3 refuses a plainly -// repeated key when decoding into a typed value, as the model parse does. An -// explicit pair and an aliased one are two nodes here and one key to the -// parser. -func TestRawChildNode_RepeatedKeyReadsTheLastPair(t *testing.T) { - t.Parallel() - for _, tc := range []struct{ name, body, want string }{ - {"aliased pair last", "anchor: &k dup\ndup: first\n*k : last\n", "last"}, - {"aliased pair first", "anchor: &k dup\n*k : first\ndup: last\n", "last"}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte(tc.body), &doc)) - - found := RawChildNode(&doc, "dup") - - require.NotNil(t, found) - assert.Equal(t, tc.want, found.Value, "the last pair spelling the key is the effective one") - }) - } -} - // TestRawPropertyNode_NilSchemaReadsNothing pins the nil guard on the schema // side of the same reader, which every caller relies on to ask about a position // that may have no body written at it. @@ -208,46 +22,41 @@ func TestRawPropertyNode_NilSchemaReadsNothing(t *testing.T) { assert.Nil(t, RawPropertyNode(s, "title"), "a keyword the schema did not write") } -// TestRawChildNodes_AgreesWithRawChildNode holds the all-keys reader to the one -// it batches, key by key: every name either reader answers for, a name merged -// in through `<<`, and one the mapping never writes. The fixtures are the -// spellings RawChildNode has a rule for: a key repeated through an alias, whose -// last pair wins; a key written as an alias, found under the name it resolves -// to; and a merge key, read as the pair it is rather than the pairs it names. -func TestRawChildNodes_AgreesWithRawChildNode(t *testing.T) { +// TestRawPropertyNode_ReadsAKeywordAsTheParserDoes pins the raw keyword reader +// every verbatim keyword and every bound goes through to the model it sits +// beside: a keyword merged in through `<<` and one written through an alias +// key are read (GitHub #395, #384), a keyword written twice at its last copy, +// a value written as an alias as what it names, and a schema whose whole body +// is an alias, which the model is built from as the alias node, through it. +func TestRawPropertyNode_ReadsAKeywordAsTheParserDoes(t *testing.T) { t.Parallel() - for _, tc := range []struct{ name, body string }{ - {"repeated key", "anchor: &k dup\ndup: first\n*k : last\n"}, - {"aliased key", "anchor: &k aliasedKey\n*k : found\n"}, - {"merge key", "base: &b {title: merged}\n<<: *b\nown: 1\n"}, + doc, _, err := soa.Unmarshal(t.Context(), strings.NewReader(`openapi: 3.1.0 +info: {title: t, version: "1"} +paths: {} +x-keys: [&max maximum] +x-base: &base {maximum: 9} +components: + schemas: + Merged: {type: integer, <<: *base} + AliasKey: {type: integer, *max : 7} + Repeated: {type: integer, maximum: 1, maximum: 2} + Anchored: &anchored {type: integer, maximum: &n 5} + Aliased: *anchored + AliasValue: {type: integer, maximum: *n} +`)) + require.NoError(t, err) + for name, want := range map[string]string{ + "Merged": "9", "AliasKey": "7", "Repeated": "2", "AliasValue": "5", "Aliased": "5", } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte(tc.body), &doc)) - - children := RawChildNodes(&doc) - - require.NotEmpty(t, children, "a document node is stepped through to its mapping") - keys := append([]string{"title", "absent"}, RawMappingKeys(&doc)...) - keys = slices.AppendSeq(keys, maps.Keys(children)) - for _, key := range keys { - assert.Same(t, RawChildNode(&doc, key), children[key], "the two readers disagree on %q", key) - } - }) + js, ok := doc.Components.Schemas.Get(name) + require.True(t, ok, name) + node := RawPropertyNode(js.GetSchema(), "maximum") + require.NotNil(t, node, "%s writes maximum as the parser reads it", name) + assert.Equal(t, yaml.ScalarNode, node.Kind, "%s: the value's alias is followed", name) + assert.Equal(t, want, node.Value, name) } } -// TestRawChildNodes_ReadsOnlyAMapping is TestRawChildNode_ReadsOnlyAMappingChild -// for the reader that batches it. -func TestRawChildNodes_ReadsOnlyAMapping(t *testing.T) { - t.Parallel() - assert.Nil(t, RawChildNodes(nil)) - assert.Nil(t, RawChildNodes(openapitest.YAMLNode(t, "[1, 2]")), "a sequence has no keyed children") - assert.Nil(t, RawChildNodes(openapitest.YAMLNode(t, "plain")), "nor does a scalar") - assert.Nil(t, RawChildNodes(&yaml.Node{Kind: yaml.DocumentNode}), "nor an empty document") -} - // TestDeclaresAny_AsksTheRawNodes pins the gate's agreement with the recorders it // gates: it must answer from the same raw nodes they read, so a keyword one sees // and the other does not cannot arise. diff --git a/compilers/openapi/internal/annotation/rawjson.go b/compilers/openapi/internal/annotation/rawjson.go index c4fa1e85a..707b39f17 100644 --- a/compilers/openapi/internal/annotation/rawjson.go +++ b/compilers/openapi/internal/annotation/rawjson.go @@ -5,10 +5,11 @@ import ( "encoding/json/v2" "errors" "fmt" - "time" + "strconv" yaml "gopkg.in/yaml.v3" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/value" "github.com/dexpace/morphic/ir" ) @@ -18,16 +19,16 @@ import ( // cannot be represented yields an error, so no caller announces a preservation // that never happened (GitHub #144). // -// Conversion fails on a non-string or repeated mapping key (an aliased repeat -// aside, GitHub #1072), a `<<` merging no mapping, .nan or .inf, a scalar whose -// text does not satisfy its tag (a !!null aside, GitHub #245), invalid UTF-8, -// and an alias that cycles or expands past the node budget. A tag yaml.v3 -// leaves untyped is kept. Numeric literals stay exact (GitHub #32). +// A mapping holds the members the parser reads (nodeview.View.MappingPairs). +// Conversion fails on a mapping that writes a key twice (GitHub #1072) or +// merges past nodeview.MergeDepthLimit, a scalar value.Classify refuses, +// invalid UTF-8, and an alias that cycles or expands past the node budget. +// Numeric literals stay exact (GitHub #32). func RawFromNode(node *yaml.Node) (ir.RawValue, error) { if node == nil { return nil, nil } - var conv rawConv + conv := rawConv{view: nodeview.New()} data, err := conv.node(node, 0) if err != nil { return nil, fmt.Errorf("render yaml node as json: %w", err) @@ -48,33 +49,10 @@ const ( maxRawNodes = 1 << 20 ) -// errMergeWantsMap reports a `<<` whose value is neither a mapping nor a -// sequence of mappings, matching yaml.v3's own refusal to guess. -var errMergeWantsMap = errors.New("map merge requires map or sequence of maps as the value") - -// maxAliasHops bounds an alias chain followed outside the node walk. An anchor -// may name an alias node, so a key or a merge source can sit a hop or more from -// the node it means; a chain past this is a cycle, and these two readers resolve -// without the budget node() would otherwise charge for the hop. -const maxAliasHops = 100 - -// resolveAlias follows an alias chain to the node it names. -func resolveAlias(n *yaml.Node) (*yaml.Node, error) { - for hops := 0; n.Kind == yaml.AliasNode; hops++ { - if hops >= maxAliasHops { - return nil, fmt.Errorf("alias %q chains past %d hops", n.Value, maxAliasHops) - } - if n.Alias == nil { - return nil, fmt.Errorf("alias %q resolves to nothing", n.Value) - } - n = n.Alias - } - return n, nil -} - // rawConv renders one YAML node as canonical JSON. It carries the node budget -// shared across the whole walk; depth is a parameter because it is a property -// of the path rather than of the conversion. +// shared across the whole walk and the view every mapping is read through; +// depth is a parameter because it is a property of the path rather than of the +// conversion. // // A scalar renders to JSON text directly, never through a Go value: decoding a // number into `any` rounds it through float64, the bug this file exists to close @@ -83,6 +61,7 @@ func resolveAlias(n *yaml.Node) (*yaml.Node, error) { // the JSON it holds rather than reinterpreting it. type rawConv struct { nodes int + view *nodeview.View } // node renders any YAML node as canonical JSON. @@ -110,7 +89,7 @@ func (c *rawConv) node(n *yaml.Node, depth int) (jsontext.Value, error) { } return c.node(n.Alias, depth+1) case yaml.ScalarNode: - return c.scalar(n) + return scalar(n) case yaml.SequenceNode: return c.sequence(n, depth) case yaml.MappingNode: @@ -120,76 +99,36 @@ func (c *rawConv) node(n *yaml.Node, depth int) (jsontext.Value, error) { } } -// scalar renders a scalar node by its resolved tag. -// -// The tag is read through ShortTag rather than off the node so that an untagged -// node resolves by its value, the way the Decode this replaced would have. A -// parser never hands one over — it resolves every scalar it emits, and writes -// the short form even for a spelled-out `!` — but a -// caller assembling nodes can, and reading n.Tag would spell such a scalar as a -// string. -// -// A tag yaml.v3 assigns no type to keeps its text rather than being refused, -// which is also what that Decode did. -func (c *rawConv) scalar(n *yaml.Node) (jsontext.Value, error) { - switch n.ShortTag() { - case "!!null": +// scalar renders a scalar node as value.Classify reads it, the classification +// an ir.Value is lowered from too, so a scalar kept verbatim and one lowered +// are never two kinds (GitHub #783). What is kept differs only where JSON has +// no type for the kind: a timestamp, a !!binary and a scalar under the +// document's own tag keep the text the source wrote, because the resolved form +// is derivable from the spelling and the spelling is not derivable from the +// resolved form (GitHub #242). +func scalar(n *yaml.Node) (jsontext.Value, error) { + s, err := value.Classify(n) + if err != nil { + return nil, err + } + switch s.Kind { + case value.ScalarNull: return jsontext.Value("null"), nil - case "!!bool": - var b bool - if err := n.Decode(&b); err != nil { - return nil, fmt.Errorf("bool literal %q: %w", n.Value, err) - } - if b { - return jsontext.Value("true"), nil - } - return jsontext.Value("false"), nil - case "!!int", "!!float": - // The whole point of this file: the literal's exact decimal, never a - // float64 rounding of it (GitHub #32). NumericLiteral resolves YAML's - // own bases too, so 0o17 still reads 15 rather than 17. - num, err := value.NumericLiteral(n) - if err != nil { - return nil, fmt.Errorf("numeric literal %q: %w", n.Value, err) - } - return spliceNumber(n.Value, string(num)) - case "!!str": - return jsonString(n.Value) - case "!!timestamp", "!!binary": - // The two tags YAML gives a type and JSON does not. Both keep the text - // the source wrote, because the resolved form is derivable from the - // spelling and the spelling is not derivable from the resolved form - // (GitHub #242). - // - // Rendering the resolved form instead lost data both ways: a timestamp - // came back RFC 3339, so `2021-1-1` acquired a padding, a time and a - // zone the source never wrote, and a !!binary came back as its decoded - // bytes, so `/w==` — the byte 0xFF — reached the IR as the U+FFFD v1's - // encoding/json substituted for it, indistinguishable from a source - // that wrote U+FFFD itself. - // - // The decode stays, and stays the tag check it has always been: a - // scalar that does not satisfy the type it declares is refused exactly - // as before, so only the kept spelling moved. Whether such a scalar - // should instead survive as text — JSON can name it, and refusing costs - // the caller the whole construct — is a question about what this walk - // accepts rather than what it preserves, and is open as GitHub #245 - // rather than settled as a side effect here. - return c.verbatimTagged(n) + case value.ScalarBool: + return jsontext.Value(strconv.FormatBool(s.Bool)), nil + case value.ScalarNumber: + return spliceNumber(n.Value, string(s.Num)) default: - // A tag yaml.v3 cannot resolve carries no type it could decode to, so - // its scalar comes back as the text it was written with — the behaviour - // this walk replaced, and the lossless one: refusing would drop an - // `!acme/thing` extension value the source did write. - return jsonString(n.Value) + // A string, and the kinds JSON names only as the text they were written as. + return jsonString(s.Text) } } -// spliceNumber renders num, an already-validated NumericLiteral result, as raw -// JSON, refusing it if it is not a JSON number. scalar is the one caller that -// splices a numeric literal straight into a document rather than an ir.Value -// field, so BigVal's contract (its text always renders as a JSON-valid number) -// is checked here. +// spliceNumber renders num, an already-validated number value.Classify read, +// as raw JSON, refusing it if it is not a JSON number. scalar is the one caller +// that splices a numeric literal straight into a document rather than an +// ir.Value field, so BigVal's contract (its text always renders as a +// JSON-valid number) is checked here. // // NewBigVal enforces that contract itself, so the check is unreachable through // scalar and tests call this directly. It stays because dropping it would turn @@ -202,34 +141,6 @@ func spliceNumber(source, num string) (jsontext.Value, error) { return jsontext.Value(num), nil } -// verbatimTagged renders a scalar whose tag YAML resolves and JSON cannot name. -// Each arm only checks the tag, by the decode that used to supply the output, -// and the one return states the rule they share: what is kept is the source -// text. The two decode to different Go types and report differently — a -// timestamp names the offending text, a binary payload deliberately does not. -func (c *rawConv) verbatimTagged(n *yaml.Node) (jsontext.Value, error) { - switch tag := n.ShortTag(); tag { - case "!!timestamp": - var when time.Time - if err := n.Decode(&when); err != nil { - return nil, fmt.Errorf("timestamp literal %q: %w", n.Value, err) - } - case "!!binary": - // yaml.v3 base64-decodes a !!binary node into a string (it rejects a - // []byte target), so the check decodes to string. - var decoded string - if err := n.Decode(&decoded); err != nil { - return nil, fmt.Errorf("binary literal: %w", err) - } - default: - // Unreachable through scalar, which routes only the two tags above - // here. It stays so a third tag added to that arm is answered rather - // than silently read as base64. - return nil, fmt.Errorf("scalar tag %q is not kept verbatim", tag) - } - return jsonString(n.Value) -} - // sequence renders a YAML sequence as a JSON array, in source order. func (c *rawConv) sequence(n *yaml.Node, depth int) (jsontext.Value, error) { out := jsontext.Value{'['} @@ -246,151 +157,51 @@ func (c *rawConv) sequence(n *yaml.Node, depth int) (jsontext.Value, error) { return append(out, ']'), nil } -// mapping renders a YAML mapping as a JSON object, members in sorted key order. +// mapping renders a YAML mapping as a JSON object holding the members the +// parser reads, in sorted key order. // // Sorted rather than source order on purpose: the decode this replaced produced // sorted keys, and sorted keys are what the IR's determinism invariant asks of // every map it serializes. +// +// A key written twice, by the mapping or a mapping it merges, is refused, as +// yaml.v3 refuses it by default: the member would hold one copy and drop the +// other. A key written through an alias is the key it names, so it repeats +// one written plainly (GitHub #1072). func (c *rawConv) mapping(n *yaml.Node, depth int) (jsontext.Value, error) { - members := map[string]jsontext.Value{} - if err := c.mappingInto(members, n, depth); err != nil { - return nil, err - } - - out, err := json.Marshal(members, json.Deterministic(true)) - if err != nil { - return nil, fmt.Errorf("mapping: %w", err) - } - return out, nil -} - -// mappingInto fills dst from n, leaving keys dst already holds untouched. That -// one rule is the whole of YAML's merge precedence: a mapping's own keys are -// written before its `<<` is read, and a sequence of merge sources is read in -// order, so the nearer declaration always wins. -func (c *rawConv) mappingInto(dst map[string]jsontext.Value, n *yaml.Node, depth int) error { - if n.Kind != yaml.MappingNode { - return fmt.Errorf("expected a mapping, got yaml node kind %d", n.Kind) + if key, in, repeated := c.view.Repeat(n); repeated { + return nil, repeatedKeyError(in, key) } - if err := checkUniqueKeys(n); err != nil { - return err + pairs := c.view.MappingPairs(n) + if c.view.Exhausted() { + return nil, fmt.Errorf("mapping merges through `<<` deeper than %d levels", nodeview.MergeDepthLimit) } - - var merge *yaml.Node - for i := 0; i+1 < len(n.Content); i += 2 { - key, val := n.Content[i], n.Content[i+1] - if isMergeKey(key) { - merge = val - continue - } - name, err := mapKey(key) + members := make(map[string]jsontext.Value, len(pairs)) + for _, p := range pairs { + enc, err := c.node(p.Val, depth+1) if err != nil { - return err - } - if _, taken := dst[name]; taken { - continue - } - enc, err := c.node(val, depth+1) - if err != nil { - return err - } - dst[name] = enc - } - - if merge == nil { - return nil - } - return c.merge(dst, merge, depth) -} - -// merge folds a `<<` value into dst: a mapping, an alias to one, or a sequence -// of either. -func (c *rawConv) merge(dst map[string]jsontext.Value, merge *yaml.Node, depth int) error { - if depth > maxRawDepth { - return fmt.Errorf("nesting exceeds %d", maxRawDepth) - } - if merge.Kind == yaml.SequenceNode { - for _, item := range merge.Content { - src, err := mergeSource(item) - if err != nil { - return err - } - if err := c.mappingInto(dst, src, depth+1); err != nil { - return err - } + return nil, err } - return nil - } - - src, err := mergeSource(merge) - if err != nil { - return err + members[p.Key] = enc } - return c.mappingInto(dst, src, depth+1) -} -// mergeSource resolves one merge source to the mapping it names. -func mergeSource(n *yaml.Node) (*yaml.Node, error) { - named, err := resolveAlias(n) + out, err := json.Marshal(members, json.Deterministic(true)) if err != nil { - return nil, err - } - if named.Kind != yaml.MappingNode { - return nil, errMergeWantsMap + return nil, fmt.Errorf("mapping: %w", err) } - return named, nil + return out, nil } -// checkUniqueKeys rejects a mapping that names one key twice, which yaml.v3 -// rejects by default and this compiler has therefore always refused. It names -// the earliest key written again, and where it is written next, in one pass: -// comparing every pair of keys cost the square of the mapping's width (GitHub -// #776). -func checkUniqueKeys(n *yaml.Node) error { - type key struct { - kind yaml.Kind - value string - } - first := make(map[key]int, len(n.Content)/2) - earliest, next := -1, -1 - for j := 0; j < len(n.Content); j += 2 { - k := key{kind: n.Content[j].Kind, value: n.Content[j].Value} - i, seen := first[k] - if !seen { - first[k] = j - continue - } - // A key's later repeats share its first index, so its first repeat is kept. - if earliest < 0 || i < earliest { - earliest, next = i, j +// repeatedKeyError names key, which the mapping n writes more than once, and +// the lines of its first two copies. +func repeatedKeyError(n *yaml.Node, key string) error { + var lines []int + for i := 0; i+1 < len(n.Content) && len(lines) < 2; i += 2 { + if k, own := nodeview.KeyOf(n.Content[i]); own && k == key { + lines = append(lines, n.Content[i].Line) } } - if earliest < 0 { - return nil - } - return fmt.Errorf("mapping key %q already defined at line %d", n.Content[next].Value, n.Content[earliest].Line) -} - -// mapKey gives a mapping key its JSON name, rejecting every key JSON has no -// name for. The rule matches yaml.v3's own: a mapping decodes into Go's JSON -// model only when every key is a string, and a non-string key is what makes the -// whole construct unrepresentable rather than merely awkward (GitHub #144). -func mapKey(n *yaml.Node) (string, error) { - named, err := resolveAlias(n) - if err != nil { - return "", err - } - if named.Kind != yaml.ScalarNode || named.ShortTag() != "!!str" { - return "", fmt.Errorf("mapping key %q is not a string", named.Value) - } - return named.Value, nil -} - -// isMergeKey reports whether a key node is YAML's `<<` merge key, by the same -// test yaml.v3 applies. -func isMergeKey(n *yaml.Node) bool { - return n.Kind == yaml.ScalarNode && n.Value == "<<" && - (n.Tag == "" || n.Tag == "!" || n.ShortTag() == "!!merge") + return fmt.Errorf("mapping key %q already defined at line %d, and written again at line %d", key, lines[0], lines[1]) } // jsonString encodes s as a JSON string with RFC 8785's minimal escaping, diff --git a/compilers/openapi/internal/annotation/rawjson_internal_test.go b/compilers/openapi/internal/annotation/rawjson_internal_test.go index 27554068d..b87ce5a59 100644 --- a/compilers/openapi/internal/annotation/rawjson_internal_test.go +++ b/compilers/openapi/internal/annotation/rawjson_internal_test.go @@ -11,7 +11,9 @@ import ( "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v3" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" + "github.com/dexpace/morphic/compilers/openapi/internal/ynode" ) // decodeAndMarshal is the conversion RawFromNode used before GitHub #32: decode @@ -123,7 +125,12 @@ func TestRawFromNode_RendersEveryScalarTag(t *testing.T) { {"quoted number stays a string", `"123"`, `"123"`}, {"empty string", `""`, `""`}, {"HTML-special characters are not escaped, matching the IR's minimal-escaping canonical form", `"a&c"`, `"a&c"`}, - {"out-of-float64-range plain scalar stays a string", "1e400", `"1e400"`}, + // A plain scalar past float64's range is the number it spells, as value + // lowering reads it (GitHub #783); quoted, it is the string it says. + {"out-of-float64-range plain scalar is a number", "1e400", "1e400"}, + {"out-of-float64-range negative is a number", "-1.8e308", "-1.8e308"}, + {"out-of-float64-range quoted scalar stays a string", `"1e400"`, `"1e400"`}, + {"merge text as a value is the text", "<<", `"<<"`}, // A tag YAML has a type for and JSON does not keeps the source text // (GitHub #242). The resolved form is derivable from the spelling; the // spelling is not derivable from the resolved form. @@ -156,10 +163,10 @@ func TestRawFromNode_RendersEveryScalarTag(t *testing.T) { } } -// TestRawFromNode_PreservesMergeAndOrderingSemantics pins the mapping rules the -// walk had to reimplement when it stopped delegating to yaml.v3's decoder. -// Losing any of them would be a silent regression: the construct still converts, -// just to something else. +// TestRawFromNode_PreservesMergeAndOrderingSemantics pins the mapping rules a +// payload is read by, which are the parser's (nodeview.View.MappingPairs). +// Losing any of them would be a silent regression: the construct still +// converts, just to something else. func TestRawFromNode_PreservesMergeAndOrderingSemantics(t *testing.T) { t.Parallel() tests := []struct{ name, yaml, want string }{ @@ -172,6 +179,16 @@ func TestRawFromNode_PreservesMergeAndOrderingSemantics(t *testing.T) { {"nested merge inside a merged map", "a: &m {<<: {deep: 1}, p: 2}\nb: {<<: *m}", `{"a":{"deep":1,"p":2},"b":{"deep":1,"p":2}}`}, {"alias to a scalar", "a: &n 5\nb: *n", `{"a":5,"b":5}`}, {"alias as a mapping key", "a: &k mykey\nb: {*k: v}", `{"a":"mykey","b":{"mykey":"v"}}`}, + {"every merge key merges", "{<<: {p: 1}, <<: {q: 2}}", `{"p":1,"q":2}`}, + {"a merge naming a scalar merges nothing", "{<<: 1, q: 2}", `{"q":2}`}, + {"a scalar in a merge list merges nothing", "{<<: [1, {p: 1}]}", `{"p":1}`}, + // A key is a scalar string under the Format section's Failsafe rule, so + // a key YAML types otherwise is the text it spells (GitHub #617). + {"a key of another scalar type is its text", "{1: a, true: b, null: c, ~: d}", + `{"1":"a","null":"c","true":"b","~":"d"}`}, + {"an alias key naming an integer is its text", "a: &k 1\nb: {*k: v}", `{"a":1,"b":{"1":"v"}}`}, + {"a locally tagged key is its text", "{!foo k: v}", `{"k":"v"}`}, + {"a key that is no scalar is the empty string", "{? [1, 2]\n: v}", `{"":"v"}`}, {"alias to a sequence", "a: &s [1, 2]\nb: *s", `{"a":[1,2],"b":[1,2]}`}, {"empty mapping", "{}", `{}`}, {"empty sequence", "[]", `[]`}, @@ -186,28 +203,31 @@ func TestRawFromNode_PreservesMergeAndOrderingSemantics(t *testing.T) { } } -// TestCheckUniqueKeys_NamesTheEarliestKeyWrittenAgain pins which duplicate a -// mapping is refused for: the key written earliest of those written again, at -// the line it was first written, whichever repeat comes first in the mapping. -func TestCheckUniqueKeys_NamesTheEarliestKeyWrittenAgain(t *testing.T) { +// TestRawFromNode_NamesTheRepeatedKey pins which repeat a mapping is refused +// for and how it is named: the key whose second copy comes first, among the +// mapping's own pairs and then each source it merges (nodeview.View.Repeat), +// at the lines of its first two copies. A key written through an alias is the +// key it names (GitHub #1072). +func TestRawFromNode_NamesTheRepeatedKey(t *testing.T) { t.Parallel() tests := []struct{ name, yaml, want string }{ - {"one key twice", "a: 1\nb: 2\na: 3\n", `mapping key "a" already defined at line 1`}, - {"a later key repeated first", "x: 1\ny: 1\ny: 2\nx: 2\n", `mapping key "x" already defined at line 1`}, - {"a later key repeated last", "x: 1\ny: 1\nx: 2\ny: 2\n", `mapping key "x" already defined at line 1`}, - {"a key three times", "z: 0\nw: 1\nw: 2\nw: 3\n", `mapping key "w" already defined at line 2`}, + {"one key twice", "a: 1\nb: 2\na: 3\n", `mapping key "a" already defined at line 1, and written again at line 3`}, + {"the key repeated first", "x: 1\ny: 1\ny: 2\nx: 2\n", `mapping key "y" already defined at line 2, and written again at line 3`}, + {"a key three times", "z: 0\nw: 1\nw: 2\nw: 3\n", `mapping key "w" already defined at line 2, and written again at line 3`}, + {"a key repeated through an alias", "&k rate: 1\n*k : 2\n", `mapping key "rate" already defined at line 1, and written again at line 2`}, + {"a key a merge source repeats", "base: &b {p: 1,\n p: 2}\nuse: {<<: *b}\n", `mapping key "p" already defined at line 1, and written again at line 2`}, {"no key twice", "a: 1\nb: 2\n", ""}, {"an alias key and a key spelled as its anchor's name", "a: &k v\n*k : 1\nk: 2\n", ""}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() - err := checkUniqueKeys(openapitest.YAMLNode(t, tc.yaml)) + _, err := RawFromNode(openapitest.YAMLNode(t, tc.yaml)) if tc.want == "" { assert.NoError(t, err) return } - assert.EqualError(t, err, tc.want) + assert.ErrorContains(t, err, tc.want) }) } } @@ -218,24 +238,15 @@ func TestCheckUniqueKeys_NamesTheEarliestKeyWrittenAgain(t *testing.T) { func TestRawFromNode_RefusesWhatJSONCannotName(t *testing.T) { t.Parallel() tests := []struct{ name, yaml, wantErr string }{ - {"integer key", "{1: v}", "not a string"}, - {"null key", "{null: v}", "not a string"}, - {"bool key", "{true: v}", "not a string"}, - {"sequence key", "{? [1, 2]\n: v}", "not a string"}, - {"nested non-string key", "{outer: {1: v}}", "not a string"}, - {"alias key naming an integer", "a: &k 1\nb: {*k: v}", "not a string"}, - {"alias key naming a sequence", "a: &k [1]\nb: {*k: v}", "not a string"}, - {"locally tagged key", "{!foo k: v}", "not a string"}, {"duplicate key", "{k: 1, k: 2}", "already defined"}, {"duplicate key after a merge", "{<<: {p: 1}, k: 1, k: 2}", "already defined"}, + {"duplicate key through an alias", "{&k rate: 1, *k : 2}", "already defined"}, + {"duplicate key in a merge source", "{<<: {p: 1, p: 2}}", "already defined"}, {"not a number", ".nan", "numeric literal"}, {"positive infinity", ".inf", "numeric literal"}, {"negative infinity", "-.inf", "numeric literal"}, - {"merge from a scalar", "{<<: 1}", "map merge requires"}, - {"merge from a sequence of scalars", "{<<: [1]}", "map merge requires"}, - {"merge from an alias to a scalar", "a: &n 1\nb: {<<: *n}", "map merge requires"}, - {"non-string key inside a merged mapping", "{<<: [{1: v}]}", "not a string"}, {"boolean tag on a non-boolean", "!!bool notabool", "bool literal"}, + {"null tag on text", "!!null abc", "null literal"}, // The tag check outlives the decode that used to supply the output: a // scalar that does not satisfy the type it declares is still refused, // so preserving the source spelling widened nothing (GitHub #242). @@ -314,22 +325,10 @@ func TestRawFromNode_RejectsMalformedNodes(t *testing.T) { {"unresolved alias", &yaml.Node{Kind: yaml.AliasNode, Value: "x"}, "resolves to nothing"}, {"childless document", &yaml.Node{Kind: yaml.DocumentNode}, "document node holds 0 children"}, { - "merge from an unresolved alias", - mappingOf(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!merge", Value: "<<"}, + "unresolved alias as a value", + mappingOf(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "k"}, &yaml.Node{Kind: yaml.AliasNode, Value: "x"}), - "resolves to nothing", - }, - { - "unresolved alias as a key", - mappingOf(&yaml.Node{Kind: yaml.AliasNode, Value: "x"}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "v"}), - "resolves to nothing", - }, - { - "alias chain longer than a source could mean", - mappingOf(aliasChain(maxAliasHops+1), - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "v"}), - "chains past", + "nil yaml node", }, // yaml.v3 never hands this walk a node whose Value is not valid UTF-8 — // a real parse only produces one from a well-formed source document — @@ -358,6 +357,26 @@ func TestRawFromNode_RejectsMalformedNodes(t *testing.T) { } } +// TestRawFromNode_ReadsKeyShapesNoParseProducesAsTheParserWould pins the +// shapes a hand-built tree can hold in a key or a merge, which the walk reads +// as nodeview reads them rather than refusing: a merge naming an alias with no +// target merges nothing, and a key that is such an alias reads as its own +// text, as the library reads both. +func TestRawFromNode_ReadsKeyShapesNoParseProducesAsTheParserWould(t *testing.T) { + t.Parallel() + merge := mappingOf(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!merge", Value: "<<"}, + &yaml.Node{Kind: yaml.AliasNode, Value: "x"}) + got, err := RawFromNode(merge) + require.NoError(t, err) + assert.Equal(t, `{}`, string(got)) + + key := mappingOf(&yaml.Node{Kind: yaml.AliasNode, Value: "x"}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "v"}) + got, err = RawFromNode(key) + require.NoError(t, err) + assert.Equal(t, `{"x":"v"}`, string(got)) +} + // TestRawFromNode_WalksThroughADocumentNode covers the entry a caller reaches // when it hands over a whole parsed document rather than a node inside one. func TestRawFromNode_WalksThroughADocumentNode(t *testing.T) { @@ -381,7 +400,10 @@ func TestRawFromNode_WalksThroughADocumentNode(t *testing.T) { // the row honest, since a divergence that quietly stopped being one would leave // the entry asserting nothing, the same hole as excluding the row. var rawDivergences = map[string]struct{ old, want string }{ - "2021-1-1": {`"2021-01-01T00:00:00Z"`, `"2021-1-1"`}, + "2021-1-1": {`"2021-01-01T00:00:00Z"`, `"2021-1-1"`}, + // A plain number past float64's range, which yaml.v3 resolves to a string + // and its decode kept as one, is the number it spells (GitHub #783). + "1e400": {`"1e400"`, `1e400`}, "2021-01-01 10:20:30": {`"2021-01-01T10:20:30Z"`, `"2021-01-01 10:20:30"`}, `!!binary aGVsbG8=`: {`"hello"`, `"aGVsbG8="`}, // The old spelling is the replacement character decodeAndMarshal's v2 @@ -491,56 +513,31 @@ func TestRawDivergences_NamesOnlyCorpusRows(t *testing.T) { } } -// aliasChain builds n alias nodes each naming the next, ending on a string. A -// parser reaches this shape only by anchoring an alias, and never this deep. -func aliasChain(n int) *yaml.Node { - node := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "end"} - for range n { - node = &yaml.Node{Kind: yaml.AliasNode, Value: "a", Alias: node} - } - return node -} - // mappingOf builds a one-pair mapping node, for the malformed shapes no parser // emits. func mappingOf(key, val *yaml.Node) *yaml.Node { return &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{key, val}} } -// TestRawConv_RefusesNodesNoCallerShouldPass covers the preconditions the walk -// asserts on itself rather than on its input. None is reachable through -// RawFromNode — it rejects a nil node before walking, every mapping handed to -// mappingInto has already been proven to be one, and scalar routes only two -// tags into verbatimTagged — so they are exercised here directly. They stay -// because dropping one turns a caller's mistake into a silently wrong answer -// rather than an answer at all. +// TestRawConv_RefusesNodesNoCallerShouldPass covers the precondition the walk +// asserts on itself rather than on its input. It is not reachable through +// RawFromNode, which rejects a nil node before walking, so it is exercised +// here directly. It stays because dropping it turns a caller's mistake into a +// silently wrong answer rather than an answer at all. func TestRawConv_RefusesNodesNoCallerShouldPass(t *testing.T) { t.Parallel() - var c rawConv + c := rawConv{view: nodeview.New()} got, err := c.node(nil, 0) require.Error(t, err, "a nil node is a caller bug, not an absent construct") assert.Nil(t, got) assert.Contains(t, err.Error(), "nil yaml node") - - err = c.mappingInto(map[string]jsontext.Value{}, openapitest.YAMLNode(t, "[1]"), 0) - require.Error(t, err, "filling a mapping from a sequence is a caller bug") - assert.Contains(t, err.Error(), "expected a mapping") - - // scalar routes only !!timestamp and !!binary into verbatimTagged, so no - // input reaches this arm. It answers rather than falling through to the - // base64 check, which is what a third tag added to that case would hit. - got, err = c.verbatimTagged(openapitest.YAMLNode(t, "plain")) - require.Error(t, err, "a tag scalar does not route here is a caller bug") - assert.Nil(t, got) - assert.Contains(t, err.Error(), `scalar tag "!!str" is not kept verbatim`) } -// TestSpliceNumber_RefusesANonJSONNumber covers the refusal scalar's !!int/ -// !!float arm cannot reach today, the same way TestRawConv_RefusesNodesNoCallerShouldPass -// covers verbatimTagged's default case: no value.NumericLiteral result can -// fail spliceNumber's IsValid check, since NewBigVal enforces BigVal's -// "always JSON-valid" contract itself now. Until GitHub #45 was fixed a binary +// TestSpliceNumber_RefusesANonJSONNumber covers the refusal scalar's number arm +// cannot reach today: no number value.Classify reads can fail spliceNumber's +// IsValid check, since NewBigVal enforces BigVal's "always JSON-valid" +// contract itself now. Until GitHub #45 was fixed a binary // exponent slipped through that contract, and this check was what caught it // here. It stays as insurance against a future regression in that contract, so // it is exercised directly with a literal NumericLiteral itself would now @@ -555,27 +552,22 @@ func TestSpliceNumber_RefusesANonJSONNumber(t *testing.T) { assert.Contains(t, err.Error(), "which is not JSON") } -// TestRawFromNode_BoundsAMergeChainThatNeverRevisitsANode proves the bound on -// the one recursion that does not pass through node(): a `<<` whose source is -// itself nothing but a `<<`. Each level adds depth while adding no value to -// walk, so only merge's own check stops it. -// -// The chain is built rather than parsed because yaml.v3 caps parse nesting at -// the same figure, so no document could express one this deep inline; aliases -// are how a real source would reach it. -func TestRawFromNode_BoundsAMergeChainThatNeverRevisitsANode(t *testing.T) { +// TestRawFromNode_RefusesAMergeChainPastTheBound pins the bound on the merges +// a mapping is read through: a `<<` whose source is itself nothing but a `<<` +// adds depth while adding no value to walk, so only the view's own bound +// (nodeview.MergeDepthLimit) stops it, and a payload missing the members past +// it is refused rather than kept short. A chain at the bound is read whole. +func TestRawFromNode_RefusesAMergeChainPastTheBound(t *testing.T) { t.Parallel() - innermost := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - chain := innermost - for range maxRawDepth + 10 { - chain = mappingOf(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!merge", Value: "<<"}, chain) - } - - got, err := RawFromNode(chain) + got, err := RawFromNode(ynode.MergeChain(nodeview.MergeDepthLimit + 1)) require.Error(t, err, "a merge chain past the depth bound is refused") assert.Nil(t, got) - assert.Contains(t, err.Error(), "nesting exceeds") + assert.Contains(t, err.Error(), "merges through `<<` deeper than") + + got, err = RawFromNode(ynode.MergeChain(nodeview.MergeDepthLimit)) + require.NoError(t, err) + assert.Equal(t, `{"leaf":"v"}`, string(got)) } // TestRawFromNode_ResolvesAnUntaggedScalarByItsValue pins the one input for @@ -617,7 +609,7 @@ func TestRawFromNode_DistinguishesAbsentFromUnconvertible(t *testing.T) { require.Error(t, err, "a value that decodes but does not marshal is a failure") assert.Nil(t, got) - got, err = RawFromNode(openapitest.YAMLNode(t, "{? [1, 2]\n: v}")) - require.Error(t, err, "a mapping with a non-string key does not decode into the JSON model") + got, err = RawFromNode(openapitest.YAMLNode(t, "{k: 1, k: 2}")) + require.Error(t, err, "a mapping writing a key twice holds no one JSON object") assert.Nil(t, got) } diff --git a/compilers/openapi/internal/annotation/scalar_internal_test.go b/compilers/openapi/internal/annotation/scalar_internal_test.go new file mode 100644 index 000000000..f763a7043 --- /dev/null +++ b/compilers/openapi/internal/annotation/scalar_internal_test.go @@ -0,0 +1,81 @@ +package annotation + +import ( + "encoding/json/jsontext" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" + "github.com/dexpace/morphic/compilers/openapi/internal/value" + "github.com/dexpace/morphic/ir" +) + +// TestScalarRenderers_AgreeOnKind holds the two renderers of a YAML scalar, an +// ir.Value and a payload kept verbatim, to one classification (value.Classify, +// GitHub #784): over every tag, style and spelling classification turns on, +// both refuse a scalar or both render it, as a number, a bool, a null or a +// string alike. Two differences are deliberate, each a kind JSON has no type +// for: a !!binary is bytes in a value and its source text in a payload, and a +// scalar under the document's own tag has no value and keeps its text in a +// payload. +func TestScalarRenderers_AgreeOnKind(t *testing.T) { + t.Parallel() + for _, src := range []string{ + "null", "~", "!!null ''", "!!null abc", "true", "false", "!!bool maybe", + "42", "-7", "0644", "0o17", "0x1f", "0b1010", "1_000", "+5", "1.5", "1.10", ".5", "1e10", + "1.8e308", "-1.8e308", "1e400", ".inf", "-.inf", ".nan", "!!int 1.5", "!!float abc", "!!int 5", + `!!int "5"`, "! 5", "12345678901234567890123", + "hello", `"hello"`, "'hello'", `"1.8e308"`, "'42'", "!!str 42", "<<", "|\n block", ">\n folded", + "2021-1-1", "2021-01-01T10:20:30Z", "!!timestamp 2021-1-1", "!!timestamp notadate", + "!!binary aGVsbG8=", `!!binary "###"`, "!acme thing", "!!python/object x", + } { + t.Run(src, func(t *testing.T) { + t.Parallel() + node := openapitest.YAMLNode(t, src) + v, valueErr := value.FromNode(node) + raw, rawErr := RawFromNode(node) + s, classErr := value.Classify(node) + + if classErr != nil { + require.Error(t, valueErr, "a scalar Classify refuses no value holds") + require.Error(t, rawErr, "and no payload keeps") + return + } + require.NoError(t, rawErr) + switch s.Kind { + case value.ScalarBinary: + require.NoError(t, valueErr) + assert.Equal(t, ir.ValueBytes, v.Kind) + assert.Equal(t, jsontext.Kind('"'), raw.Kind(), "a payload keeps the source text") + case value.ScalarTagged: + require.Error(t, valueErr, "no ir.Value holds a document's own tag") + assert.Equal(t, jsontext.Kind('"'), raw.Kind(), "a payload keeps the text") + default: + require.NoError(t, valueErr) + assert.Equal(t, jsonKindOf(v), raw.Kind(), "the two renderers read one kind") + } + }) + } +} + +// jsonKindOf is the JSON kind the scalar ir.Value v is written as. +func jsonKindOf(v ir.Value) jsontext.Kind { + switch v.Kind { + case ir.ValueNull: + return 'n' + case ir.ValueBool: + if v.Bool { + return 't' + } + return 'f' + case ir.ValueNumber: + return '0' + case ir.ValueString: + return '"' + case ir.ValueBytes, ir.ValueList, ir.ValueObject, ir.ValueSymbol, ir.ValueRefKind, ir.ValueCtor: + return 0 + } + return 0 +} diff --git a/compilers/openapi/internal/annotation/unknown.go b/compilers/openapi/internal/annotation/unknown.go index ace6ac65f..f45f079d1 100644 --- a/compilers/openapi/internal/annotation/unknown.go +++ b/compilers/openapi/internal/annotation/unknown.go @@ -12,24 +12,24 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) -// unreachableKeyDiag reports a key the census named whose value the raw mapping -// does not present, so nothing of it reached the IR. +// unreachableKeyDiag reports a key the census named whose value the mapping, +// read as the parser reads it (nodeview.Child), does not present, so nothing of +// it reached the IR. // -// The document does write the key: the parser reads a mapping through its `<<` -// merge keys, so a key merged in from an anchored mapping is reported here -// while the mapping this reads holds no pair for it. Resolving that needs the -// merge-expanded view (internal/nodeview), which no raw-node reader here has -// (GitHub #395). +// The document does write the key: the parser reads every `<<` merge however +// deep, while the compiler reads none past nodeview.MergeDepthLimit, so a key +// merged in from past that bound is reported here. // // A warning, not the error unpreservableDiag gives: merging keys is legal input // that still lowers, and an error would refuse it under the default --fail-on. func unreachableKeyDiag(entry string, at ir.Provenance) ir.Diagnostic { return diag.Newf(ir.SeverityWarning, diag.UnknownKeyUnreachable, at, - "%s is written at a key the source mapping does not present directly, most likely "+ - "merged in through a `<<`; it is represented in the IR in no form at all", entry) + "%s is written at a key merged in through `<<` from deeper than the %d levels the compiler "+ + "reads; it is represented in the IR in no form at all", entry, nodeview.MergeDepthLimit) } // occupiedEntryDiag reports a key whose Unmodeled entry is already held by a @@ -168,13 +168,12 @@ func UnknownKeysNamed(p *ir.Unmodeled, keys []string, root *yaml.Node, // // A scalar the parser decodes as a string is returned, unless valid refuses a // non-empty one; empty or null is read as unset. Any other value is kept -// verbatim on p at the census's own entry, with a warning. A key root does not -// present directly, absent or merged in through `<<`, is not read; the census -// reports the second. +// verbatim on p at the census's own entry, with a warning. root is read as the +// parser reads it, through its `<<` merges (nodeview.Child). func TakeString(p *ir.Unmodeled, root *yaml.Node, key string, valid func(string) bool, locate Locator, owner jsontext.Pointer, scope string, ) (string, bool, []ir.Diagnostic) { - node := RawChildNode(root, key) + node := nodeview.Child(root, key) if node == nil { return "", false, nil } @@ -265,7 +264,7 @@ func keep(p *ir.Unmodeled, root *yaml.Node, key string, locate Locator, owner js if taken, occupied := (*p)[entry]; occupied { return []ir.Diagnostic{occupiedEntryDiag(entry, at, taken.Provenance.Pointer)} } - node := RawChildNode(root, key) + node := nodeview.Child(root, key) if node == nil { return []ir.Diagnostic{unreachableKeyDiag(entry, at)} } diff --git a/compilers/openapi/internal/annotation/unknown_internal_test.go b/compilers/openapi/internal/annotation/unknown_internal_test.go index ea870f892..d138c8bbb 100644 --- a/compilers/openapi/internal/annotation/unknown_internal_test.go +++ b/compilers/openapi/internal/annotation/unknown_internal_test.go @@ -458,12 +458,13 @@ func TestTakeString_DecodesAsTheParserDoes(t *testing.T) { } } -// TestTakeString_AppliesValidAndReadsOnlyDirectKeys pins what TakeString decides -// itself, whatever the parser decodes: a non-empty value valid refuses is kept -// verbatim as a value that does not decode is, an empty or null one is not put to -// valid, and a key the mapping does not present directly, absent or merged in -// through `<<`, is left unread for the census. -func TestTakeString_AppliesValidAndReadsOnlyDirectKeys(t *testing.T) { +// TestTakeString_AppliesValidAndReadsAsTheParserDoes pins what TakeString +// decides itself, whatever the parser decodes: a non-empty value valid refuses +// is kept verbatim as a value that does not decode is, an empty or null one is +// not put to valid, a key merged in through `<<` is read as one written there +// is, and a key the mapping does not present, absent or merged in from past +// nodeview.MergeDepthLimit, is left unread for the census. +func TestTakeString_AppliesValidAndReadsAsTheParserDoes(t *testing.T) { t.Parallel() only := func(v string) bool { return v == "ok" } for _, tc := range []struct { @@ -479,7 +480,8 @@ func TestTakeString_AppliesValidAndReadsOnlyDirectKeys(t *testing.T) { {"an empty string valid would refuse", `k: ""`, only, "", true, ""}, {"a null valid would refuse", "k: ~", only, "", true, ""}, {"absent", "j: x", nil, "", false, ""}, - {"merged in", "<<: {k: m}", nil, "", false, ""}, + {"merged in", "<<: {k: m}", nil, "m", true, ""}, + {"merged in past the bound", "<<: " + pastMergeBound("{k: m}"), nil, "", false, ""}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/compilers/openapi/internal/auth/flows.go b/compilers/openapi/internal/auth/flows.go index 700379dc4..fd0e6c570 100644 --- a/compilers/openapi/internal/auth/flows.go +++ b/compilers/openapi/internal/auth/flows.go @@ -10,6 +10,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -181,8 +182,9 @@ func applyFlowAnnotations(c lowering.Ctx, lowered []ir.OAuthFlow, flows *soa.OAu // without this they would be lost: the parser models every field on every flow, // and the unknown-key census never sees one. // -// It reads the flow's own mapping, as every preservation site does, so a field -// merged in through a `<<` key is not kept (GitHub #384). +// It reads the flow's mapping as the parser does, as every preservation site +// does, so a field merged in through a `<<` key is kept as a written one is +// (GitHub #384). func preserveUndefinedFlowFields(c lowering.Ctx, fl *ir.OAuthFlow, f sourceFlow, fptr jsontext.Pointer) []ir.Diagnostic { defined := flowFieldsDefinedBy(f.keyword) var diags []ir.Diagnostic @@ -192,7 +194,7 @@ func preserveUndefinedFlowFields(c lowering.Ctx, fl *ir.OAuthFlow, f sourceFlow, } at := fptr + ids.Ptr(field.name) kept, keptDiags := annotation.PreserveNodeInto(&fl.Unmodeled, "openapi:"+field.name, - annotation.RawChildNode(f.src.GetRootNode(), field.name), ir.ReasonDegradedLowering, c.ProvenanceAt(at)) + nodeview.Child(f.src.GetRootNode(), field.name), ir.ReasonDegradedLowering, c.ProvenanceAt(at)) diags = append(diags, keptDiags...) if !kept { continue diff --git a/compilers/openapi/internal/auth/flows_test.go b/compilers/openapi/internal/auth/flows_test.go index d13e27c35..15520ecb1 100644 --- a/compilers/openapi/internal/auth/flows_test.go +++ b/compilers/openapi/internal/auth/flows_test.go @@ -444,24 +444,20 @@ components: // TestLowerSecuritySchemes_AMergedOrAliasedFlowFieldFillsOnlyItsKindsSlot pins // the flow field spellings the raw mapping does not hold as written. A field // the kind does not define never fills a slot, whether merged in through `<<` -// or written as an alias, and the alias, which the mapping does hold, is kept -// as its value. A merged field the kind defines still lowers. Whether a merged -// stray field is kept is GitHub #384's, as for every preservation site, so it -// is not asserted here. +// or written as an alias, and either is kept as its value, read as the parser +// reads it (GitHub #384). A merged field the kind defines still lowers. func TestLowerSecuritySchemes_AMergedOrAliasedFlowFieldFillsOnlyItsKindsSlot(t *testing.T) { t.Parallel() cases := []struct { name, keyword, kind, body string wantAuth, wantTok string - // wantKept is the field expected under Unmodeled, or "" for none; merged - // says the stray field arrives through `<<`, which leaves that to #384. + // wantKept is the field expected under Unmodeled, or "" for none. wantKept string - merged bool }{ { name: "a merge key supplying a field the kind does not define", keyword: "implicit", kind: "implicit", body: `{<<: *base, authorizationUrl: "https://a.example/a", scopes: {}}`, - wantAuth: "https://a.example/a", merged: true, + wantKept: "tokenUrl", wantAuth: "https://a.example/a", }, { name: "an alias value", keyword: "implicit", kind: "implicit", @@ -495,9 +491,6 @@ components: assert.Equal(t, tc.kind, fl.Kind) assert.Equal(t, tc.wantAuth, fl.AuthorizationURL) assert.Equal(t, tc.wantTok, fl.TokenURL, "a field the kind does not define fills no slot") - if tc.merged { - return - } if tc.wantKept == "" { assert.Empty(t, fl.Unmodeled) assert.Empty(t, messagesAt(diags, diag.DegradedConstruct), "%+v", diags) diff --git a/compilers/openapi/internal/auth/schemes.go b/compilers/openapi/internal/auth/schemes.go index 720b360f2..977e8a60b 100644 --- a/compilers/openapi/internal/auth/schemes.go +++ b/compilers/openapi/internal/auth/schemes.go @@ -12,6 +12,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" "github.com/dexpace/morphic/ir" ) @@ -291,7 +292,7 @@ func preserveUnreadFields(c lowering.Ctx, scheme *ir.AuthScheme, ss *soa.Securit } at := decl + ids.Ptr(field) kept, keptDiags := annotation.PreserveNodeInto(&scheme.Unmodeled, "openapi:"+field, - annotation.RawChildNode(ss.GetRootNode(), field), ir.ReasonDegradedLowering, c.ProvenanceAt(at)) + nodeview.Child(ss.GetRootNode(), field), ir.ReasonDegradedLowering, c.ProvenanceAt(at)) diags = append(diags, keptDiags...) if !kept { continue diff --git a/compilers/openapi/internal/auth/schemes_test.go b/compilers/openapi/internal/auth/schemes_test.go index 1ee9284e3..7cca76885 100644 --- a/compilers/openapi/internal/auth/schemes_test.go +++ b/compilers/openapi/internal/auth/schemes_test.go @@ -474,7 +474,7 @@ info: {title: T, version: "1"} paths: {} components: securitySchemes: - s: {type: apiKey, in: header, name: X-Key, x-bad: {1: intkey}} + s: {type: apiKey, in: header, name: X-Key, x-bad: {n: .nan}} ` doc, _, diags := serviceSpec(t, spec) assert.Positive(t, openapitest.CountDiagsAt(diags, diag.DegradedConstruct, ir.SeverityWarning), diff --git a/compilers/openapi/internal/bind/export_test.go b/compilers/openapi/internal/bind/export_test.go index 7158b7b68..45a77fed1 100644 --- a/compilers/openapi/internal/bind/export_test.go +++ b/compilers/openapi/internal/bind/export_test.go @@ -10,3 +10,7 @@ func (t *Table) Find(name string) (DocID, bool) { } return 0, false } + +// MaxPointerTokens is maxPointerTokens, for the external tests that read a +// pointer to its cap. +const MaxPointerTokens = maxPointerTokens diff --git a/compilers/openapi/internal/bind/kinds.go b/compilers/openapi/internal/bind/kinds.go new file mode 100644 index 000000000..828be2e26 --- /dev/null +++ b/compilers/openapi/internal/bind/kinds.go @@ -0,0 +1,107 @@ +package bind + +import ( + "encoding/json/jsontext" + "strings" +) + +// Kind is what the OpenAPI grammar reads a position as, in the terms the +// nodes built beneath it are named in (Table.Kinds). An object or a list the +// grammar models and naming does not tell apart is KindOther. +type Kind uint8 + +const ( + // KindRaw is a position the model holds as raw YAML or as a value. + KindRaw Kind = iota + // KindOther is any other object or list the model holds. + KindOther + // KindComponents is the Components Object. + KindComponents + // KindSchemaMap is a map of schemas: a component section, or a schema's + // properties, $defs or the like. + KindSchemaMap + // KindSchema is a Schema Object. + KindSchema + // KindOperation is an Operation Object. + KindOperation + // KindParameter is a Parameter Object. + KindParameter + // KindHeader is a Header Object. + KindHeader + // KindRequestBody is a Request Body Object. + KindRequestBody + // KindResponse is a Response Object. + KindResponse + // KindMediaType is a Media Type Object. + KindMediaType +) + +// kindOf is the Kind of a position read at s. +func kindOf(s state) Kind { + switch s { + case sRaw, sValue: + return KindRaw + case sComponents: + return KindComponents + case sSchemaMap: + return KindSchemaMap + case sSchema: + return KindSchema + case sOperation: + return KindOperation + case sParameter: + return KindParameter + case sHeader: + return KindHeader + case sRequestBody: + return KindRequestBody + case sResponse: + return KindResponse + case sMediaType: + return KindMediaType + default: + return KindOther + } +} + +// Kinds returns how the grammar reads each position on the way to p from its +// document's root, the root's first: the source's own grammar from its root, +// and, at a position it reads as raw YAML, or in another document, the kind a +// reference read the position at, where one did (GitHub #981). A position +// written in the source is what its own position makes it, whatever a +// reference expects it to be; where references read one position as two +// kinds, the lesser counts, so the answer depends on no reading order. +func (t *Table) Kinds(p Position) []Kind { + s := sRaw + if p.Doc == Source { + s = sDocument + } + s = t.rooted(Position{Doc: p.Doc}, s) + out := []Kind{kindOf(s)} + pointer := string(p.Pointer) + for i := 0; i < len(pointer) && len(out) <= maxPointerTokens; { + end := strings.IndexByte(pointer[i+1:], '/') + if end < 0 { + end = len(pointer) + } else { + end += i + 1 + } + token := jsontext.Pointer(pointer[i:end]).LastToken() + s = t.rooted(Position{Doc: p.Doc, Pointer: jsontext.Pointer(pointer[:end])}, step(s, token)) + out = append(out, kindOf(s)) + i = end + } + return out +} + +// rooted returns s, the state the grammar reads at, or, where that is raw +// YAML, the state a reference read at as a root of its own, where one did. +func (t *Table) rooted(at Position, s state) state { + if s != sRaw { + return s + } + if r, ok := t.roots[at]; ok { + return r + } + return s +} diff --git a/compilers/openapi/internal/bind/kinds_test.go b/compilers/openapi/internal/bind/kinds_test.go new file mode 100644 index 000000000..d502a0de4 --- /dev/null +++ b/compilers/openapi/internal/bind/kinds_test.go @@ -0,0 +1,155 @@ +package bind_test + +import ( + "encoding/json/jsontext" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dexpace/morphic/compilers/openapi/internal/bind" +) + +// The kinds the rows below spell, by their initials. +const ( + kRaw = bind.KindRaw + kOth = bind.KindOther + kCom = bind.KindComponents + kMap = bind.KindSchemaMap + kSch = bind.KindSchema + kOp = bind.KindOperation + kPar = bind.KindParameter + kHdr = bind.KindHeader + kBod = bind.KindRequestBody + kRes = bind.KindResponse + kMT = bind.KindMediaType +) + +// TestKinds_ReadsEachPositionByTheGrammar pins the kind the source's own +// grammar reads at each position on the way to a pointer, the root's first, +// whether or not the document writes anything there. +func TestKinds_ReadsEachPositionByTheGrammar(t *testing.T) { + t.Parallel() + table := bound(t, head+"components: {schemas: {A: {type: object}}}\n", self) + tests := []struct { + pointer jsontext.Pointer + want []bind.Kind + }{ + {"", []bind.Kind{kOth}}, + {"/components/schemas/A/properties/x", []bind.Kind{kOth, kCom, kMap, kSch, kMap, kSch}}, + {"/components/schemas/A/oneOf/0/items", []bind.Kind{kOth, kCom, kMap, kSch, kOth, kSch, kSch}}, + {"/components/schemas/A/enum/0", []bind.Kind{kOth, kCom, kMap, kSch, kOth, kRaw}}, + {"/paths/~1a/post/requestBody/content/application~1json/schema", + []bind.Kind{kOth, kOth, kOth, kOp, kBod, kOth, kMT, kSch}}, + {"/paths/~1a/get/parameters/0/schema", []bind.Kind{kOth, kOth, kOth, kOp, kOth, kPar, kSch}}, + {"/paths/~1a/get/responses/200/headers/X/schema", []bind.Kind{kOth, kOth, kOth, kOp, kOth, kRes, kOth, kHdr, kSch}}, + {"/info/title", []bind.Kind{kOth, kOth, kRaw}}, + {"/x-ext/a", []bind.Kind{kOth, kRaw, kRaw}}, + } + for _, tc := range tests { + t.Run(string(tc.pointer), func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, table.Kinds(at(tc.pointer))) + }) + } +} + +// TestKinds_ReadsWhatAReferenceReads pins a position the grammar reads as raw +// YAML, or one in another document: it is read as the reference that lands on +// it reads it, a schema an extension holds or a parameter another document +// holds, and another document's root as a schema where a schema's $ref names +// the whole document. A position no reference reads stays raw. +func TestKinds_ReadsWhatAReferenceReads(t *testing.T) { + t.Parallel() + cfg := withDocuments(map[string]string{ + "/params.yaml": "Size: {name: size, in: query, schema: {type: object}}\n", + "/pet.yaml": "type: object\n", + }) + cfg.Self, cfg.Locate = resolving.Self, resolving.Locate + // A reference object is read where the parser resolved it, as the load + // phase hands the binder. + cfg.Through = func(name string, p jsontext.Pointer) (bind.ReferenceEnd, bool) { + if name != "/spec.yaml" || p != "/components/parameters/P" { + return bind.ReferenceEnd{}, false + } + return bind.ReferenceEnd{Name: "/params.yaml", Pointer: "/Size"}, true + } + table := bound(t, head+`x-lib: {Thing: {type: object}, Other: {type: object}} +components: + schemas: + A: {$ref: '#/x-lib/Thing'} + B: {$ref: 'pet.yaml'} + parameters: + P: {$ref: 'params.yaml#/Size'} +`, cfg) + params, ok := table.Find("/params.yaml") + require.True(t, ok) + pet, ok := table.Find("/pet.yaml") + require.True(t, ok) + + assert.Equal(t, []bind.Kind{kOth, kRaw, kSch, kMap, kSch}, table.Kinds(at("/x-lib/Thing/properties/p"))) + assert.Equal(t, []bind.Kind{kOth, kRaw, kRaw}, table.Kinds(at("/x-lib/Other")), "no reference reads it") + assert.Equal(t, []bind.Kind{kRaw, kPar, kSch}, table.Kinds(bind.Position{Doc: params, Pointer: "/Size/schema"})) + assert.Equal(t, []bind.Kind{kSch, kMap, kSch}, table.Kinds(bind.Position{Doc: pet, Pointer: "/properties/p"})) +} + +// TestKinds_KeepsWhatTheSourceWrites pins that a position the source's own +// grammar reads is that, whatever a reference landing on it expects: a +// parameter's $ref to a schema, which the binder reads as a parameter, does +// not make the schema a parameter. +func TestKinds_KeepsWhatTheSourceWrites(t *testing.T) { + t.Parallel() + cfg := self + cfg.Through = func(name string, p jsontext.Pointer) (bind.ReferenceEnd, bool) { + if p != "/components/parameters/P" { + return bind.ReferenceEnd{}, false + } + return bind.ReferenceEnd{Name: name, Pointer: "/components/schemas/S"}, true + } + table := bound(t, head+`components: + schemas: + S: {name: s, in: query} + parameters: + P: {$ref: '#/components/schemas/S'} +`, cfg) + assert.Equal(t, []bind.Kind{kOth, kCom, kMap, kSch}, table.Kinds(at("/components/schemas/S"))) +} + +// TestKinds_ReadsTheLesserOfTwoReadings pins a raw position two references +// read as two kinds: the lesser counts, so the answer depends neither on which +// reference the document declares first nor on which the binder reaches +// first. In the last row the schema reading arrives a round after the +// parameter's, through a property of U, which only a walk of U finds. +func TestKinds_ReadsTheLesserOfTwoReadings(t *testing.T) { + t.Parallel() + cfg := self + cfg.Through = func(name string, p jsontext.Pointer) (bind.ReferenceEnd, bool) { + if p != "/components/parameters/P" { + return bind.ReferenceEnd{}, false + } + return bind.ReferenceEnd{Name: name, Pointer: "/x-lib/T"}, true + } + lib := "x-lib: {T: {name: t, in: query}, U: {properties: {t: {$ref: '#/x-lib/T'}}}}\ncomponents:\n" + schema := " schemas:\n A: {$ref: '#/x-lib/T'}\n" + later := " schemas:\n A: {$ref: '#/x-lib/U'}\n" + parameter := " parameters:\n P: {$ref: '#/x-lib/T'}\n" + for name, src := range map[string]string{ + "the schema first": lib + schema + parameter, + "the parameter first": lib + parameter + schema, + "the schema reached a round later": lib + later + parameter, + } { + assert.Equal(t, []bind.Kind{kOth, kRaw, kSch}, bound(t, head+src, cfg).Kinds(at("/x-lib/T")), name) + } +} + +// TestKinds_StopsAtTheTokenCap pins that Kinds reads a pointer through as +// many tokens as the table reads any pointer through, and no further. +func TestKinds_StopsAtTheTokenCap(t *testing.T) { + t.Parallel() + table := bound(t, head, self) + within := table.Kinds(at(jsontext.Pointer(strings.Repeat("/x-p", bind.MaxPointerTokens)))) + past := table.Kinds(at(jsontext.Pointer(strings.Repeat("/x-p", bind.MaxPointerTokens+1)))) + assert.Len(t, within, bind.MaxPointerTokens+1, "every token is read") + assert.Len(t, past, bind.MaxPointerTokens+1, "the one past the cap is not") +} diff --git a/compilers/openapi/internal/bind/table.go b/compilers/openapi/internal/bind/table.go index 72ad9182c..bbacdc570 100644 --- a/compilers/openapi/internal/bind/table.go +++ b/compilers/openapi/internal/bind/table.go @@ -177,7 +177,7 @@ func (b *binder) table(t *Table) *Table { } } t.loops, t.steps, t.limit, t.exhausted = b.loops, b.work.spent, b.work.limit, b.work.over() - t.ends = b.find.ends + t.ends, t.roots = b.find.ends, b.find.roots for _, d := range b.docs.list { t.docs = append(t.docs, Document{Name: d.name, Root: d.root}) t.truncated = t.truncated || d.view.Exhausted() @@ -246,6 +246,7 @@ type Document struct { // once and never written after. type Table struct { entries map[Position]Entry + roots map[Position]state mappings map[mappingKey]Mapping loops []Loop landings []Landing diff --git a/compilers/openapi/internal/bind/walk.go b/compilers/openapi/internal/bind/walk.go index abdf7509a..78fb3b160 100644 --- a/compilers/openapi/internal/bind/walk.go +++ b/compilers/openapi/internal/bind/walk.go @@ -47,13 +47,14 @@ type finder struct { landings map[Position]bool pending []root covered map[root]bool + roots map[Position]state } // newFinder returns a finder over docs that has found nothing. func newFinder(d *docs, work *budget) *finder { return &finder{docs: d, ids: newIdentifiers(resources{d}.document(Source)), work: work, ends: map[Position]Position{}, sites: map[Position]bool{}, mappingSites: map[mappingKey]bool{}, landings: map[Position]bool{}, - covered: map[root]bool{}} + covered: map[root]bool{}, roots: map[Position]state{}} } // walk reads every position below r by the grammar, noting each schema position @@ -64,6 +65,9 @@ func newFinder(d *docs, work *budget) *finder { // a later root inside one already read is not read again; the source's own walk // from its root, which reads identifiers, notes none. func (w *finder) walk(r root, mark bool) { + if prev, seen := w.roots[r.at]; !seen || r.state < prev { + w.roots[r.at] = r.state + } stack := []frame{{node: w.docs.node(r.at), at: r.at, state: r.state}} for len(stack) > 0 && w.work.spend(1) { f := stack[len(stack)-1] diff --git a/compilers/openapi/internal/canary/behaviours_test.go b/compilers/openapi/internal/canary/behaviours_test.go index 128845200..2f74c5edf 100644 --- a/compilers/openapi/internal/canary/behaviours_test.go +++ b/compilers/openapi/internal/canary/behaviours_test.go @@ -28,6 +28,8 @@ func canaries() map[string]func(t *testing.T) { "anchored-entry-dropped": anchoredEntryDropped, "embedded-map-keys": embeddedMapKeys, "duplicate-key": duplicateKey, + "merge-keys": mergeKeys, + "non-scalar-key": nonScalarKey, "unmodeled-keywords": unmodeledKeywords, "null-schema-is-false": nullSchemaIsFalse, "overlay-alias-clone": overlayAliasClone, diff --git a/compilers/openapi/internal/canary/decoding_test.go b/compilers/openapi/internal/canary/decoding_test.go index 6ffd435b2..a5bf027e9 100644 --- a/compilers/openapi/internal/canary/decoding_test.go +++ b/compilers/openapi/internal/canary/decoding_test.go @@ -9,6 +9,7 @@ import ( "runtime" "runtime/debug" "runtime/metrics" + "slices" "strconv" "strings" "testing" @@ -554,3 +555,49 @@ func nullSchemaIsFalse(t *testing.T) { assert.Equal(t, wantTag, js.GetCore().GetRootNode().ShortTag(), revisit(t, result{report: name})) } } + +// mergeKeys pins merge-keys: a mapping's merged pairs come first, an earlier +// source winning and none overriding the mapping's own key; a source writing a +// key twice gives its first copy; and a `<<` naming no mapping merges nothing, +// with no finding. +func mergeKeys(t *testing.T) { + doc, found, err := soa.Unmarshal(t.Context(), strings.NewReader(head+ + "x-base: &base {k: {type: string}, z: {type: string}}\n"+ + "x-other: &other {z: {type: integer}, y: {type: string}}\n"+ + "components:\n schemas:\n"+ + " S: {properties: {a: {type: integer}, <<: [*base, *other], k: {type: boolean}}}\n"+ + " T: {properties: {<<: 5, q: {type: string}}}\n"+ + " D: {properties: {<<: {d: {type: string}, d: {type: integer}}}}\n")) + require.NoError(t, err) + assert.Empty(t, found, revisit(t, result{})) + + keys := func(name string) []string { + s, ok := doc.Components.Schemas.Get(name) + require.True(t, ok, name) + return slices.Collect(s.GetSchema().GetProperties().Keys()) + } + assert.Equal(t, []string{"z", "y", "a", "k"}, keys("S"), "merged pairs first: %s", revisit(t, result{})) + s, _ := doc.Components.Schemas.Get("S") + z, _ := s.GetSchema().GetProperties().Get("z") + assert.Equal(t, []oas3.SchemaType{oas3.SchemaTypeString}, z.GetSchema().GetType(), + "the earlier source wins: %s", revisit(t, result{})) + k, _ := s.GetSchema().GetProperties().Get("k") + assert.Equal(t, []oas3.SchemaType{oas3.SchemaTypeBoolean}, k.GetSchema().GetType(), + "the mapping's own key wins: %s", revisit(t, result{})) + assert.Equal(t, []string{"q"}, keys("T"), "a `<<` naming a scalar merges nothing: %s", revisit(t, result{})) + d, _ := doc.Components.Schemas.Get("D") + dd, _ := d.GetSchema().GetProperties().Get("d") + assert.Equal(t, []oas3.SchemaType{oas3.SchemaTypeString}, dd.GetSchema().GetType(), + "a source's repeated key gives its first copy: %s", revisit(t, result{})) +} + +// nonScalarKey pins non-scalar-key: a mapping key that is no scalar is read +// as the empty string, with no finding. +func nonScalarKey(t *testing.T) { + doc, found, err := soa.Unmarshal(t.Context(), strings.NewReader(head+ + "components:\n responses:\n ? {k: v}\n : {description: x}\n Plain: {description: y}\n")) + require.NoError(t, err) + assert.Empty(t, found, revisit(t, result{})) + assert.Equal(t, []string{"", "Plain"}, slices.Collect(doc.Components.Responses.Keys()), + "the mapping key reads as the empty string: %s", revisit(t, result{})) +} diff --git a/compilers/openapi/internal/canary/testdata/validation_disagreements.txt b/compilers/openapi/internal/canary/testdata/validation_disagreements.txt index a637a3427..a0cd1fc0d 100644 --- a/compilers/openapi/internal/canary/testdata/validation_disagreements.txt +++ b/compilers/openapi/internal/canary/testdata/validation_disagreements.txt @@ -255,6 +255,7 @@ testdata/conformance/openapi/schema-ref-siblings-30.yaml 17:84 later-version-con testdata/conformance/openapi/security-schemes.yaml 46:21 validation-allowed-values library over-strict testdata/conformance/openapi/xml-nodetype-32.yaml 7:7 validation-invalid-schema library old-grammar testdata/conformance/openapi/xml-nodetype-32.yaml 7:7 validation-invalid-schema library old-grammar +testdata/conformance/openapi/yaml-merge-keys.yaml 41:30 validation-allowed-values library over-strict testdata/dangling/openapi/f07-discriminator.yaml 13:10 validation-invalid-syntax compiler component-names testdata/dangling/openapi/f31-discriminator-empty-name.yaml 17:7 validation-invalid-syntax compiler component-names testdata/dangling/openapi/f32-ref-noncanonical-escape.yaml 7:7 validation-invalid-syntax compiler component-names diff --git a/compilers/openapi/internal/load/budgets.go b/compilers/openapi/internal/load/budgets.go index 10c36c033..4204d1ba1 100644 --- a/compilers/openapi/internal/load/budgets.go +++ b/compilers/openapi/internal/load/budgets.go @@ -4,6 +4,8 @@ import ( yaml "gopkg.in/yaml.v3" "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" + "github.com/dexpace/morphic/compilers/openapi/internal/sourceindex" "github.com/dexpace/morphic/ir" ) @@ -36,25 +38,20 @@ func OverByteBudget(prov ir.Provenance, data []byte, limit int) (ir.Diagnostic, "source document is %d bytes, past the %d-byte budget", len(data), limit), true } +// maxTreeNodes bounds every walk of a tree the load phase holds. The pre-parse +// refusals refuse a tree past sourceindex.MaxIndexedNodes before any of these +// walks runs over it, so the bound, one more for the document node, cuts no +// tree they are handed. +const maxTreeNodes = sourceindex.MaxIndexedNodes + 1 + // nodeCount returns the number of nodes in the parsed tree rooted at root. An -// alias node has empty Content in yaml.v3 and its edge is never followed, so -// the iterative walk visits each node once. +// alias is counted as itself and its edge is never followed, so each node is +// counted once. // // sourceindex.Index.Nodes, which scan's alias-amplification bound reads, takes // the same measure of the document's content, leaving the document node out. // The node budget does not read it, since refusals builds that index and keeps // none of it. func nodeCount(root *yaml.Node) int { - if root == nil { - return 0 - } - count := 0 - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - count++ - stack = append(stack, n.Content...) - } - return count + return nodeview.Walk{Nodes: maxTreeNodes}.Run(root).Visited } diff --git a/compilers/openapi/internal/load/escapes.go b/compilers/openapi/internal/load/escapes.go index 525760b87..d1c4c9416 100644 --- a/compilers/openapi/internal/load/escapes.go +++ b/compilers/openapi/internal/load/escapes.go @@ -195,35 +195,29 @@ func (e *escapes) rewritable(root *yaml.Node) bool { // returns false. A value that is no scalar names no reference. The tree is // walked by its content, never through an alias, so each mapping is read once. func eachRefValue(root *yaml.Node, visit func(*yaml.Node) bool) { - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - stack = append(stack, n.Content...) + nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { for i, c := range n.Content { v := nodeview.Deref(c) if refValue(n, i) && v != nil && v.Kind == yaml.ScalarNode && !visit(c) { - return + return nodeview.Stop } } - } + return nodeview.Descend + }}.Run(root) } // aliasesOutsideRefs returns every alias in the tree under root that is not // the value of a $ref key, walked as eachRefValue walks. func aliasesOutsideRefs(root *yaml.Node) []*yaml.Node { var out []*yaml.Node - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - stack = append(stack, n.Content...) + nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { for i, c := range n.Content { if c.Kind == yaml.AliasNode && !refValue(n, i) { out = append(out, c) } } - } + return nodeview.Descend + }}.Run(root) return out } diff --git a/compilers/openapi/internal/load/external.go b/compilers/openapi/internal/load/external.go index 43d8933a6..dcf3789e6 100644 --- a/compilers/openapi/internal/load/external.go +++ b/compilers/openapi/internal/load/external.go @@ -245,17 +245,26 @@ func (e external) holdObject(key string, o heldObject) { // of what it names (see Open). It visits each node of tree once, following an // alias only to read a $ref's key and value, as the model does. func (e external) holdSpellings(tree *yaml.Node, base string) { - stack := []*yaml.Node{tree} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - stack = append(stack, n.Content...) - for i := 0; n.Kind == yaml.MappingNode && i+1 < len(n.Content); i += 2 { - if key, value := nodeview.Deref(n.Content[i]), nodeview.Deref(n.Content[i+1]); key.Value == "$ref" && value.Kind == yaml.ScalarNode { - e.holdSpelling(references.Reference(value.Value), base) - } + nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { + for _, ref := range writtenAt(n, "$ref") { + e.holdSpelling(references.Reference(ref.Value), base) + } + return nodeview.Descend + }}.Run(tree) +} + +// writtenAt returns the scalar values the mapping n writes under key itself, +// each alias followed: the pairs written in place, every copy, since a walk +// visiting every node meets whatever a `<<` merges in where it is written. +func writtenAt(n *yaml.Node, key string) []*yaml.Node { + var out []*yaml.Node + for i := 0; n.Kind == yaml.MappingNode && i+1 < len(n.Content); i += 2 { + k, own := nodeview.KeyOf(n.Content[i]) + if v := nodeview.Deref(n.Content[i+1]); own && k == key && v != nil && v.Kind == yaml.ScalarNode { + out = append(out, v) } } + return out } // holdSpelling holds the source under the key ref, read against base, names a @@ -530,23 +539,32 @@ func reachesLocal(root *yaml.Node, base string) (reason string, refuse bool) { if !overNetwork(base) { return "", false } - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - stack = append(stack, n.Content...) - for i := 0; n.Kind == yaml.MappingNode && i+1 < len(n.Content); i += 2 { - key, value := nodeview.Deref(n.Content[i]), nodeview.Deref(n.Content[i+1]) - if value.Kind != yaml.ScalarNode || (key.Value != "$ref" && key.Value != "$id") { - continue - } - t, local := localTarget(value.Value, base) - if local && key.Value == "$ref" { - return fmt.Sprintf("its $ref %q reaches the local file %s", value.Value, t), true - } - if local { - return fmt.Sprintf("its $id %q rebases references onto the local %s", value.Value, t), true - } + nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { + reason, refuse = localReach(n, base) + if refuse { + return nodeview.Stop + } + return nodeview.Descend + }}.Run(root) + return reason, refuse +} + +// localReach returns why the mapping n, in a document read from base, takes +// the resolver to the local file system, and whether it does: through the +// first $ref or $id it writes itself that names a local location. +func localReach(n *yaml.Node, base string) (reason string, refuse bool) { + for i := 0; n.Kind == yaml.MappingNode && i+1 < len(n.Content); i += 2 { + key, own := nodeview.KeyOf(n.Content[i]) + value := nodeview.Deref(n.Content[i+1]) + if !own || value == nil || value.Kind != yaml.ScalarNode || (key != "$ref" && key != "$id") { + continue + } + t, local := localTarget(value.Value, base) + if local && key == "$ref" { + return fmt.Sprintf("its $ref %q reaches the local file %s", value.Value, t), true + } + if local { + return fmt.Sprintf("its $id %q rebases references onto the local %s", value.Value, t), true } } return "", false diff --git a/compilers/openapi/internal/load/load.go b/compilers/openapi/internal/load/load.go index a718ca758..f23520fb5 100644 --- a/compilers/openapi/internal/load/load.go +++ b/compilers/openapi/internal/load/load.go @@ -14,6 +14,7 @@ import ( "github.com/dexpace/morphic/compilers" "github.com/dexpace/morphic/compilers/openapi/internal/bind/typed" "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/oas" "github.com/dexpace/morphic/compilers/openapi/internal/overlay" "github.com/dexpace/morphic/compilers/openapi/internal/scan" @@ -85,14 +86,11 @@ type Options struct { // too (GitHub #501), and replaces the resolver's own parse on a second // resolution (GitHub #538). func releaseAnchors(root *yaml.Node) { - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] + // An alias is a leaf of the walk: a recursive anchor cannot loop it. + nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { n.Anchor = "" - // Follows Content, never an alias: a recursive anchor cannot loop it. - stack = append(stack, n.Content...) - } + return nodeview.Descend + }}.Run(root) } // ErrParse marks a hard failure to read a source document: bytes that are not diff --git a/compilers/openapi/internal/load/reached.go b/compilers/openapi/internal/load/reached.go index 763b932f0..074556449 100644 --- a/compilers/openapi/internal/load/reached.go +++ b/compilers/openapi/internal/load/reached.go @@ -5,6 +5,7 @@ import ( "encoding/json/jsontext" "iter" "maps" + "math" "slices" soa "github.com/speakeasy-api/openapi/openapi" @@ -258,8 +259,8 @@ func (v *reached) check(site ir.Provenance, t reachedTarget) []ir.Diagnostic { if d, over := v.charge(site, t.doc, t.node); over { return d } - found := v.checker.Check(t.node, v.version, oas.Options{Start: t.kind, Budget: v.limit, Locate: v.locatorFor(t), - ReadAt: true, Root: t.home.pointer}) + opts := oas.Options{Start: t.kind, Budget: v.limit, Locate: v.locatorFor(t), ReadAt: true, Root: t.home.pointer} + found := append(v.checker.Check(t.node, v.version, opts), oas.CheckFormat(t.node, v.version, formatOptions(opts))...) var diags []ir.Diagnostic for _, d := range found { if key := identity(d, t.doc); !v.reported[key] { @@ -330,29 +331,17 @@ func reachedNode(r resolvable) (*yaml.Node, bool) { } // spanned counts the nodes under root as a model built from it holds them, -// each alias counted as a copy of what it names, and reports whether the count -// stays within limit, negative meaning none. It stops one node past limit, so -// refusing an object costs the budget, not the object. No alias leads back up: -// a document whose anchor names one of its own ancestors was refused. -// -// root and every node under it are real: yaml.v3 leaves no nil in Content, and -// a well-formed alias's target is never nil, or the cycle refusals ahead -// would have refused the document. +// each alias standing for a copy of what it names (nodeview.AliasTarget), as +// scan's alias weigher weighs it, and reports whether the count stays within +// limit, negative meaning none. It stops one node past limit, so refusing an +// object costs the budget, not the object. No alias leads back up: a document +// whose anchor names one of its own ancestors was refused. A limit of +// math.MaxInt has no node past it to stop at. func spanned(root *yaml.Node, limit int) (int, bool) { - count := 0 - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - count++ - if limit >= 0 && count > limit { - return count, false - } - if n.Kind == yaml.AliasNode { - stack = append(stack, n.Alias) - continue - } - stack = append(stack, n.Content...) + bound := math.MaxInt + if limit >= 0 && limit < math.MaxInt { + bound = limit + 1 } - return count, true + visited := nodeview.Walk{Nodes: bound, Aliases: nodeview.AliasTarget}.Run(root).Visited + return visited, limit < 0 || visited <= limit } diff --git a/compilers/openapi/internal/load/reached_internal_test.go b/compilers/openapi/internal/load/reached_internal_test.go index c2934dcf2..12df4e1dd 100644 --- a/compilers/openapi/internal/load/reached_internal_test.go +++ b/compilers/openapi/internal/load/reached_internal_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/jsontext" "errors" "fmt" + "math" "net/http" "net/http/httptest" "os" @@ -53,10 +54,9 @@ func loadExternal(t *testing.T, dir, root string, opts Options) (*Document, []ir } // TestSpanned covers spanned's properties: it counts every node under root, -// an alias counts as a copy of what it names, a tree of exactly limit nodes -// fits, and a limit stops the count one node past it, never at the whole tree. -// It has no case for a nil node: see spanned's own doc comment for why yaml.v3 -// never gives it one to skip. +// an alias counts as the copy of what it names and nothing more, a tree of +// exactly limit nodes fits, a limit stops the count one node past it, never at +// the whole tree, and math.MaxInt, past which no node lies, counts it all. func TestSpanned(t *testing.T) { t.Parallel() wide := &yaml.Node{Kind: yaml.MappingNode} @@ -72,10 +72,11 @@ func TestSpanned(t *testing.T) { count int fits bool }{ - "every node under root": {wide, -1, 1001, true}, - "an alias counts its target as a copy": {aliased, -1, 3, true}, - "a tree of exactly limit nodes fits": {wide, 1001, 1001, true}, - "a limit stops the count one past it": {wide, 6, 7, false}, + "every node under root": {wide, -1, 1001, true}, + "an alias counts as its target's copy": {aliased, -1, 2, true}, + "a tree of exactly limit nodes fits": {wide, 1001, 1001, true}, + "a limit stops the count one past it": {wide, 6, 7, false}, + "the largest limit counts the whole tree": {wide, math.MaxInt, 1001, true}, } { count, fits := spanned(c.root, c.limit) assert.Equal(t, c.count, count, name) diff --git a/compilers/openapi/internal/load/siblings.go b/compilers/openapi/internal/load/siblings.go index 59b57648e..231375531 100644 --- a/compilers/openapi/internal/load/siblings.go +++ b/compilers/openapi/internal/load/siblings.go @@ -284,7 +284,7 @@ func (s *Siblings) add(ctx context.Context, site jsontext.Pointer, rp *soa.Refer // writtenKeys returns the keys the mapping fields, a use site less its $ref, is // written under, read as the parser reads it. func writtenKeys(fields *yaml.Node) map[string]bool { - pairs := nodeview.New().MappingPairs(fields) + pairs := nodeview.Pairs(fields) written := make(map[string]bool, len(pairs)) for _, p := range pairs { written[p.Key] = true @@ -331,10 +331,15 @@ func writesBeside(node *yaml.Node) bool { } // isRefKey reports whether key spells $ref, directly or through an alias, as -// the library reads a mapping's key. +// the library reads a mapping's key (nodeview.KeyOf). +// +// It and the readers above take the pairs a mapping writes itself, not those +// a `<<` merges in: the library tells a reference by its own pairs, so a $ref +// merged in makes no reference, while a `<<` beside an own $ref is a field +// the decoder of the copy merges (TestWithoutRef_DropsTheRefAsTheLibraryReadsIt). func isRefKey(key *yaml.Node) bool { - k := nodeview.Deref(key) - return k != nil && k.Value == refKey + k, own := nodeview.KeyOf(key) + return own && k == refKey } // buildSibling builds the fields beside a $ref into a Path Item, as the source's diff --git a/compilers/openapi/internal/load/tree.go b/compilers/openapi/internal/load/tree.go index 9d341d9b1..7e6c6ef35 100644 --- a/compilers/openapi/internal/load/tree.go +++ b/compilers/openapi/internal/load/tree.go @@ -4,7 +4,7 @@ import ( oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" yaml "gopkg.in/yaml.v3" - "github.com/dexpace/morphic/compilers/openapi/internal/sourceindex" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" ) // schemaRef is a schema of the parser's model, as each position holds one. @@ -14,21 +14,14 @@ type schemaRef = oas3.JSONSchema[oas3.Referenceable] // one. type oasDiscriminator = oas3.Discriminator -// anyNode reports whether any node in the tree under root satisfies match. It -// walks each node once, never through an alias, so its cost is the node count -// the source budget already bounds; the loader refuses a tree past the bound. +// anyNode reports whether any node but an alias in the tree under root +// satisfies match. It walks each node once, never through an alias, so its +// cost is the node count the source budget already bounds. func anyNode(root *yaml.Node, match func(*yaml.Node) bool) bool { - stack := []*yaml.Node{root} - for visited := 0; len(stack) > 0 && visited < sourceindex.MaxIndexedNodes; visited++ { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if n == nil || n.Kind == yaml.AliasNode { - continue + return nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { + if n.Kind != yaml.AliasNode && match(n) { + return nodeview.Stop } - if match(n) { - return true - } - stack = append(stack, n.Content...) - } - return false + return nodeview.Descend + }}.Run(root).Stopped } diff --git a/compilers/openapi/internal/load/treereads.go b/compilers/openapi/internal/load/treereads.go index a66aaaf48..4380e9178 100644 --- a/compilers/openapi/internal/load/treereads.go +++ b/compilers/openapi/internal/load/treereads.go @@ -96,6 +96,13 @@ func (t *treeReads) drain() int { } // index returns the index of mapping n, building it the first time. +// +// It reads the keys as GetTarget does, not as nodeview reads them for the +// model: the first copy of a key, a key that is no scalar matching no token, +// and any scalar spelled `<<` taken for a merge whatever its tag, a quoted one +// included. Pricing the library's read is what this type is for, so it mirrors +// that reader's rules rather than the parser's +// (TestTreeReads_CountsWhatTheLibraryReadTakes). func (t *treeReads) index(n *yaml.Node) *mappingKeys { if k, ok := t.keys[n]; ok { return k diff --git a/compilers/openapi/internal/load/unprepared.go b/compilers/openapi/internal/load/unprepared.go index 511727649..85a523886 100644 --- a/compilers/openapi/internal/load/unprepared.go +++ b/compilers/openapi/internal/load/unprepared.go @@ -18,6 +18,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/fragment" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -439,18 +440,16 @@ func anyParsed(doc *soa.OpenAPI, missed []usedDocument, holds func(*yaml.Node) b } // hasAnchor reports whether any node of the tree under root carries an anchor. -// It follows Content and never an alias, so it visits each node once. +// It follows Content and never an alias, so it visits each node once. The +// resolver parsed the tree from bytes the reader admitted (external.prepare), +// so the walk's bound cuts none. func hasAnchor(root *yaml.Node) bool { - stack := []*yaml.Node{root} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] + return nodeview.Walk{Nodes: maxTreeNodes, Visit: func(n *yaml.Node, _ int) nodeview.Step { if n.Anchor != "" { - return true + return nodeview.Stop } - stack = append(stack, n.Content...) - } - return false + return nodeview.Descend + }}.Run(root).Stopped } // stillUnprepared reports each document the second resolution found diff --git a/compilers/openapi/internal/load/validate.go b/compilers/openapi/internal/load/validate.go index 0515f8eec..09fcaf7cc 100644 --- a/compilers/openapi/internal/load/validate.go +++ b/compilers/openapi/internal/load/validate.go @@ -30,7 +30,7 @@ type judgment struct { // "openapi", or "" when it writes none as a scalar. It reads the tree rather // than the model, which is built from a respelled copy. func declaredVersion(root *yaml.Node) string { - for _, p := range nodeview.New().MappingPairs(nodeview.DocumentRoot(root)) { + for _, p := range nodeview.Pairs(nodeview.DocumentRoot(root)) { if v := nodeview.Deref(p.Val); p.Key == "openapi" && v != nil && v.Kind == yaml.ScalarNode && v.Tag != "!!null" { return v.Value } @@ -45,17 +45,28 @@ func declaredVersion(root *yaml.Node) string { // check reads (see nodesAdmitted). func judgeSource(root *yaml.Node, tree sourceTree, budget int) judgment { j := judgment{seen: map[objectKey]bool{}} - j.diags = oas.NewChecker().Check(root, declaredVersion(root), oas.Options{ + version := declaredVersion(root) + opts := oas.Options{ Budget: budget, Locate: tree.locatorIn(root), Seen: func(kind oas.Kind, node *yaml.Node) { j.seen[objectKey{node: nodeview.Deref(node), kind: kind}] = true }, ReadAt: true, - }) + } + j.diags = append(oas.NewChecker().Check(root, version, opts), oas.CheckFormat(root, version, formatOptions(opts))...) return j } +// formatOptions returns opts for oas.CheckFormat, which reads each node of a +// tree once: bounded by the size of a tree the load phase holds rather than +// by the work budget a check of its objects spends, so a tree the node budget +// admitted is read whole. +func formatOptions(opts oas.Options) oas.Options { + opts.Budget, opts.Seen = maxTreeNodes, nil + return opts +} + // nodesAdmitted is how many nodes a document opts admits may hold with its // aliases expanded, the node budget plus the alias budget, or 0 when either is // unbounded. A check of the table reads no more objects than that, so a diff --git a/compilers/openapi/internal/lowering/documents_test.go b/compilers/openapi/internal/lowering/documents_test.go index 63b625dfc..b34033ca4 100644 --- a/compilers/openapi/internal/lowering/documents_test.go +++ b/compilers/openapi/internal/lowering/documents_test.go @@ -121,54 +121,33 @@ func TestIDsAt_NamesEachSourceEntrysDocument(t *testing.T) { assert.Nil(t, lowering.OtherIDs(lowering.Ctx{})) } -// TestDeclareHint_KeepsOnePointerInTwoDocumentsApart pins that one pointer in -// two documents is two positions: a hint recorded at the source's /X names -// nothing at another document's /X, and each keeps its own. ext.yaml's content -// read in its scope at the source's coordinates (InScope) is a third, though -// its scope is ext.yaml's. -func TestDeclareHint_KeepsOnePointerInTwoDocumentsApart(t *testing.T) { +// TestNameAt_NamesAPositionInTheContextsDocument pins that one pointer in two +// documents is two positions: ext.yaml's /P/schema is a parameter's schema, +// named by the parameter's name, while the source's /P/schema is no schema the +// grammar reads, named from its pointer. ext.yaml's content read in its scope +// at the source's coordinates (InScope) names the source's, and a context +// given no load names every position from its pointer. +func TestNameAt_NamesAPositionInTheContextsDocument(t *testing.T) { t.Parallel() - c, _ := loadSplit(t, "X: {type: string}\nP: {name: p, in: query}\n") + c, _ := loadSplit(t, "X: {type: string}\nP: {name: p, in: query, schema: {type: string}}\n") there, ok := lowering.InDocument(c, extDoc(t, c)) require.True(t, ok) - c.DeclareHint("/X", "source") - _, recorded := there.HintAt("/X") - assert.False(t, recorded, "the source's /X is not ext.yaml's") - there.DeclareHint("/X", "ext") - hint, _ := c.HintAt("/X") - assert.Equal(t, "source", hint) - hint, _ = there.HintAt("/X") - assert.Equal(t, "ext", hint) - _, recorded = there.InScope(there.RefScope()).HintAt("/X") - assert.False(t, recorded, "ext.yaml's scope at the source's coordinates is neither") + assert.Equal(t, "p", lowering.NameAt(there, "/P/schema")) + assert.Equal(t, "schema", lowering.NameAt(c, "/P/schema")) + assert.Equal(t, "schema", lowering.NameAt(there.InScope(there.RefScope()), "/P/schema")) + assert.Equal(t, "schema", lowering.NameAt(lowering.Ctx{}, "/P/schema")) } -// TestOperationIDAt_ReadsTheDeclaringDocument pins where a position's -// operationId is read: in the source's YAML, and its model where only a -// reference reaches the position; in another document's YAML alone, never the -// source's model, for that document's; and nowhere for a context given no -// document. -func TestOperationIDAt_ReadsTheDeclaringDocument(t *testing.T) { +// TestNameThrough_NamesWhereAReferenceLands pins that the name a $ref suggests +// is that of the position it lands on, in whichever document, and that a +// position writing no $ref, or a context given no load, suggests none. +func TestNameThrough_NamesWhereAReferenceLands(t *testing.T) { t.Parallel() - c, _ := loadSplit(t, "X: {type: string}\nP: {name: p, in: query}\nO: {operationId: extOp}\n") - there, ok := lowering.InDocument(c, extDoc(t, c)) - require.True(t, ok) - assert.Equal(t, "extOp", lowering.OperationIDAt(there, "/O"), "ext.yaml's, read in its YAML") - assert.Empty(t, lowering.OperationIDAt(c, "/O"), "the source holds no /O") - assert.Equal(t, "getA", lowering.OperationIDAt(c, "/paths/~1a/get")) - assert.Empty(t, lowering.OperationIDAt(there, "/paths/~1a/get"), "ext.yaml writes none there, whatever the source does") - - doc, _, err := load.Load(t.Context(), 0, compilers.Source{Path: "spec.yaml", Data: []byte( - "openapi: 3.1.0\ninfo: {title: T, version: \"1\"}\npaths:\n /a: {get: {operationId: getA}}\n" + - " /b: {$ref: '#/components/pathItems/P'}\ncomponents:\n pathItems:\n P: {put: {operationId: putP}}\n")}, - load.Options{}) - require.NoError(t, err) - source := lowering.New(0, doc.Doc, doc.Source, "", lowering.Limits{}, lowering.StreamingMedia{}, - lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}).WithLoaded(doc) - assert.Equal(t, "getA", lowering.OperationIDAt(source, "/paths/~1a/get"), "the source's, read where it is written") - assert.Equal(t, "putP", lowering.OperationIDAt(source, "/paths/~1b/put"), "and in its model past a $ref") - assert.Empty(t, lowering.OperationIDAt(lowering.Ctx{}, "/paths/~1a/get"), "a context given no document") + c, _ := loadSplit(t, "X: {type: string}\nP: {name: p, in: query}\n") + assert.Equal(t, "X", lowering.NameThrough(c, "/components/schemas/A")) + assert.Empty(t, lowering.NameThrough(c, "/paths/~1a/get"), "nothing is referenced there") + assert.Empty(t, lowering.NameThrough(lowering.Ctx{}, "/components/schemas/A")) } // TestDeclared_LowersAReferencedObjectWhereItIsDeclared pins where a reference diff --git a/compilers/openapi/internal/lowering/lowering.go b/compilers/openapi/internal/lowering/lowering.go index b957b6d8e..f94d4168b 100644 --- a/compilers/openapi/internal/lowering/lowering.go +++ b/compilers/openapi/internal/lowering/lowering.go @@ -15,11 +15,13 @@ import ( soa "github.com/speakeasy-api/openapi/openapi" + "github.com/dexpace/morphic/compilers/openapi/internal/anchors" "github.com/dexpace/morphic/compilers/openapi/internal/bind" "github.com/dexpace/morphic/compilers/openapi/internal/bind/typed" "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/load" + "github.com/dexpace/morphic/compilers/openapi/internal/naming" "github.com/dexpace/morphic/compilers/openapi/internal/overlay" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" "github.com/dexpace/morphic/ir" @@ -138,13 +140,11 @@ type Ctx struct { // Nil in a zero context. typePositions map[scopedPosition]bind.Position - // hints records the hint each schema position is named by, as the first - // lowering to reach it names it. Copies share it on purpose, as - // typePositions: a node hoisted where a position's own lowering left none - // takes the name that lowering gave the position, which every declaration - // has recorded before such a node is built (compile.Types.Finish). Nil in a - // zero context, which records none. - hints map[scopedPosition]string + // anchors is the source's $dynamicAnchor index, built on first use + // (DynamicAnchorSites). Copies share it on purpose, as typePositions: it + // reads only the document. Nil in a zero context, which builds it on + // every ask. + anchors *anchorMemo // reached records where each reference a lowering resolved landed in the // source. Copies share it on purpose: no lowering reads it, so it cannot @@ -160,6 +160,9 @@ type Ctx struct { // in RefScope would allocate each time. type loadResult struct { siblings load.Siblings + // names names the nodes built at the positions of the documents the + // bindings read, remembering each name (NameAt). + names *naming.Namer // others is every document a reference read beside the source. others load.Others // reads holds those readings in the scope fields they fill: Holds over the @@ -169,8 +172,7 @@ type loadResult struct { } // scopedPosition is a position, the document it is in, and the scope it is -// read in (see Within), which a TypePosition answer and a recorded hint -// (DeclareHint) are kept under. +// read in (see Within), which a TypePosition answer is kept under. type scopedPosition struct { foreign bool holder string @@ -202,7 +204,7 @@ func New(srcIndex int, doc *soa.OpenAPI, src ir.SourceInfo, grouping GroupingStr overlay: origin, typePositions: map[scopedPosition]bind.Position{}, - hints: map[scopedPosition]string{}, + anchors: &anchorMemo{}, reached: resolve.NewReached(), } } @@ -257,12 +259,12 @@ func (c Ctx) WithAuth(auth map[ir.AuthID]ir.AuthScheme) Ctx { // being derived from it, since resolving either reads every document the // source names, which only the load phase may (GitHub #757). One call sets // both, so no context carries one without the other. The copy remembers no -// TypePosition answer or hint of c's, which read c's bindings and documents. +// TypePosition answer or name of c's, which read c's bindings and documents. func (c Ctx) WithLoaded(doc *load.Document) Ctx { - c.loaded = &loadResult{siblings: doc.Siblings, others: doc.Others, reads: resolve.Scope{Holds: doc.Holds, - Beside: doc.Siblings.Beside, Writes: doc.Siblings.Writes, Bindings: doc.Bindings}} + c.loaded = &loadResult{siblings: doc.Siblings, others: doc.Others, names: naming.New(doc.Bindings.Table()), + reads: resolve.Scope{Holds: doc.Holds, Beside: doc.Siblings.Beside, Writes: doc.Siblings.Writes, + Bindings: doc.Bindings}} c.typePositions = map[scopedPosition]bind.Position{} - c.hints = map[scopedPosition]string{} return c } @@ -293,30 +295,6 @@ func (c Ctx) TypePosition(at bind.Position, read func() bind.Position) bind.Posi return named } -// DeclareHint records hint as the name the lowering reaching pointer, in c's -// document and scope, gives it, where none is recorded there yet. Every -// lowering reaching a position names it alike, so the first record stands for -// all of them. -func (c Ctx) DeclareHint(pointer jsontext.Pointer, hint string) { - key := hintKey(c, pointer) - if _, seen := c.hints[key]; !seen && c.hints != nil { - c.hints[key] = hint - } -} - -// HintAt returns the hint recorded for pointer in c's document and scope -// (DeclareHint), and false where none is. -func (c Ctx) HintAt(pointer jsontext.Pointer) (string, bool) { - hint, ok := c.hints[hintKey(c, pointer)] - return hint, ok -} - -// hintKey is what a hint for pointer, in c's document and scope, is kept under. -func hintKey(c Ctx, pointer jsontext.Pointer) scopedPosition { - return scopedPosition{foreign: c.foreign, holder: c.holder, home: c.home, - at: bind.Position{Doc: c.homeDoc, Pointer: pointer}} -} - // Siblings returns the fields the load phase modelled beside the document's // path item $refs (see WithLoaded). A load.Siblings only answers questions, so // handing it out lets no copy of c change what another reads. @@ -503,20 +481,26 @@ func IDsAt(c Ctx, source int) ids.Document { return ids.Document{} } -// OperationIDAt returns the operationId of the operation written at pointer in -// c's document, and "" where none is there or it names none: read in that -// document's YAML, and for the source's also in its model, which reaches a -// position only a reference leads to (resolve.OperationIDAt). -func OperationIDAt(c Ctx, pointer jsontext.Pointer) string { - model := c.Doc - if _, other := homeOf(c); other { - model = nil +// NameAt returns the hint the node built at pointer in c's document is named +// by (naming.Namer.At): a function of the documents and the coordinate alone, +// whichever lowering builds the node, read once for every copy of c. A +// context given no bindings names a position by its pointer alone. +func NameAt(c Ctx, pointer jsontext.Pointer) string { + at := bind.Position{Doc: c.homeDoc, Pointer: pointer} + if c.loaded == nil { + return naming.New(nil).At(at) } - table := c.Bindings().Table() - if table == nil { - return resolve.OperationIDAt(nil, model, pointer) + return c.loaded.names.At(at) +} + +// NameThrough returns the name the $ref written at pointer in c's document +// suggests (naming.Namer.Through), and "" where it suggests none. A context +// given no bindings resolves no $ref, so it suggests none. +func NameThrough(c Ctx, pointer jsontext.Pointer) string { + if c.loaded == nil { + return "" } - return resolve.OperationIDAt(table.Document(c.homeDoc).Root, model, pointer) + return c.loaded.names.Through(bind.Position{Doc: c.homeDoc, Pointer: pointer}) } // homeOf returns the document other than the source whose positions c reads, @@ -566,3 +550,36 @@ func (c Ctx) ProvenanceAt(pointer jsontext.Pointer, from ...jsontext.Pointer) ir } return ir.Provenance{Source: c.overlay.IndexOf(pointer, from, c.SrcIndex), Pointer: pointer} } + +// anchorMemo holds the source's $dynamicAnchor index once it is built. +type anchorMemo struct { + built bool + index anchors.Index +} + +// DynamicAnchorSites returns the pointers of the source declaring the +// $dynamicAnchor name (anchors.Build), and, the first time any copy of c asks, +// a warning when the index walk stopped at a bound. It is built on first use +// rather than at entry because that warning would report on documents that +// never write $dynamicRef (GitHub #172). A context holding no document finds +// none. +func (c Ctx) DynamicAnchorSites(name string) ([]jsontext.Pointer, []ir.Diagnostic) { + if c.Doc == nil { + return nil, nil + } + memo := c.anchors + if memo == nil { + memo = &anchorMemo{} + } + if memo.built { + return memo.index.Sites(name), nil + } + memo.index, memo.built = anchors.Build(c.Doc.GetRootNode()), true + if memo.index.Complete() { + return memo.index.Sites(name), nil + } + return memo.index.Sites(name), []ir.Diagnostic{c.DiagAt(ir.SeverityWarning, diag.DegradedConstruct, "", + "the $dynamicAnchor index stopped at its walk bounds (%d levels, %d nodes); "+ + "a $dynamicRef expanded below is not verified to name the document's only anchor of its name", + anchors.MaxDepth, anchors.MaxNodes)} +} diff --git a/compilers/openapi/internal/lowering/lowering_test.go b/compilers/openapi/internal/lowering/lowering_test.go index 5656beb83..fc8c33bb0 100644 --- a/compilers/openapi/internal/lowering/lowering_test.go +++ b/compilers/openapi/internal/lowering/lowering_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" @@ -14,6 +15,7 @@ import ( yaml "gopkg.in/yaml.v3" "github.com/dexpace/morphic/compilers" + "github.com/dexpace/morphic/compilers/openapi/internal/anchors" "github.com/dexpace/morphic/compilers/openapi/internal/bind" "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/load" @@ -580,27 +582,57 @@ func TestWithLoaded_ReadsRawYAMLThroughTheLoadIndex(t *testing.T) { assert.NotNil(t, base.WithLoaded(doc).RefScope().Holds) } -// TestCtx_DeclareHintKeepsTheFirstRecordForEveryCopy pins the record a hoisted -// node takes its name from: the first hint recorded for a position stands, -// every copy of a context reads and writes the one record, except a copy given -// another load, and a zero context records nothing. -func TestCtx_DeclareHintKeepsTheFirstRecordForEveryCopy(t *testing.T) { +// TestDynamicAnchorSites_ReportsATruncatedWalkOnce pins what a reader is told +// when the $dynamicAnchor index is partial. Truncation only ever drops +// declarations, and a dropped duplicate reads as "declared exactly once", the +// count that expands, so an unreported truncation would turn an ambiguous +// reference into a confident one. The index is built once for every copy of +// the context, so the warning is too. +func TestDynamicAnchorSites_ReportsATruncatedWalkOnce(t *testing.T) { t.Parallel() - c := lowering.New(0, &soa.OpenAPI{}, ir.SourceInfo{}, "", lowering.Limits{}, lowering.StreamingMedia{}, + // The walk descends into every key, so an extension at the document root + // nests the tree past the cap without any schema lowering seeing it. + deep := strings.Repeat("{a: ", anchors.MaxDepth) + "1" + strings.Repeat("}", anchors.MaxDepth) + src := openapitest.ComponentSpec(" A: {$dynamicAnchor: m, type: string}\n") + "x-deep: " + deep + "\n" + doc, diags, err := load.Load(t.Context(), 0, compilers.Source{Path: "spec.yaml", Data: []byte(src)}, load.Options{}) + require.NoError(t, err) + openapitest.RequireNoErrorDiags(t, diags) + c := lowering.New(0, doc.Doc, doc.Source, "", lowering.Limits{}, lowering.StreamingMedia{}, + lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}).WithLoaded(doc) + + sites, got := c.DynamicAnchorSites("m") + assert.Equal(t, []jsontext.Pointer{"/components/schemas/A"}, sites, "the index is built even when partial") + require.Len(t, got, 1, "the truncation is announced: %+v", got) + assert.Equal(t, ir.SeverityWarning, got[0].Severity) + assert.Contains(t, got[0].Message, "not verified") + + _, again := c.InScope(c.RefScope()).DynamicAnchorSites("absent") + assert.Empty(t, again, "a copy reads the built index and does not announce it again") + + bare := lowering.Ctx{Doc: doc.Doc} + _, first := bare.DynamicAnchorSites("m") + _, second := bare.DynamicAnchorSites("m") + assert.Len(t, first, 1, "a context New did not build keeps no index") + assert.Len(t, second, 1, "so it builds one, and announces it, on every ask") +} + +// TestDynamicAnchorSites_ReadsACompleteIndexQuietly pins the ordinary case: a +// document the walk reads whole names each anchor's sites and reports +// nothing, and a context holding no document finds none. +func TestDynamicAnchorSites_ReadsACompleteIndexQuietly(t *testing.T) { + t.Parallel() + src := openapitest.ComponentSpec(" A: {$dynamicAnchor: m, type: string}\n B: {$dynamicAnchor: m}\n") + doc, diags, err := load.Load(t.Context(), 0, compilers.Source{Path: "spec.yaml", Data: []byte(src)}, load.Options{}) + require.NoError(t, err) + openapitest.RequireNoErrorDiags(t, diags) + c := lowering.New(0, doc.Doc, doc.Source, "", lowering.Limits{}, lowering.StreamingMedia{}, lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}) - _, recorded := c.HintAt("/x") - assert.False(t, recorded, "nothing is recorded before a lowering reaches the position") - - c.DeclareHint("/x", "declared") - c.InScope(c.RefScope()).DeclareHint("/x", "later") - hint, recorded := c.InScope(c.RefScope()).HintAt("/x") - require.True(t, recorded, "a copy reads what another recorded") - assert.Equal(t, "declared", hint, "and the first record stands") - _, recorded = c.WithLoaded(&load.Document{}).HintAt("/x") - assert.False(t, recorded, "a copy given another load remembers none of the old one's") - - var zero lowering.Ctx - zero.DeclareHint("/x", "declared") - _, recorded = zero.HintAt("/x") - assert.False(t, recorded, "a zero context records nothing") + + sites, got := c.DynamicAnchorSites("m") + assert.Equal(t, []jsontext.Pointer{"/components/schemas/A", "/components/schemas/B"}, sites) + assert.Empty(t, got) + + sites, got = lowering.Ctx{}.DynamicAnchorSites("m") + assert.Empty(t, sites) + assert.Empty(t, got) } diff --git a/compilers/openapi/internal/naming/doc.go b/compilers/openapi/internal/naming/doc.go new file mode 100644 index 000000000..b6845f3ca --- /dev/null +++ b/compilers/openapi/internal/naming/doc.go @@ -0,0 +1,11 @@ +// Package naming names every node the OpenAPI compiler builds as a function +// of the documents and the node's coordinate (GitHub #981), so which +// lowering reaches a coordinate first cannot change what it is called. +// +// A name is an owner's, then the structure beneath it. The owner is the +// nearest object holding a schema as the grammar reads it (bind.Table.Kinds): +// a component schema, request body, response, parameter or header, or a +// document whose root it is. Each keyword beneath composes the name of the +// position it leads to; a branch holding a $ref takes the name of the +// position it lands on. +package naming diff --git a/compilers/openapi/internal/naming/naming.go b/compilers/openapi/internal/naming/naming.go new file mode 100644 index 000000000..1b97c11fe --- /dev/null +++ b/compilers/openapi/internal/naming/naming.go @@ -0,0 +1,110 @@ +package naming + +import ( + "cmp" + "encoding/json/jsontext" + "slices" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/bind" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" +) + +// MaxHops bounds how many $refs one name follows through composition +// branches that each hold one: a loop of them, and any longer chain, stops +// there, keeping the name it has reached. An acyclic chain that long is legal, +// though no real document writes one. +const MaxHops = 64 + +// Namer names the nodes built at the positions of the documents a table read. +// It remembers each name it gives, and a name is a function of the documents +// alone, so the memo answers as a fresh Namer would. A Namer must not outlive +// the trees its table holds, and is not safe for concurrent use. +type Namer struct { + table *bind.Table + views map[bind.DocID]*nodeview.View + memo map[bind.Position]string +} + +// New returns a Namer over the documents table read. A nil table reads none, +// and names each position by its pointer alone. +func New(table *bind.Table) *Namer { + return &Namer{table: table, views: map[bind.DocID]*nodeview.View{}, memo: map[bind.Position]string{}} +} + +// At returns the hint the node at p is named by: the hint compile.NamingHint +// takes, before it is made neutral. A component schema's own node is named by +// its key and carries no hint, but the positions beneath it compose theirs +// from it, so At answers for it too. +func (n *Namer) At(p bind.Position) string { + if hint, ok := n.memo[p]; ok { + return hint + } + hint := n.at(p, 0) + n.memo[p] = hint + return hint +} + +// at is At having followed hops $refs already. It neither reads nor writes +// the memo: a chain followed from another position stops at MaxHops sooner +// than one followed from p, so a name remembered from either would make the +// other depend on which was asked first. +func (n *Namer) at(p bind.Position, hops int) string { + tokens := slices.Collect(p.Pointer.Tokens()) + start, hint := n.owner(p, tokens) + hint, branch := steps(hint, tokens[start:]) + if branch { + hint = n.branch(p, hint, hops) + } + return hint +} + +// branch names the composition branch at p, which steps named positionally: +// a branch holding a $ref is named after it (Through), keeping its positional +// name where the $ref suggests none. +func (n *Namer) branch(p bind.Position, positional string, hops int) string { + return cmp.Or(n.through(p, hops), positional) +} + +// Through returns the name the $ref written at p suggests: the hint of the +// position it lands on, else what its text suggests (unresolvedName). It is "" +// where p holds no $ref, and where a $ref names nothing either way, such as +// one landing on a component keyed "". +func (n *Namer) Through(p bind.Position) string { + return n.through(p, 0) +} + +// through is Through having followed hops $refs already. +func (n *Namer) through(p bind.Position, hops int) string { + if n.table == nil || hops >= MaxHops { + return "" + } + e, ok := n.table.Entry(p) + switch { + case !ok: + return "" + case e.Hop.Status == bind.Resolved: + return n.at(e.Hop.Target, hops+1) + default: + return unresolvedName(e.Hop.Ref) + } +} + +// node returns the node written at pointer in doc, read as the parser reads +// it, or nil where the document holds none. +func (n *Namer) node(doc bind.DocID, pointer jsontext.Pointer) *yaml.Node { + if n.table == nil || doc < 0 || int(doc) >= n.table.Documents() { + return nil + } + v, ok := n.views[doc] + if !ok { + v = nodeview.New() + n.views[doc] = v + } + path, complete := v.DocumentPath(n.table.Document(doc).Root, pointer) + if !complete { + return nil + } + return path[len(path)-1] +} diff --git a/compilers/openapi/internal/naming/naming_internal_test.go b/compilers/openapi/internal/naming/naming_internal_test.go new file mode 100644 index 000000000..9b9769bd3 --- /dev/null +++ b/compilers/openapi/internal/naming/naming_internal_test.go @@ -0,0 +1,72 @@ +package naming + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dexpace/morphic/compilers" + "github.com/dexpace/morphic/compilers/openapi/internal/bind" + "github.com/dexpace/morphic/compilers/openapi/internal/load" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" + "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" + "github.com/dexpace/morphic/engine/reader" +) + +// loaded returns the table a load of spec binds. +func loaded(t *testing.T, spec string) *bind.Table { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "root.yaml"), []byte(spec), 0o600)) + doc, _, err := load.Load(t.Context(), 0, compilers.Source{Path: filepath.Join(dir, "root.yaml"), + Data: []byte(spec)}, load.Options{Reader: reader.Disk{}}) + require.NoError(t, err) + require.NotNil(t, doc) + return doc.Bindings.Table() +} + +// TestOwnerOf_NamesOnlyAMediaTypeAContentMapHolds pins the guards on a media +// type's owner. The grammar reads a media type only as an entry of a content +// map, whose holder is a request body, a response, a parameter or a header, +// so neither guard is reached through At; each stands for a grammar that +// grows a media type anywhere else, which must name it nothing rather than +// read past the pointer's start. +func TestOwnerOf_NamesOnlyAMediaTypeAContentMapHolds(t *testing.T) { + t.Parallel() + n := New(nil) + hint, ok := n.ownerOf(bind.Source, []string{"schema"}, []bind.Kind{bind.KindMediaType, bind.KindSchema}, 0) + assert.False(t, ok, "a media type no content map holds") + assert.Empty(t, hint) + + hint, ok = n.contentOwner(bind.Source, "/x", bind.KindOther) + assert.False(t, ok, "content held by no object the grammar names") + assert.Empty(t, hint) +} + +// TestNode_ReadsNothingTheTableDoesNotHold pins that node reads only a +// document the table holds, and nothing a namer with no table is asked for. +func TestNode_ReadsNothingTheTableDoesNotHold(t *testing.T) { + t.Parallel() + tb := loaded(t, openapitest.ComponentSpec(" A: {type: object}\n")) + n := New(tb) + require.NotNil(t, n.node(bind.Source, "/components/schemas/A"), "sanity: the source holds A") + assert.Nil(t, n.node(-1, "/components/schemas/A")) + assert.Nil(t, n.node(bind.DocID(tb.Documents()), "/components/schemas/A")) + assert.Nil(t, New(nil).node(bind.Source, "/components/schemas/A")) +} + +// TestNode_ReadsNothingPastTheMergeBound pins that a position reached through +// a `<<` chain deeper than the view reads is no node: the parser may read one +// there, so reading none names nothing rather than guessing. +func TestNode_ReadsNothingPastTheMergeBound(t *testing.T) { + t.Parallel() + spec := func(levels int) string { + return openapitest.ComponentSpec(" A: {type: object}\n") + + "x-deep: " + openapitest.NestedMerges(levels, "{b: {name: x}}") + "\n" + } + assert.NotNil(t, New(loaded(t, spec(nodeview.MergeDepthLimit))).node(bind.Source, "/x-deep/b")) + assert.Nil(t, New(loaded(t, spec(nodeview.MergeDepthLimit+1))).node(bind.Source, "/x-deep/b")) +} diff --git a/compilers/openapi/internal/naming/naming_test.go b/compilers/openapi/internal/naming/naming_test.go new file mode 100644 index 000000000..4ccd63668 --- /dev/null +++ b/compilers/openapi/internal/naming/naming_test.go @@ -0,0 +1,489 @@ +package naming_test + +import ( + "encoding/json/jsontext" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dexpace/morphic/compilers" + "github.com/dexpace/morphic/compilers/compile" + "github.com/dexpace/morphic/compilers/openapi/internal/bind" + "github.com/dexpace/morphic/compilers/openapi/internal/load" + "github.com/dexpace/morphic/compilers/openapi/internal/naming" + "github.com/dexpace/morphic/engine/reader" +) + +// head is the part of every fixture before its paths. +const head = "openapi: 3.1.0\ninfo: {title: T, version: \"1\"}\n" + +// table loads spec as root.yaml beside others, each written to disk under its +// name, and returns the table the load bound. +func table(t *testing.T, spec string, others map[string]string) *bind.Table { + t.Helper() + dir := t.TempDir() + for name, body := range others { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + doc, _, err := load.Load(t.Context(), 0, compilers.Source{Path: filepath.Join(dir, "root.yaml"), + Data: []byte(spec)}, load.Options{Reader: reader.Disk{}}) + require.NoError(t, err) + require.NotNil(t, doc) + return doc.Bindings.Table() +} + +// at is the source's position at pointer. +func at(pointer string) bind.Position { + return bind.Position{Doc: bind.Source, Pointer: jsontext.Pointer(pointer)} +} + +// docNamed returns the number of the document the table read as name. +func docNamed(t *testing.T, tb *bind.Table, name string) bind.DocID { + t.Helper() + for id := range tb.Documents() { + if strings.HasSuffix(tb.Document(bind.DocID(id)).Name, "/"+name) { + return bind.DocID(id) + } + } + require.FailNow(t, "the load read no document named "+name) + return 0 +} + +// TestAt_NamesASchemaByItsStructure pins the walk down a schema from its +// owner: each step composes the enclosing name by role (items, +// additionalProperties, contentSchema, a patternProperties entry, a +// prefixItems slot) or, for a keyed map, takes the key itself. A key is never +// read as a keyword (GitHub #518), and a branch holding no $ref is named +// variant_N. A namer with no table names these as one with the document does, +// since the owner is the component. +func TestAt_NamesASchemaByItsStructure(t *testing.T) { + t.Parallel() + tb := table(t, head+"paths: {}\ncomponents: {schemas: {Foo: {type: object}}}\n", nil) + tests := []struct { + name, pointer, want string + }{ + {"a bare component", "/components/schemas/Foo", "Foo"}, + {"items under a component", "/components/schemas/Foo/items", compile.SubHint("Foo", "item")}, + {"nested roles", "/components/schemas/Foo/items/additionalProperties", + compile.SubHint(compile.SubHint("Foo", "item"), "value")}, + {"a property literally named items is not the items keyword", "/components/schemas/Foo/properties/items", "items"}, + {"a property named items holding an array names its own items by role", + "/components/schemas/Foo/properties/items/items", compile.SubHint("items", "item")}, + {"a property literally named properties holding items names its own items by role", + "/components/schemas/Foo/properties/properties/items", compile.SubHint("properties", "item")}, + {"a property named additionalProperties whose own additionalProperties is a role", + "/components/schemas/Foo/properties/additionalProperties/additionalProperties", + compile.SubHint("additionalProperties", "value")}, + {"a patternProperties entry whose pattern holds ~1", "/components/schemas/Foo/patternProperties/a~1b", + compile.SubHint("Foo", "pattern")}, + {"a patternProperties entry keyed items is not the items keyword", + "/components/schemas/Foo/patternProperties/items", compile.SubHint("Foo", "pattern")}, + {"an items entry inside a patternProperties entry keyed items", + "/components/schemas/Foo/patternProperties/items/items", + compile.SubHint(compile.SubHint("Foo", "pattern"), "item")}, + {"a $defs entry keyed items", "/components/schemas/Foo/$defs/items", "items"}, + {"a dependentSchemas entry keyed items", "/components/schemas/Foo/dependentSchemas/items", "items"}, + {"a definitions entry keyed items", "/components/schemas/Foo/definitions/items", "items"}, + {"a dependencies entry keyed items", "/components/schemas/Foo/dependencies/items", "items"}, + {"a contentSchema", "/components/schemas/Foo/contentSchema", compile.SubHint("Foo", "content")}, + {"a prefixItems slot", "/components/schemas/Foo/prefixItems/2", compile.SubHint("Foo", "2")}, + {"a prefixItems member that is no slot is named after its own token", + "/components/schemas/Foo/prefixItems/x", "x"}, + {"an enum member a literal is lowered to", "/components/schemas/Foo/enum/1", compile.SubHint("Foo", "1")}, + {"a variant of a multi-typed schema is named as the schema is", "/components/schemas/Foo/type/1", "Foo"}, + {"a component literally named items", "/components/schemas/items", "items"}, + {"a oneOf branch holding no $ref is named variant_N", "/components/schemas/Foo/oneOf/0", "variant_0"}, + {"an anyOf branch", "/components/schemas/Foo/anyOf/12", "variant_12"}, + {"an allOf branch", "/components/schemas/Foo/allOf/1", "variant_1"}, + {"items beneath a branch is named off the branch", + "/components/schemas/Foo/oneOf/0/items", compile.SubHint("variant_0", "item")}, + {"a branch reached through a property named oneOf", + "/components/schemas/Foo/properties/oneOf/anyOf/0", "variant_0"}, + {"a property named oneOf is not a branch", "/components/schemas/Foo/properties/oneOf", "oneOf"}, + {"a non-numeric child of a composition keyword is not a branch", "/components/schemas/Foo/oneOf/x", "x"}, + {"a negative index is not a branch", "/components/schemas/Foo/oneOf/-1", "-1"}, + {"an empty index is not a branch", "/components/schemas/Foo/oneOf/", ""}, + {"an unrecognised keyword is named after its own token", "/components/schemas/Foo/not/items", + compile.SubHint("not", "item")}, + {"an extension keyword likewise", "/components/schemas/Foo/x-ext/items", compile.SubHint("x-ext", "item")}, + {"a keyed keyword with no key names itself", "/components/schemas/Foo/properties", "properties"}, + {`the component keyed ""`, "/components/schemas//items", compile.SubHint("", "item")}, + {"the schemas map holds no schema to own it", "/components/schemas", "schemas"}, + {"the empty pointer", "", ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, naming.New(tb).At(at(tc.pointer)), "read") + assert.Equal(t, tc.want, naming.New(nil).At(at(tc.pointer)), "by pointer alone") + }) + } +} + +// ownersSpec writes a schema under every owner the grammar names one for. +const ownersSpec = head + `paths: + /a: + parameters: + - {name: shared, in: query, schema: {type: object}} + post: + operationId: postA + parameters: + - {name: limit, in: query, schema: {type: object}} + - {name: filter, in: query, content: {application/json: {schema: {type: object}}}} + - $ref: '#/components/parameters/P' + requestBody: + content: + application/json: {schema: {type: object}} + application/jsonl: {itemSchema: {type: object}} + responses: + "200": + description: ok + headers: + X-Rate: {schema: {type: object}} + X-Body: {content: {application/json: {schema: {type: object}}}} + content: + application/json: {schema: {type: object}} + callbacks: + onEvent: + '{$request.body#/url}': + post: + operationId: notify + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} + put: + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} +webhooks: + hook: + post: + operationId: onHook + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} +components: + parameters: + P: {name: page, in: query, schema: {type: object}} + requestBodies: + Body: {content: {application/json: {schema: {type: object}}}} + responses: + Err: {description: e, content: {application/json: {schema: {type: object}}}} + headers: + Trace: {schema: {type: object}} + pathItems: + Item: + get: + operationId: getItem + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} +` + +// TestAt_NamesASchemaAfterItsOwner pins the owner a schema position is named +// for, as the grammar reads where it is written: a request body by its +// component's key or the operationId of the operation declaring it, else +// "request"; a response by its component's key, else "response"; a parameter +// by its name, its content included; a header by its key; an itemSchema as +// its media type's item. Each holds wherever the operation is written: a path +// item, a callback, a webhook or a component path item. +func TestAt_NamesASchemaAfterItsOwner(t *testing.T) { + t.Parallel() + tb := table(t, ownersSpec, nil) + const op = "/paths/~1a/post" + const json = "/content/application~1json/schema" + tests := []struct { + name, pointer, want string + }{ + {"a request body, by its operation", op + "/requestBody" + json, "postA_request"}, + {"beneath it, off that name", op + "/requestBody" + json + "/properties/x/items", compile.SubHint("x", "item")}, + {"a pattern's schema beneath it, off that name", op + "/requestBody" + json + "/patternProperties/a", + compile.SubHint("postA_request", "pattern")}, + {"an enum literal of it, off that name", op + "/requestBody" + json + "/enum/1", + compile.SubHint("postA_request", "1")}, + {"a variant of it multi-typed, by that name", op + "/requestBody" + json + "/type/1", "postA_request"}, + {"an itemSchema, as its media type's item", op + "/requestBody/content/application~1jsonl/itemSchema", + compile.SubHint("postA_request", "item")}, + {"a request body no operationId names", "/paths/~1a/put/requestBody" + json, "request"}, + {"a component request body, by its key", "/components/requestBodies/Body" + json, "Body"}, + {"a response", op + "/responses/200" + json, "response"}, + {"a tuple slot beneath it, off that name", op + "/responses/200" + json + "/prefixItems/1", + compile.SubHint("response", "1")}, + {"a component response, by its key", "/components/responses/Err" + json, "Err"}, + {"a parameter, by its name", op + "/parameters/0/schema", "limit"}, + {"a parameter's content", op + "/parameters/1" + json, "filter"}, + {"a path item's parameter", "/paths/~1a/parameters/0/schema", "shared"}, + {"a component parameter, by its name", "/components/parameters/P/schema", "page"}, + {"a header, by its key", op + "/responses/200/headers/X-Rate/schema", "X-Rate"}, + {"a header's content", op + "/responses/200/headers/X-Body" + json, "X-Body"}, + {"a component header, by its key", "/components/headers/Trace/schema", "Trace"}, + {"a callback's request body", op + "/callbacks/onEvent/{$request.body#~1url}/post/requestBody" + json, + "notify_request"}, + {"a webhook's request body", "/webhooks/hook/post/requestBody" + json, "onHook_request"}, + {"a component path item's request body", "/components/pathItems/Item/get/requestBody" + json, + "getItem_request"}, + {"a position the grammar reads as no schema, from its pointer", op + "/responses/200/description", "description"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, naming.New(tb).At(at(tc.pointer))) + }) + } +} + +// TestAt_AnswersAsAFreshNamerWould pins that a name At remembers is the one a +// Namer that never asked gives. +func TestAt_AnswersAsAFreshNamerWould(t *testing.T) { + t.Parallel() + tb := table(t, ownersSpec, nil) + p := at("/paths/~1a/post/requestBody/content/application~1json/schema") + read := naming.New(tb) + first := read.At(p) + assert.Equal(t, "postA_request", first) + assert.Equal(t, first, read.At(p), "asked again") + assert.Equal(t, first, naming.New(tb).At(p), "asked of a fresh namer") +} + +// TestAt_ReadsAnOwnerAsTheParserDoes pins that an owner's name is read from +// the document as the parser reads it: through an alias and a `<<` merge, a +// number as the text it spells, and not at all where the field is null, no +// scalar, or written in a mapping merged from past the depth the parser reads. +func TestAt_ReadsAnOwnerAsTheParserDoes(t *testing.T) { + t.Parallel() + spec := head + `x-names: {id: &id aliased} +paths: + /a: + post: + operationId: *id + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} + parameters: + - {<<: {name: merged}, in: query, schema: {type: object}} + - {name: ~, in: query, schema: {type: object}} + - {name: [a], in: query, schema: {type: object}} + put: + operationId: 7 + requestBody: {content: {application/json: {schema: {type: object}}}} + responses: {"200": {description: ok}} + parameters: + - {name: 42, in: query, schema: {type: object}} +` + read := naming.New(table(t, spec, nil)) + assert.Equal(t, "aliased_request", read.At(at("/paths/~1a/post/requestBody/content/application~1json/schema"))) + assert.Equal(t, "merged", read.At(at("/paths/~1a/post/parameters/0/schema"))) + assert.Empty(t, read.At(at("/paths/~1a/post/parameters/1/schema")), "a null name names nothing") + assert.Empty(t, read.At(at("/paths/~1a/post/parameters/2/schema")), "nor does one that is no scalar") + assert.Equal(t, "7_request", read.At(at("/paths/~1a/put/requestBody/content/application~1json/schema"))) + assert.Equal(t, "42", read.At(at("/paths/~1a/put/parameters/0/schema"))) +} + +// TestAt_NamesABranchAfterItsTarget pins that a composition branch holding a +// $ref takes the name of the position the $ref lands on (GitHub #521, #729), +// beneath a component or not, in this document or another, whose root is +// named for the document; keeps its own where that names nothing; and, where +// the $ref lands on no position, takes the name its text suggests: the +// position a fragment spells, decoded at both layers (GitHub #505), else what +// follows its last '/', decoded when that is UTF-8 (GitHub #520). +func TestAt_NamesABranchAfterItsTarget(t *testing.T) { + t.Parallel() + const under = "/paths/~1x/get/responses/200/content/application~1json/schema" + tests := []struct { + name, at, to, want string + }{ + {"a branch beneath a component", "/components/schemas/X/oneOf/0", "#/components/schemas/Y", "Y"}, + {"a branch elsewhere", under + "/oneOf/0", "#/components/schemas/Y", "Y"}, + {"a branch landing beneath a response", "/components/schemas/X/oneOf/0", "#" + under + "/items", + compile.SubHint("response", "item")}, + {"a branch landing on another branch", "/components/schemas/X/oneOf/0", "#/components/schemas/Z/anyOf/0", "Y"}, + {"a branch whose target names nothing", "/components/schemas/X/oneOf/0", "#/components/schemas/", "variant_0"}, + {"a branch landing in another document", "/components/schemas/X/oneOf/0", "./other.yaml#/Pet", "Pet"}, + {"a branch landing on another document's root", "/components/schemas/X/oneOf/0", "./other.yaml", "other.yaml"}, + {"a branch landing on another document's root by an empty fragment", "/components/schemas/X/oneOf/0", + "./other.yaml#", "other.yaml"}, + {"an unread document, by its fragment", "/components/schemas/X/oneOf/0", + "./absent.yaml#/components/schemas/Cat~1Dog", "Cat/Dog"}, + {"an unread document, a percent escape decoded", "/components/schemas/X/oneOf/0", + "./absent.yaml#/components/schemas/Fish%2DTank", "Fish-Tank"}, + {"an unread document, a structural position by its role", "/components/schemas/X/oneOf/0", + "./absent.yaml#/components/schemas/A/items", compile.SubHint("A", "item")}, + {"an unread document, a branch by its position", "/components/schemas/X/oneOf/0", + "./absent.yaml#/components/schemas/A/oneOf/3", "variant_3"}, + {"an unread document, no fragment", "/components/schemas/X/oneOf/0", "./schemas/Fish%2DTank.yaml", "Fish-Tank.yaml"}, + {"an unread document whose escape is invalid", "/components/schemas/X/oneOf/0", "./bad%ZZ.yaml", "bad%ZZ.yaml"}, + {"an unread document whose escape is not UTF-8", "/components/schemas/X/oneOf/0", "./x%FF.yaml", "x%FF.yaml"}, + {"an unread document, a fragment that is not UTF-8", "/components/schemas/X/oneOf/0", + "./absent.yaml#/components/schemas/%FF", "%FF"}, + {"an unread document, a lone slash", "/components/schemas/X/oneOf/0", "./absent.yaml#/", "variant_0"}, + {"an unread document, a $anchor", "/components/schemas/X/oneOf/0", "./absent.yaml#anchor", "absent.yaml#anchor"}, + {"a position that is no branch keeps its own", "/components/schemas/X/items", "#/components/schemas/Y", + compile.SubHint("X", "item")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + read := naming.New(table(t, branchSpec(tc.at, tc.to), map[string]string{ + "other.yaml": "Pet: {type: object}\n", + })) + assert.Equal(t, tc.want, read.At(at(tc.at))) + }) + } +} + +// branchSpec writes a $ref to to at, a composition branch or an items +// position, beneath the component X or a response, beside the schema Y, a +// branch of Z holding a $ref to Y, and the component keyed "". +func branchSpec(at, to string) string { + holder := "{oneOf: [{$ref: '" + to + "'}]}" + if strings.HasSuffix(at, "/items") { + holder = "{type: array, items: {$ref: '" + to + "'}}" + } + paths := " /x: {get: {responses: {'200': {description: ok, content: {application/json: {schema: " + + "{type: array, items: {type: string}}}}}}}}\n" + x := "" + if strings.HasPrefix(at, "/components/schemas/X") { + x = " X: " + holder + "\n" + } else { + paths = " /x: {get: {responses: {'200': {description: ok, content: {application/json: {schema: " + + holder + "}}}}}}\n" + } + return head + "paths:\n" + paths + "components:\n schemas:\n" + x + + " Y: {type: object}\n Z: {anyOf: [{$ref: '#/components/schemas/Y'}]}\n \"\": {type: object}\n" +} + +// TestAt_StopsAtTheHopCap pins MaxHops at its boundary. Each branch holds a +// $ref to the next and the last to T: the first is named after T while T is +// within MaxHops hops, and keeps its own name once it is not. The second +// branch starts a hop further on, so it still reaches T, whichever of the two +// is asked first: a name a capped chain reaches on the way is not remembered. +func TestAt_StopsAtTheHopCap(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + branches int + first string + }{ + {"T within reach", naming.MaxHops, "T"}, + {"T one hop past the cap", naming.MaxHops + 1, "variant_0"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var spec strings.Builder + spec.WriteString(head + "paths: {}\ncomponents:\n schemas:\n") + for i := range tc.branches { + next := "#/components/schemas/T" + if i+1 < tc.branches { + next = "#/components/schemas/S" + strconv.Itoa(i+1) + "/oneOf/0" + } + spec.WriteString(" S" + strconv.Itoa(i) + ": {oneOf: [{$ref: '" + next + "'}]}\n") + } + spec.WriteString(" T: {type: object}\n") + tb := table(t, spec.String(), nil) + first, second := at("/components/schemas/S0/oneOf/0"), at("/components/schemas/S1/oneOf/0") + + read := naming.New(tb) + assert.Equal(t, tc.first, read.At(first)) + assert.Equal(t, "T", read.At(second), "asked after the first") + reversed := naming.New(tb) + assert.Equal(t, "T", reversed.At(second)) + assert.Equal(t, tc.first, reversed.At(first), "asked after the second") + }) + } +} + +// TestThrough_IsTheNameARefSuggests pins Through apart from the branch rule it +// serves: the name a $ref suggests wherever it is written, and "" where none +// is written or the one written suggests none. +func TestThrough_IsTheNameARefSuggests(t *testing.T) { + t.Parallel() + read := naming.New(table(t, head+`paths: {} +components: + schemas: + A: {allOf: [{$ref: '#/components/schemas/B'}, {$ref: '#/components/schemas/'}, {type: object}]} + B: {type: object} + "": {type: object} + C: {$ref: '#/components/schemas/B'} +`, nil)) + assert.Equal(t, "B", read.Through(at("/components/schemas/A/allOf/0"))) + assert.Empty(t, read.Through(at("/components/schemas/A/allOf/1")), `a component keyed "" names nothing`) + assert.Empty(t, read.Through(at("/components/schemas/A/allOf/2")), "no $ref is written there") + assert.Equal(t, "B", read.Through(at("/components/schemas/C")), "a $ref that is no branch suggests a name too") + assert.Empty(t, naming.New(nil).Through(at("/components/schemas/A/allOf/0")), "a namer with no table reads none") +} + +// TestAt_NamesWhatAReferenceReadsAsASchema pins positions the grammar reads as +// raw YAML, or in another document, named as the reference reaching them +// reads them (bind.Table.Kinds): a schema an extension holds is named from +// its pointer, as no owner names it, a parameter another document holds by +// its name, and a document a reference reads as a schema after the document. +func TestAt_NamesWhatAReferenceReadsAsASchema(t *testing.T) { + t.Parallel() + tb := table(t, head+`x-lib: {Thing: {type: object, properties: {p: {type: object}}}} +paths: + /a: + get: + parameters: [{$ref: './params.yaml#/Size'}] + responses: {"200": {description: ok}} +components: + schemas: + A: {$ref: '#/x-lib/Thing'} + B: {$ref: './pet.yaml'} + C: {$ref: './list.yaml'} +`, map[string]string{ + "list.yaml": "type: array\nitems: {type: object}\n", + "params.yaml": "Size: {name: size, in: query, schema: {type: object, properties: {q: {type: object}}}}\n", + "pet.yaml": "type: object\nproperties: {tag: {type: object, items: {type: string}}}\n", + }) + read := naming.New(tb) + params, pet, list := docNamed(t, tb, "params.yaml"), docNamed(t, tb, "pet.yaml"), docNamed(t, tb, "list.yaml") + + assert.Equal(t, "Thing", read.At(at("/x-lib/Thing")), "an extension's schema is named from its pointer") + assert.Equal(t, "p", read.At(at("/x-lib/Thing/properties/p"))) + assert.Equal(t, "size", read.At(bind.Position{Doc: params, Pointer: "/Size/schema"})) + assert.Equal(t, compile.SubHint("q", "item"), + read.At(bind.Position{Doc: params, Pointer: "/Size/schema/properties/q/items"})) + assert.Equal(t, "pet.yaml", read.At(bind.Position{Doc: pet}), "a document's root schema is named for the document") + assert.Equal(t, compile.SubHint("tag", "item"), read.At(bind.Position{Doc: pet, Pointer: "/properties/tag/items"})) + assert.Equal(t, compile.SubHint("list.yaml", "item"), read.At(bind.Position{Doc: list, Pointer: "/items"}), + "the root's items are named by role, not as a key") +} + +// TestAt_NamesAStructureAReferenceReadsInsideRawYAML pins that a reference +// reading raw YAML as an owner starts a structure of its own: a parameter, a +// request body or a response kept under a schema's x- key is named as one +// written in its own place is, not after the schema holding the raw YAML, and +// so is a parameter in a document another reference reads whole as a schema. +func TestAt_NamesAStructureAReferenceReadsInsideRawYAML(t *testing.T) { + t.Parallel() + tb := table(t, head+`paths: + /a: + post: + parameters: + - $ref: '#/components/schemas/S/x-param' + - $ref: './lib.yaml#/x-params/P' + requestBody: {$ref: '#/components/schemas/S/x-body'} + responses: + "200": {$ref: '#/components/schemas/S/x-resp'} +components: + schemas: + S: + type: object + x-param: {name: foo, in: query, schema: {type: object}} + x-body: {content: {application/json: {schema: {type: object}}}} + x-resp: {description: r, content: {application/json: {schema: {type: array, items: {type: object}}}}} + L: {$ref: './lib.yaml'} +`, map[string]string{ + "lib.yaml": "type: object\nx-params:\n P: {name: page, in: query, schema: {type: object}}\n", + }) + read := naming.New(tb) + lib := docNamed(t, tb, "lib.yaml") + const json = "/content/application~1json/schema" + + assert.Equal(t, "foo", read.At(at("/components/schemas/S/x-param/schema"))) + assert.Equal(t, "request", read.At(at("/components/schemas/S/x-body"+json))) + assert.Equal(t, compile.SubHint("response", "item"), read.At(at("/components/schemas/S/x-resp"+json+"/items"))) + assert.Equal(t, "page", read.At(bind.Position{Doc: lib, Pointer: "/x-params/P/schema"}), + "a parameter in a document read whole as a schema") + assert.Equal(t, "lib.yaml", read.At(bind.Position{Doc: lib}), "the document read whole") +} diff --git a/compilers/openapi/internal/naming/owners.go b/compilers/openapi/internal/naming/owners.go new file mode 100644 index 000000000..7cec1c399 --- /dev/null +++ b/compilers/openapi/internal/naming/owners.go @@ -0,0 +1,180 @@ +package naming + +import ( + "encoding/json/jsontext" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/compile" + "github.com/dexpace/morphic/compilers/openapi/internal/bind" + "github.com/dexpace/morphic/compilers/openapi/internal/ids" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" +) + +// componentSchemaTokens is the length of /components/schemas/, the +// pointer a component schema's walk is rooted at. +const componentSchemaTokens = 3 + +// owner returns the hint of the object p belongs to and how many of p's +// tokens lead to its schema: the schema whose structure holds p +// (structureRoot), named for what holds it there (ownerOf) or, where that +// schema is a document's root, for the document (documentName). A structure +// nothing names an owner of, such as one only a $ref reaches under an x- key, +// is named from its pointer alone (pointerOwner). +func (n *Namer) owner(p bind.Position, tokens []string) (int, string) { + if n.table == nil { + return pointerOwner(tokens) + } + kinds := n.table.Kinds(p) + root, ok := structureRoot(kinds) + switch { + case !ok: + return pointerOwner(tokens) + case root == 0: + return 0, n.documentName(p.Doc) + } + if hint, ok := n.ownerOf(p.Doc, tokens, kinds, root-1); ok { + return root, hint + } + return pointerOwner(tokens) +} + +// structureRoot returns the index in kinds of the schema whose structure +// holds the last position: the innermost schema on the way, which holds what +// lies past it, such as an enum literal, then the topmost schema above it that +// no position outside a structure (inStructure) separates it from. An object +// a reference reads inside a schema's raw YAML, such as a parameter kept +// under its x- key, so starts a structure of its own. ok is false where no +// schema is on the way. +func structureRoot(kinds []bind.Kind) (int, bool) { + root := -1 + for i, k := range kinds { + if k == bind.KindSchema { + root = i + } + } + if root < 0 { + return 0, false + } + for i := root - 1; i >= 0 && inStructure(kinds[i]); i-- { + if kinds[i] == bind.KindSchema { + root = i + } + } + return root, true +} + +// inStructure reports whether a position of kind k can lie inside a schema's +// structure: a schema, or a map or a list of them, a list being +// bind.KindOther. That kind covers objects such as a path item too, but each +// holds a schema only through an owner, which ends a structure as raw YAML +// does. +func inStructure(k bind.Kind) bool { + return k == bind.KindSchema || k == bind.KindSchemaMap || k == bind.KindOther +} + +// documentName is the hint the root of the document doc is named by where a +// reference reads it as a schema, which no owner holds: the document's name, +// what follows its last '/' (refHint), as a $ref spelling the document +// suggests it. doc is one the table holds, since only a reading the table +// recorded makes a root a schema (bind.Table.Kinds). +func (n *Namer) documentName(doc bind.DocID) string { + return refHint(n.table.Document(doc).Name) +} + +// ownerOf returns the hint of the schema whose key is tokens[i], held at the +// position tokens[:i] leads to: a component schema's key, the owner of the +// content a media type's schema or itemSchema belongs to, a parameter's name +// and a header's key. ok is false for a schema nothing names an owner of. +func (n *Namer) ownerOf(doc bind.DocID, tokens []string, kinds []bind.Kind, i int) (string, bool) { + holder := pointerOf(tokens[:i]) + switch kinds[i] { + case bind.KindSchemaMap: + return tokens[i], i > 0 && kinds[i-1] == bind.KindComponents + case bind.KindParameter: + return n.parameterName(doc, holder), true + case bind.KindHeader: + return ids.DeclarationHint(holder, holder.LastToken()), true + case bind.KindMediaType: + // The media type is keyed under its holder's content: holder is + // /content/. + if i < 2 { + return "", false + } + hint, ok := n.contentOwner(doc, holder.Parent().Parent(), kinds[i-2]) + if ok && tokens[i] == "itemSchema" { + return compile.SubHint(hint, "item"), true + } + return hint, ok + default: + return "", false + } +} + +// contentOwner returns the hint of the object at at, of kind k, whose content +// a media type's schema is: a request body (requestBodyName), a response by +// its component key, else "response", a parameter by its name and a header by +// its key. +func (n *Namer) contentOwner(doc bind.DocID, at jsontext.Pointer, k bind.Kind) (string, bool) { + switch k { + case bind.KindRequestBody: + return n.requestBodyName(doc, at), true + case bind.KindResponse: + return ids.DeclarationHint(at, "response"), true + case bind.KindParameter: + return n.parameterName(doc, at), true + case bind.KindHeader: + return ids.DeclarationHint(at, at.LastToken()), true + default: + return "", false + } +} + +// elsewhereBody names a request body no operationId names: one an operation +// without an operationId declares, or one no operation declares. +const elsewhereBody = "request" + +// requestBodyName is the hint the request body written at at is named by: its +// component's key, else `_request` for the operation it is +// written in, else elsewhereBody. It is a function of where the body is +// written, never of which operation mounts it (GitHub #433, #107). +func (n *Namer) requestBodyName(doc bind.DocID, at jsontext.Pointer) string { + if id := n.text(doc, at.Parent(), "operationId"); id != "" { + return ids.DeclarationHint(at, id+"_request") + } + return ids.DeclarationHint(at, elsewhereBody) +} + +// parameterName is the name the parameter written at at declares. +func (n *Namer) parameterName(doc bind.DocID, at jsontext.Pointer) string { + return n.text(doc, at, "name") +} + +// text returns the scalar the mapping written at at holds under field, as the +// parser reads a string field, null read as none. +func (n *Namer) text(doc bind.DocID, at jsontext.Pointer, field string) string { + v := nodeview.Child(n.node(doc, at), field) + if v == nil || v.Kind != yaml.ScalarNode || v.ShortTag() == "!!null" { + return "" + } + return v.Value +} + +// pointerOwner names a position from its pointer alone: beneath +// /components/schemas/, from that name; elsewhere from its first token, +// the name a position only references reach takes. +func pointerOwner(tokens []string) (int, string) { + start := min(1, len(tokens)) + if len(tokens) >= componentSchemaTokens && tokens[0] == "components" && tokens[1] == "schemas" { + start = componentSchemaTokens + } + if start == 0 { + return 0, "" + } + return start, tokens[start-1] +} + +// pointerOf joins tokens into a pointer. +func pointerOf(tokens []string) jsontext.Pointer { + return ids.Ptr(tokens...) +} diff --git a/compilers/openapi/internal/naming/steps.go b/compilers/openapi/internal/naming/steps.go new file mode 100644 index 000000000..f172320e2 --- /dev/null +++ b/compilers/openapi/internal/naming/steps.go @@ -0,0 +1,143 @@ +package naming + +import ( + "net/url" + "slices" + "strings" + "unicode/utf8" + + "github.com/dexpace/morphic/compilers/compile" + "github.com/dexpace/morphic/compilers/openapi/internal/fragment" +) + +// steps composes the hint of the position the schema keywords in rest lead to +// from enclosing, the hint of the position they start at, and reports whether +// that position is a composition branch, which branch names further. Each +// step consumes a token, bounding the walk. +func steps(enclosing string, rest []string) (hint string, branch bool) { + c := chain{base: enclosing} + for i := 0; i < len(rest); { + var step int + c, branch, step = stepOf(c, rest[i:]) + i += step + } + return compile.SubHints(c.base, c.parts...), branch +} + +// chain is a hint being composed: base, then each part compile.SubHint would +// append to it, joined once (compile.SubHints) rather than at every step, so a +// position n roles deep costs n parts rather than n² bytes. +type chain struct { + base string + parts []string +} + +// then is c with part composed onto it. steps keeps only the chain a step +// returns, so appending in place shares nothing still read. +func (c chain) then(part string) chain { + c.parts = append(c.parts, part) + return c +} + +// stepOf applies the keyword at rest[0] to the enclosing hint, returning the +// hint of the position it leads to, whether that position is a composition +// branch, and how many tokens the step consumed: two for a keyword whose +// children are keyed or indexed, one otherwise. A keyword the lowering does not +// walk (not, if, then, else, an unknown one) is named after its own token: only +// a reference reaches it, so any stable spelling keeps its name independent of +// declaration order. +func stepOf(enclosing chain, rest []string) (hint chain, branch bool, step int) { + keyword := rest[0] + if role, ok := structuralRoles[keyword]; ok { + return enclosing.then(role), false, 1 + } + if len(rest) < 2 { + return chain{base: keyword}, false, 1 + } + child := rest[1] + switch { + case keyedSchemaMaps[keyword]: + return chain{base: child}, false, 2 + case keyword == "patternProperties": + return enclosing.then("pattern"), false, 2 + case (keyword == "prefixItems" || keyword == "enum") && isDecimalIndex(child): + // A tuple's positional item, or the literal a member of an enum no + // enumeration holds is lowered to. + return enclosing.then(child), false, 2 + case compositionKeywords[keyword] && isDecimalIndex(child): + return chain{base: positionalBranch(child)}, true, 2 + case keyword == "type" && isDecimalIndex(child): + // One variant of a multi-typed schema, built from that schema itself, + // so it is named as the schema is. + return enclosing, false, 2 + } + return chain{base: keyword}, false, 1 +} + +// structuralRoles maps each keyword whose one schema the lowering names by role +// to that role. +var structuralRoles = map[string]string{ + "items": "item", + "additionalProperties": "value", + "contentSchema": "content", +} + +// keyedSchemaMaps are the keywords whose value maps a key to a schema that is +// named by the key alone: a property by its name, and the $defs, definitions, +// dependentSchemas and dependencies entries by theirs. patternProperties is +// keyed too but names by role, so stepOf takes it separately. +var keyedSchemaMaps = map[string]bool{ + "properties": true, "$defs": true, "definitions": true, "dependentSchemas": true, "dependencies": true, +} + +// compositionKeywords are the schema keywords whose numbered children are +// composition branches. +var compositionKeywords = map[string]bool{"allOf": true, "oneOf": true, "anyOf": true} + +// positionalBranch names an inline composition branch by its position, which +// is all there is to name it by: it has no $ref target to take a name from. +func positionalBranch(index string) string { return "variant_" + index } + +// isDecimalIndex reports whether s is a non-empty run of ASCII digits — the +// shape a branch's or a variant's pointer segment takes. +func isDecimalIndex(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// unresolvedName returns the name a $ref that lands on no position suggests +// from its text: the position its fragment spells, named from that pointer +// alone (pointerOwner) as a namer with no table names it, so +// `#/components/schemas/Cat~1Dog` suggests "Cat/Dog" (GitHub #505). A $ref +// spelling no pointer — a whole-document URI, a $anchor, a fragment whose +// bytes are not UTF-8 (GitHub #520) — suggests what refHint reads. +func unresolvedName(ref string) string { + p := fragment.Pointer(ref) + if !strings.HasPrefix(string(p), "/") || !utf8.ValidString(string(p)) { + return refHint(ref) + } + tokens := slices.Collect(p.Tokens()) + start, hint := pointerOwner(tokens) + hint, _ = steps(hint, tokens[start:]) + return hint +} + +// refHint returns what follows the last '/' of ref, percent-decoded when that +// decodes to UTF-8, so `./Fish%2DTank.yaml` suggests "Fish-Tank.yaml". +func refHint(ref string) string { + name := ref + if _, after, ok := strings.CutLast(ref, "/"); ok { + name = after + } + if decoded, err := url.PathUnescape(name); err == nil && utf8.ValidString(decoded) { + return decoded + } + return name +} diff --git a/compilers/openapi/internal/nodeview/lookup.go b/compilers/openapi/internal/nodeview/lookup.go new file mode 100644 index 000000000..09dfca436 --- /dev/null +++ b/compilers/openapi/internal/nodeview/lookup.go @@ -0,0 +1,97 @@ +package nodeview + +import ( + "slices" + + yaml "gopkg.in/yaml.v3" +) + +// Pairs is MappingPairs read by a view of its own, for a caller reading one +// mapping once, a document node unwrapped first: the view a walk shares is +// what keeps a merge chain from going cubic when every level of it is read. +func Pairs(n *yaml.Node) []Pair { + return New().MappingPairs(DocumentRoot(n)) +} + +// Child returns the value the parser reads under key in the mapping n stands +// for, its alias followed, or nil where it reads none: the value MappingPairs +// pairs key with, found without building the pairs +// (docs/parser-library.md#merge-keys, docs/parser-library.md#duplicate-key). +// n is dereferenced, a document node unwrapped first; a node that is no +// mapping holds nothing. +// +// It builds no view, and allocates nothing for a mapping that merges nothing, +// which is what lets every reader of a schema keyword ask it. Its work is the +// merge structure's, which the alias scans bound; the load phase reports a +// chain past MergeDepthLimit (bind.Table.Truncated). +func Child(n *yaml.Node, key string) *yaml.Node { + v, _ := childIn(Deref(DocumentRoot(n)), key, lastCopy, nil) + return v +} + +// childIn is Child for the mapping n, read at the copy rule names, which the +// mappings on path merge in, n's depth being len(path). Its own pair wins; +// else the first `<<` source holding key, in merge order, as flatten settles +// them. A source past MergeDepthLimit, or on path, which is a merge cycle, +// holds nothing, as for MappingPairs. found reports a pair whose value is nil. +func childIn(n *yaml.Node, key string, rule copyRule, path []*yaml.Node) (v *yaml.Node, found bool) { + if n == nil || n.Kind != yaml.MappingNode || len(path) > MergeDepthLimit || slices.Contains(path, n) { + return nil, false + } + merges := false + for i := 0; i+1 < len(n.Content); i += 2 { + if IsMergeKey(n.Content[i]) { + merges = true + continue + } + if k, own := KeyOf(n.Content[i]); own && k == key && (rule == lastCopy || !found) { + v, found = n.Content[i+1], true + } + } + if found || !merges { + return Deref(v), found + } + path = append(path, n) + for i := 0; i+1 < len(n.Content); i += 2 { + if !IsMergeKey(n.Content[i]) { + continue + } + if v, found = childOfSource(Deref(n.Content[i+1]), key, path); found { + return v, true + } + } + return nil, false +} + +// childOfSource is childIn over one `<<` value: a mapping, or a sequence of +// them in order, each read as a merge source reads, at its first copy. +func childOfSource(val *yaml.Node, key string, path []*yaml.Node) (*yaml.Node, bool) { + if val == nil || val.Kind != yaml.SequenceNode { + return childIn(val, key, firstCopy, path) + } + for _, item := range val.Content { + if v, found := childIn(Deref(item), key, firstCopy, path); found { + return v, true + } + } + return nil, false +} + +// WrittenChildren maps each key the mapping n writes itself to the value its +// text holds there: no `<<` merge read, an alias left an alias, each key as +// KeyOf names it and at its last copy. It answers where a document's text puts +// a node, which a read through merges and aliases (Child) cannot: a node +// merged or aliased into a mapping is written somewhere else. A node that is +// no mapping, an alias to one included, writes nothing. +func WrittenChildren(n *yaml.Node) map[string]*yaml.Node { + if n == nil || n.Kind != yaml.MappingNode { + return nil + } + out := make(map[string]*yaml.Node, len(n.Content)/2) + for i := 0; i+1 < len(n.Content); i += 2 { + if key, own := KeyOf(n.Content[i]); own { + out[key] = n.Content[i+1] + } + } + return out +} diff --git a/compilers/openapi/internal/nodeview/lookup_internal_test.go b/compilers/openapi/internal/nodeview/lookup_internal_test.go new file mode 100644 index 000000000..220eac669 --- /dev/null +++ b/compilers/openapi/internal/nodeview/lookup_internal_test.go @@ -0,0 +1,146 @@ +package nodeview + +import ( + "testing" + + "github.com/stretchr/testify/assert" + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/ynode" +) + +// TestChild_ReadsOnlyWhatAMappingHolds pins the inputs Child answers nil for +// and the shapes its one pass serves without a view: a node that is no +// mapping, a key nothing writes, a document node or an alias standing for the +// mapping, and a value written as an alias, which it hands back followed. +func TestChild_ReadsOnlyWhatAMappingHolds(t *testing.T) { + t.Parallel() + value := ynode.Scalar("v") + mapping := ynode.Map(ynode.Scalar("k"), ynode.Alias(value)) + + assert.Nil(t, Child(nil, "k"), "no node holds nothing") + assert.Nil(t, Child(ynode.Scalar("k"), "k"), "a scalar holds nothing") + assert.Nil(t, Child(ynode.Seq(ynode.Scalar("k")), "0"), "a sequence holds no key") + assert.Same(t, value, Child(&yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{mapping}}, "k"), + "a document node is unwrapped") + assert.Nil(t, Child(mapping, "absent")) + assert.Same(t, value, Child(mapping, "k"), "the value's alias is followed") + assert.Same(t, value, Child(ynode.Alias(mapping), "k"), "an alias standing for the mapping is followed") +} + +// TestChild_AnswersAsMappingPairs holds Child to the pair MappingPairs gives +// each key, on the shapes where a lookup could part from the expansion: a key +// written twice in the mapping (its last copy) and in a source (its first), a +// key two sources share (the first's), a list of sources, an own key holding +// nothing or an alias to nothing, a merge cycle, and a chain at MergeDepthLimit +// and one level past it, for every key either names and one neither writes. +// The cycle branches through a list, so a lookup following it would not end. +func TestChild_AnswersAsMappingPairs(t *testing.T) { + t.Parallel() + s := ynode.Scalar + source := ynode.Map(s("a"), s("src-a1"), s("a"), s("src-a2"), s("b"), s("src-b")) + other := ynode.Map(s("b"), s("other-b"), s("c"), s("other-c")) + cycle := ynode.Map(s("own"), s("v")) + inner := ynode.Map(ynode.Merge(), ynode.Seq(ynode.Alias(cycle), ynode.Alias(cycle)), s("in"), s("w")) + cycle.Content = append(cycle.Content, ynode.Merge(), ynode.Alias(inner)) + cases := map[string]*yaml.Node{ + "own copies": ynode.Map(s("k"), s("1"), s("k"), s("2")), + "a source's copies": ynode.Map(ynode.Merge(), ynode.Alias(source)), + "own over merged": ynode.Map(ynode.Merge(), ynode.Alias(source), s("a"), s("own-a")), + "two sources": ynode.Map(ynode.Merge(), ynode.Alias(source), ynode.Merge(), ynode.Alias(other)), + "a list of sources": ynode.Map(ynode.Merge(), ynode.Seq(ynode.Alias(other), ynode.Alias(source))), + "own nil over merge": ynode.Map(s("b"), &yaml.Node{Kind: yaml.AliasNode}, ynode.Merge(), ynode.Alias(source)), + "own nil value": ynode.Map(s("b"), nil, ynode.Merge(), ynode.Alias(source)), + "a cycle": cycle, + "at the bound": ynode.MergeChain(MergeDepthLimit), + "past the bound": ynode.MergeChain(MergeDepthLimit + 1), + } + for name, n := range cases { + pairs := New().MappingPairs(n) + asked := map[string]*yaml.Node{"absent": nil, "a": nil, "b": nil, "c": nil, "k": nil, "leaf": nil, "in": nil, "own": nil} + for _, p := range pairs { + asked[p.Key] = p.Val + } + for key, want := range asked { + assert.Same(t, want, Child(n, key), "%s: key %q", name, key) + } + } + assert.NotNil(t, Child(cases["at the bound"], "leaf"), "a chain at the bound is read whole") + assert.Nil(t, Child(cases["past the bound"], "leaf"), "one level past it is not") +} + +// TestPairs_IsAFreshViewsReading pins Pairs to MappingPairs read by a view of +// its own, merges included, a document node unwrapped, and to nothing for a +// node that is no mapping. +func TestPairs_IsAFreshViewsReading(t *testing.T) { + t.Parallel() + n := ynode.Map(ynode.Merge(), ynode.Map(ynode.Scalar("a"), ynode.Scalar("1")), ynode.Scalar("b"), ynode.Scalar("2")) + + assert.Equal(t, New().MappingPairs(n), Pairs(n)) + assert.Equal(t, []string{"a", "b"}, []string{Pairs(n)[0].Key, Pairs(n)[1].Key}) + assert.Equal(t, Pairs(n), Pairs(&yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{n}}), + "a document node is unwrapped") + assert.Nil(t, Pairs(ynode.Scalar("x"))) +} + +// TestWrittenChildren_ReadsTheTextNotThroughIt pins the reading that answers +// where a node is written: a mapping's own pairs, each value as written, an +// alias left an alias, nothing a `<<` merges in, each key as KeyOf names it at +// its last copy, and nothing for a node that is no mapping, an alias standing +// for one included. +func TestWrittenChildren_ReadsTheTextNotThroughIt(t *testing.T) { + t.Parallel() + target := ynode.Map(ynode.Scalar("m"), ynode.Scalar("merged")) + alias := ynode.Alias(target) + last := ynode.Scalar("last") + n := ynode.Map( + ynode.Scalar("a"), alias, + ynode.Merge(), ynode.Alias(target), + ynode.Alias(ynode.Scalar("k")), ynode.Scalar("first"), + ynode.Scalar("k"), last, + ) + + got := WrittenChildren(n) + + assert.Equal(t, map[string]*yaml.Node{"a": alias, "k": last}, got) + assert.Nil(t, WrittenChildren(nil)) + assert.Nil(t, WrittenChildren(ynode.Seq(ynode.Scalar("a")))) + assert.Nil(t, WrittenChildren(ynode.Alias(n)), "an alias is not where the mapping is written") +} + +// TestIsMergeKey_IsTheLibrarysPredicate pins IsMergeKey to the parsed `<<` +// and to nothing else: the same text untagged, an alias to it, or nil. +func TestIsMergeKey_IsTheLibrarysPredicate(t *testing.T) { + t.Parallel() + assert.True(t, IsMergeKey(ynode.Merge())) + assert.False(t, IsMergeKey(ynode.Scalar("<<")), "untagged, it is an ordinary key") + assert.False(t, IsMergeKey(ynode.Alias(ynode.Merge())), "an alias standing for it is an ordinary key") + assert.False(t, IsMergeKey(nil)) +} + +// TestRepeat_FindsAKeyWrittenTwiceWhereverTheViewReadsIt pins Repeat: a key +// the mapping writes twice, plainly or through an alias key, then one a merge +// source writes twice, in merge order, each named with the mapping writing +// it; a key a merge shares with the mapping is precedence, not a repeat. +func TestRepeat_FindsAKeyWrittenTwiceWhereverTheViewReadsIt(t *testing.T) { + t.Parallel() + src := "s1: &s1 {p: 1, p: 2}\ns2: &s2 {q: 1, q: 2}\n" + + "own: {b: 1, a: 1, b: 2, a: 2}\naliased: {&k k: 1, *k : 2}\n" + + "merged: {<<: [*s2, *s1]}\nboth: {<<: *s1, x: 1, x: 2}\nshared: {<<: {a: 1}, a: 2}\n" + root := yamlDoc(t, src) + v := New() + for _, tc := range []struct { + name, key, in string + }{ + {"own", "b", "own"}, {"aliased", "k", "aliased"}, {"merged", "q", "s2"}, {"both", "x", "both"}, + } { + key, in, ok := v.Repeat(Child(root, tc.name)) + assert.True(t, ok, tc.name) + assert.Equal(t, tc.key, key, tc.name) + assert.Same(t, Child(root, tc.in), in, tc.name) + } + _, _, ok := v.Repeat(Child(root, "shared")) + assert.False(t, ok, "a key a merge shares with the mapping is no repeat") + _, _, ok = v.Repeat(ynode.Scalar("x")) + assert.False(t, ok, "a node that is no mapping repeats nothing") +} diff --git a/compilers/openapi/internal/nodeview/model_internal_test.go b/compilers/openapi/internal/nodeview/model_internal_test.go new file mode 100644 index 000000000..2a2dccc03 --- /dev/null +++ b/compilers/openapi/internal/nodeview/model_internal_test.go @@ -0,0 +1,200 @@ +package nodeview + +import ( + "fmt" + "math/rand/v2" + "slices" + "strings" + "testing" + + soa "github.com/speakeasy-api/openapi/openapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" +) + +// TestMappingPairs_ReadsWhatTheModelReads holds MappingPairs and Child to the +// parser's model, which is what they exist to agree with +// (docs/parser-library.md#merge-keys, #duplicate-key, #non-scalar-key). Each +// case is a schema's properties, written after the anchors it merges: the +// model's property keys, in order, and the title of each one's schema are what +// the view must present. +func TestMappingPairs_ReadsWhatTheModelReads(t *testing.T) { + t.Parallel() + const anchors = " m0: &m0 {k: {title: m0k}, z: {title: m0z}}\n" + + " m1: &m1 {z: {title: m1z}, y: {title: m1y}}\n" + + " m2: &m2 {<<: *m0, w: {title: m2w}}\n" + + " dup: &dup {d: {title: first}, d: {title: second}}\n" + cases := []struct { + name, props string + want []string + }{ + {"merged pairs come first", "{a: {title: a}, <<: *m0}", []string{"k=m0k", "z=m0z", "a=a"}}, + {"an own key overrides a merged one", "{<<: *m0, k: {title: own}}", []string{"z=m0z", "k=own"}}, + {"an earlier source wins a shared key", "{<<: [*m0, *m1]}", []string{"k=m0k", "z=m0z", "y=m1y"}}, + {"every merge key merges, in order", "{<<: *m1, a: {title: a}, <<: *m0}", []string{"z=m1z", "y=m1y", "k=m0k", "a=a"}}, + {"a source's own merge comes first in it", "{<<: *m2}", []string{"k=m0k", "z=m0z", "w=m2w"}}, + {"an own key written twice is read at its last copy", "{d: {title: first}, b: {title: b}, d: {title: second}}", + []string{"b=b", "d=second"}}, + {"a source's key written twice is read at its first copy", "{<<: *dup}", []string{"d=first"}}, + {"a merge naming a scalar merges nothing", "{<<: 5, a: {title: a}}", []string{"a=a"}}, + {"a key that is no scalar is the empty string", "{? [x] : {title: list}, a: {title: a}}", + []string{"=list", "a=a"}}, + {"an alias key is the key it names", "{*ka : {title: aliased}, b: {title: b}}", []string{"a=aliased", "b=b"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + src := propertiesSource(anchors, tc.props) + assert.Equal(t, tc.want, modelPairs(t, src), "the model's reading") + assertViewReadsTheModel(t, src) + }) + } +} + +// FuzzMappingPairs_ReadsWhatTheModelReads holds the view to the model over +// properties built at random from the shapes the table above names: own keys, +// alias keys, keys repeated and keys that are no scalar, and `<<` keys naming +// an anchored mapping, a list of them, a mapping written in place or a scalar, +// anchors merging anchors before them. A library release that reads one shape +// otherwise reddens it. +func FuzzMappingPairs_ReadsWhatTheModelReads(f *testing.F) { + for seed := range uint64(16) { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, seed uint64) { + g := &propsGen{rng: rand.New(rand.NewPCG(seed, 799))} + assertViewReadsTheModel(t, propertiesSource(g.anchors(), g.mapping(true))) + }) +} + +// assertViewReadsTheModel requires MappingPairs to present the pairs the model +// holds for src's properties, in order, and Child to answer each key as they +// do and a key they hold none for with nil. +func assertViewReadsTheModel(t *testing.T, src string) { + t.Helper() + want := modelPairs(t, src) + props := propertiesNode(t, src) + pairs := New().MappingPairs(props) + got := make([]string, 0, len(pairs)) + for _, p := range pairs { + got = append(got, p.Key+"="+titleOf(p.Val)) + assert.Same(t, p.Val, Child(props, p.Key), "Child(%q) answers as the pairs do\n%s", p.Key, src) + } + require.Equal(t, want, got, "the view reads the model's pairs\n%s", src) + assert.Nil(t, Child(props, "absent"), "a key nothing writes holds nothing\n%s", src) +} + +// propertiesSource is a document whose schema S writes props as its +// properties, after anchors under x-anchors and the scalar anchors &ka..&ke +// an alias key names. +func propertiesSource(anchors, props string) string { + return "openapi: 3.1.0\ninfo: {title: t, version: '1'}\npaths: {}\n" + + "x-keys: [&ka a, &kb b, &kc c, &kd d, &ke e]\n" + + "x-anchors:\n none: {}\n" + anchors + + "components:\n schemas:\n S:\n properties: " + props + "\n" +} + +// modelPairs returns S's properties as the parser's model holds them, each as +// key=title, in order. +func modelPairs(t *testing.T, src string) []string { + t.Helper() + doc, _, err := soa.Unmarshal(t.Context(), strings.NewReader(src)) + require.NoError(t, err, src) + s, ok := doc.Components.Schemas.Get("S") + require.True(t, ok, src) + // Empty, not nil, where the model holds none, as the view's reading is. + out := make([]string, 0, s.GetSchema().GetProperties().Len()) + for key, prop := range s.GetSchema().GetProperties().All() { + out = append(out, key+"="+prop.GetSchema().GetTitle()) + } + return out +} + +// propertiesNode returns the raw properties mapping of S in src. +func propertiesNode(t *testing.T, src string) *yaml.Node { + t.Helper() + path, complete := New().DocumentPath(yamlDoc(t, src), "/components/schemas/S/properties") + require.True(t, complete, src) + return path[len(path)-1] +} + +// titleOf is the title a property's schema node writes, as the view reads it. +func titleOf(schema *yaml.Node) string { + if title := Child(schema, "title"); title != nil { + return title.Value + } + return "" +} + +// propsGen writes random properties mappings and the anchors they merge. Every +// title is unique, so a pair the view reads from the wrong copy or source +// shows in what it presents. +type propsGen struct { + rng *rand.Rand + titles int + named []string +} + +// title returns a property schema with a title no other one has. +func (g *propsGen) title() string { + g.titles++ + return fmt.Sprintf("{title: t%d}", g.titles) +} + +// anchors writes up to five anchored mappings, each of which may merge those +// before it, and returns them as x-anchors entries. +func (g *propsGen) anchors() string { + var b strings.Builder + for i := range g.rng.IntN(5) + 1 { + name := fmt.Sprintf("m%d", i) + fmt.Fprintf(&b, " %s: &%s %s\n", name, name, g.mapping(false)) + g.named = append(g.named, name) + } + return b.String() +} + +// mapping writes one flow mapping of up to six pairs. Only the properties +// themselves write a key that is no scalar: an anchored source writing one +// would merge a pair the view keys alike, adding nothing a case lacks. +func (g *propsGen) mapping(top bool) string { + pairs := make([]string, 0, 6) + for range g.rng.IntN(6) + 1 { + pairs = append(pairs, g.pair(top)) + } + return "{" + strings.Join(pairs, ", ") + "}" +} + +// pair writes one pair: an own key, plain or aliased, from a small alphabet so +// keys repeat, or a `<<` merge. +func (g *propsGen) pair(top bool) string { + keys := []string{"a", "b", "c", "d", "e"} + switch roll := g.rng.IntN(10); { + case roll < 4: + return keys[g.rng.IntN(len(keys))] + ": " + g.title() + case roll < 6: + return "*k" + keys[g.rng.IntN(len(keys))] + " : " + g.title() + case roll < 7 && top: + return "? [x] : " + g.title() + default: + return "<<: " + g.mergeValue() + } +} + +// mergeValue writes what a `<<` names: an anchor written before, a list of +// them, a mapping written in place, or a scalar, which merges nothing. +func (g *propsGen) mergeValue() string { + switch roll := g.rng.IntN(8); { + case roll < 3 && len(g.named) > 0: + return "*" + g.named[g.rng.IntN(len(g.named))] + case roll < 5 && len(g.named) > 0: + names := slices.Clone(g.named) + g.rng.Shuffle(len(names), func(i, j int) { names[i], names[j] = names[j], names[i] }) + names = names[:g.rng.IntN(len(names))+1] + return "[*" + strings.Join(names, ", *") + "]" + case roll < 7: + return "{" + []string{"a", "b", "c"}[g.rng.IntN(3)] + ": " + g.title() + "}" + default: + return "5" + } +} diff --git a/compilers/openapi/internal/nodeview/nodeview.go b/compilers/openapi/internal/nodeview/nodeview.go index 261e3b93a..64bd0f0cd 100644 --- a/compilers/openapi/internal/nodeview/nodeview.go +++ b/compilers/openapi/internal/nodeview/nodeview.go @@ -1,15 +1,19 @@ -// Package nodeview reads a YAML mapping the way the resolver will: through -// aliases, through `<<` merge keys, and with duplicate keys resolved the way the -// parser's model resolves them (docs/parser-library.md#duplicate-key). +// Package nodeview is the compiler's one reading of a raw YAML tree: a mapping +// read the way the parser reads it, through aliases and `<<` merge keys, each +// key and each key written twice settled as the parser's model settles them +// (docs/parser-library.md#merge-keys, #duplicate-key, #non-scalar-key), and +// one bounded, iterative walk of a tree as written (Walk). // -// It is separate from the scans that first needed it because the schema lowering -// needs the same view — a `$dynamicAnchor` lookup and an anchor walk both read -// mappings the raw yaml.Node tree does not present directly. A view over source -// text is neither a scan nor a lowering, so it sits below both. +// A reader that wants the parser's reading takes it from here, so no two +// disagree; one mirroring another reader on purpose, as load.treeReads +// mirrors GetTarget, says so. It is neither a scan nor a lowering, so it sits +// below both. package nodeview import ( + "cmp" "encoding/json/jsontext" + "slices" "strconv" "github.com/speakeasy-api/openapi/yml" @@ -39,8 +43,8 @@ const maxPointerSegments = 1024 // reader the bound stops reports it. const MergeDepthLimit = 64 -// maxCachedPairs bounds the total expanded pairs one View retains, about 50 MB -// at 2²¹ for the pairs themselves. MergeDepthLimit caps one mapping's expansion +// maxCachedPairs bounds the total expanded pairs one View retains, 48 MiB at +// 2²¹ for the pairs themselves. MergeDepthLimit caps one mapping's expansion // depth; this caps a document with many merged mappings. Past it the view still // answers correctly, only without memoizing. // @@ -67,7 +71,8 @@ func DocumentRoot(n *yaml.Node) *yaml.Node { } // Pair is one effective key/value pair of a mapping node, after alias and -// merge-key resolution. +// merge-key resolution: the key as KeyOf names it, and the value with its +// alias followed. type Pair struct { Key string Val *yaml.Node @@ -118,8 +123,8 @@ func New() *View { } // expansion is one mapping's effective pairs with what the memo needs to serve -// them again: how deep the expansion reached, and whether it reached -// everything. +// them again. pairs reads a key the mapping writes twice at its last copy; +// repeated, nil unless a key repeats, holds the rest (see repeated). // // height is the longest chain of `<<` merges beneath the node, 0 for a mapping // that merges nothing. The depth bound truncates from the entry point down, so @@ -127,15 +132,44 @@ func New() *View { // still fits (GitHub #404). type expansion struct { pairs []Pair + repeated *repeated height int complete bool } -// MappingPairs returns the effective pairs of a mapping node, following -// speakeasy's precedence: an explicit key beats one from a merge regardless of -// where the `<<` appears, an earlier merge source beats a later one on a -// shared key (yml.resolveMergeKeys), and a key repeated explicitly resolves to -// its last value. +// repeated is what an expansion holds for a mapping that, or a source of +// which, writes a key more than once, apart from the pairs every expansion +// holds: almost no mapping does, so the memo keeps it out of line. merged +// reads the mapping as a `<<` source does, at a repeated key's first copy as +// the library does, and is nil when only a source repeats. key is the first +// such key, and in the mapping writing it (View.Repeat). +type repeated struct { + merged []Pair + key string + in *yaml.Node +} + +// asSource returns the pairs e's mapping contributes to one that merges it. +func (e expansion) asSource() []Pair { + if e.repeated != nil && e.repeated.merged != nil { + return e.repeated.merged + } + return e.pairs +} + +// sourceRepeat returns e's repeat alone, as a mapping merging e's inherits it. +func (e expansion) sourceRepeat() *repeated { + if e.repeated == nil { + return nil + } + return &repeated{key: e.repeated.key, in: e.repeated.in} +} + +// MappingPairs returns the effective pairs of a mapping node in the order the +// parser's model holds them (yml.ResolveMergeKeys): first those its `<<` keys +// merge in, an earlier source winning a shared key and none overriding a key +// the mapping writes itself, then its own, a key written twice at its last +// copy. // // n is dereferenced, so an alias standing in for a whole mapping can be passed // directly; a non-mapping node (including nil) yields no pairs. The returned @@ -144,6 +178,20 @@ func (v *View) MappingPairs(n *yaml.Node) []Pair { return v.expand(Deref(n), 0).pairs } +// Repeat returns a key that the mapping n, or a mapping one of its `<<` keys +// merges in, writes more than once, with the mapping writing it, and whether +// there is one: n's own first, the one whose second copy comes first, then +// each source's in the order they merge. MappingPairs settles such a key by +// the library's rules, so a reader that must hold one copy faithfully, as a +// value does, asks this to refuse instead. n is dereferenced. +func (v *View) Repeat(n *yaml.Node) (key string, in *yaml.Node, ok bool) { + r := v.expand(Deref(n), 0).repeated + if r == nil { + return "", nil, false + } + return r.key, r.in, true +} + // expand returns n's expansion, incomplete if a merge cycle was broken or // MergeDepthLimit was reached. Only a complete expansion is memoized: caching // an incomplete one could make one traversal order lose a $ref another would @@ -213,18 +261,20 @@ func (v *View) isEntryPoint(depth int) bool { // memoization, so a miss recomputes exactly the same pairs — which makes the // budget a memory bound the scan can enforce without touching what it reports. func (v *View) memoize(n *yaml.Node, e expansion) { - if v.cachedPairs+len(e.pairs) > maxCachedPairs { + held := len(e.pairs) + if e.repeated != nil { + held += len(e.repeated.merged) + } + if v.cachedPairs+held > maxCachedPairs { return } v.pairs[n] = e - v.cachedPairs += len(e.pairs) + v.cachedPairs += held } // expandContent splits a mapping's raw content into the pairs it declares itself -// and the pairs its `<<` keys merge in, then applies the two precedence rules -// that govern them. They point in opposite directions, so they cannot share one -// pass: a repeated explicit key resolves to its last value, while a merged key -// yields to an explicit one and to any earlier merge source. +// and the pairs its `<<` keys merge in, then orders and settles them as +// yml.ResolveMergeKeys and the decoder do (see flatten). // // The height it records is one more than the tallest merge source's, so a // mapping that merges nothing has height 0. @@ -238,38 +288,138 @@ func (v *View) expandContent(n *yaml.Node, depth int) expansion { // disagree about which key merges. It is handed the key unresolved, as the // library hands it: an alias standing in for `<<` is an ordinary key. Every // `<<` merges, not only the last as in yaml.v3's decoder. - if yml.IsMergeKey(raw) { + if IsMergeKey(raw) { got := v.mergeSource(val, depth+1) merged = append(merged, got.pairs...) e.complete = e.complete && got.complete e.height = max(e.height, got.height+1) + e.repeated = cmp.Or(e.repeated, got.repeated) continue } key, own := KeyOf(raw) if !own { - continue // a non-scalar key cannot name a schema keyword + continue // a nil key, which no parse produces } explicit = append(explicit, Pair{Key: key, Val: val}) } - e.pairs = appendUnseen(dedupeLastWins(explicit), merged) + // The mapping's own repeat comes before any a merge source writes. + key, repeats := firstRepeat(explicit) + if !repeats && len(merged) == 0 { + // Most mappings merge nothing and write each key once: their own pairs + // are already the answer, in order, and nothing else is allocated. + e.pairs = explicit + return e + } + var asSource []Pair + e.pairs, asSource = flatten(explicit, merged) + if repeats { + e.repeated = &repeated{merged: asSource, key: key, in: n} + } return e } +// smallKeys is the width below which a mapping's keys are compared pair by +// pair rather than hashed: a document holds a mapping per object, most with a +// handful of keys, and a map for each is what a read of a large one spends on. +const smallKeys = 16 + +// firstRepeat returns the key a mapping's own pairs write twice whose second +// copy comes first, and whether there is one. +func firstRepeat(own []Pair) (string, bool) { + if len(own) <= smallKeys { + for i := 1; i < len(own); i++ { + for _, p := range own[:i] { + if p.Key == own[i].Key { + return own[i].Key, true + } + } + } + return "", false + } + seen := make(map[string]bool, len(own)) + for _, p := range own { + if seen[p.Key] { + return p.Key, true + } + seen[p.Key] = true + } + return "", false +} + +// flatten orders a mapping's pairs as the library does: the merged pairs no +// pair of the mapping's own overrides, the first source of each key winning, +// then its own pairs (docs/parser-library.md#merge-keys). It returns them +// twice, read as the decoder reads the mapping itself, a key written twice at +// its last copy (docs/parser-library.md#duplicate-key), and read as a merge +// into another mapping, which keeps the first. The second is nil when no key +// is written twice. +func flatten(explicit, merged []Pair) (own, asSource []Pair) { + written := make(map[string]int, len(explicit)) + for _, p := range explicit { + written[p.Key]++ + } + base := make([]Pair, 0, len(merged)) + taken := make(map[string]bool, len(merged)) + for _, p := range merged { + if written[p.Key] > 0 || taken[p.Key] { + continue + } + taken[p.Key] = true + base = append(base, p) + } + own = slices.Concat(base, keepCopy(explicit, written, lastCopy)) + if len(written) == len(explicit) { + return own, nil + } + return own, slices.Concat(base, keepCopy(explicit, written, firstCopy)) +} + +// copyRule says which copy of a key a mapping writes several times is read. +type copyRule bool + +const ( + firstCopy copyRule = false + lastCopy copyRule = true +) + +// keepCopy returns pairs with each key once, at the copy rule names, in the +// order those copies are written. written counts each key's copies. +func keepCopy(pairs []Pair, written map[string]int, rule copyRule) []Pair { + seen := make(map[string]int, len(written)) + out := make([]Pair, 0, len(written)) + for _, p := range pairs { + seen[p.Key]++ + if rule == firstCopy && seen[p.Key] == 1 || rule == lastCopy && seen[p.Key] == written[p.Key] { + out = append(out, p) + } + } + return out +} + +// IsMergeKey reports whether raw, a mapping key node as written, is a `<<` +// merge key, by the library's own predicate (yml.IsMergeKey). An alias +// standing in for `<<` is an ordinary key. Every reader of the parser's +// reading of a mapping asks this or KeyOf, never a test of its own. +func IsMergeKey(raw *yaml.Node) bool { return yml.IsMergeKey(raw) } + // KeyOf returns the key the parser gives the mapping pair whose raw key node is // raw, and whether the pair is one of the mapping's own: a `<<` merge key -// names none, nor does a key that is no scalar once its alias is followed. The -// view and every reader of a mapping's raw pairs ask it, so none can disagree -// with the parser about a pair's key. +// names none. A key that is no scalar once its alias is followed is read as +// the empty string, as the parser reads it +// (docs/parser-library.md#non-scalar-key), and oas.CheckFormat reports it. The +// view and every reader of a mapping's raw pairs ask KeyOf, so none can +// disagree with the parser about a pair's key. func KeyOf(raw *yaml.Node) (string, bool) { - if yml.IsMergeKey(raw) { + if raw == nil || IsMergeKey(raw) { return "", false } - key := Deref(raw) - if key == nil || key.Kind != yaml.ScalarNode { - return "", false + if key := Deref(raw); key != nil { + return key.Value, true } - return key.Value, true + // An alias naming nothing, which no parse produces, reads as its own text, + // as the library reads it. + return raw.Value, true } // RepeatedKeys returns the keys a mapping writes more than once, each once, in @@ -296,75 +446,30 @@ func RepeatedKeys(n *yaml.Node) []string { return out } -// dedupeLastWins keeps the last pair for each key, at that last occurrence's -// position: a mapping that repeats a key is ill-formed, and the parser's model -// decodes its last occurrence, with only a warning -// (docs/parser-library.md#duplicate-key). -func dedupeLastWins(pairs []Pair) []Pair { - last := make(map[string]int, len(pairs)) - for i, p := range pairs { - last[p.Key] = i - } - out := make([]Pair, 0, len(last)) - for i, p := range pairs { - if last[p.Key] == i { - out = append(out, p) - } - } - return out -} - -// appendUnseen appends the pairs of add whose key is not already present, -// keeping the first contributor of each — the rule for merged keys, which yield -// both to an explicit key and to an earlier merge source. -func appendUnseen(base, add []Pair) []Pair { - if len(add) == 0 { - return base - } - seenKey := make(map[string]bool, len(base)+len(add)) - for _, p := range base { - seenKey[p.Key] = true - } - out := base - for _, p := range add { - if seenKey[p.Key] { - continue - } - seenKey[p.Key] = true - out = append(out, p) - } - return out -} - -// mergeSource expands one `<<` value into the pairs it contributes: a mapping is -// a single merge source, a sequence is several with an earlier element taking -// precedence over a later one on a shared key. +// mergeSource expands one `<<` value into the pairs it contributes, in the +// order flatten settles them: a mapping is a single merge source, and a +// sequence is several, its mapping elements in order. Anything else merges +// nothing, as for the library (docs/parser-library.md#merge-keys), and +// oas.CheckFormat reports it. // // The height of a sequence is its tallest element's: the elements are // alternatives at one level, not links in a chain. func (v *View) mergeSource(val *yaml.Node, depth int) expansion { if val == nil || val.Kind != yaml.SequenceNode { - return v.expand(val, depth) + got := v.expand(val, depth) + return expansion{pairs: got.asSource(), repeated: got.sourceRepeat(), height: got.height, complete: got.complete} } - var out []Pair e := expansion{complete: true} for _, item := range val.Content { got := v.expand(Deref(item), depth) - out = append(out, got.pairs...) + e.pairs = append(e.pairs, got.asSource()...) e.complete = e.complete && got.complete e.height = max(e.height, got.height) + e.repeated = cmp.Or(e.repeated, got.sourceRepeat()) } - e.pairs = dedupeFirstWins(out) return e } -// dedupeFirstWins keeps only the first pair for each key, preserving order — the -// rule across the elements of a merge sequence, where an earlier source outranks -// a later one. -func dedupeFirstWins(pairs []Pair) []Pair { - return appendUnseen(nil, pairs) -} - // DocumentPath walks pointer, a position in this document, from root as RFC // 6901 reads it, returning the root and each node reached, aliases // dereferenced. complete reports whether every token was followed; if not, the diff --git a/compilers/openapi/internal/nodeview/nodeview_internal_test.go b/compilers/openapi/internal/nodeview/nodeview_internal_test.go index ac58d6a11..f059336c9 100644 --- a/compilers/openapi/internal/nodeview/nodeview_internal_test.go +++ b/compilers/openapi/internal/nodeview/nodeview_internal_test.go @@ -252,18 +252,24 @@ func TestMappingPairs_Cases(t *testing.T) { map[string]string{"k": "v"}, }, { - "non-scalar key after Deref is skipped", + "non-scalar key is read as the empty string", ynode.Map( - ynode.Map(ynode.Scalar("x"), ynode.Scalar("1")), ynode.Scalar("ignored"), + ynode.Map(ynode.Scalar("x"), ynode.Scalar("1")), ynode.Scalar("empty"), ynode.Scalar("real"), ynode.Scalar("kept"), ), - map[string]string{"real": "kept"}, + map[string]string{"": "empty", "real": "kept"}, }, { - "key aliasing a nil target is skipped", - ynode.Map(ynode.Alias(nil), ynode.Scalar("ignored"), ynode.Scalar("real"), ynode.Scalar("kept")), + "a nil key, which no parse produces, is skipped", + ynode.Map(nil, ynode.Scalar("ignored"), ynode.Scalar("real"), ynode.Scalar("kept")), map[string]string{"real": "kept"}, }, + { + "key aliasing a nil target is read as its own text", + ynode.Map(&yaml.Node{Kind: yaml.AliasNode, Value: "anchor"}, ynode.Scalar("named"), + ynode.Scalar("real"), ynode.Scalar("kept")), + map[string]string{"anchor": "named", "real": "kept"}, + }, { "odd trailing key without a value is ignored", trailingKeyNode(), @@ -407,9 +413,9 @@ func TestMappingPairs_MergeKeySources(t *testing.T) { want: map[string]string{"a": "1", "b": "2"}, }, { - name: "a non-scalar key names no keyword and is skipped", + name: "a non-scalar key is read as the empty string", src: "use:\n ? [a, b]\n : v\n k: 1\n", - want: map[string]string{"k": "1"}, + want: map[string]string{"": "v", "k": "1"}, }, } for _, tc := range tests { @@ -932,13 +938,12 @@ func TestKeyOf_KeysAPairAsTheParserDoes(t *testing.T) { } } -// TestKeyOf_NamesNoKeyForAKeyThatIsNoScalar pins a departure from the parser, +// TestKeyOf_ReadsAKeyThatIsNoScalarAsTheEmptyString holds KeyOf to the parser, // which keys a pair whose key is no scalar, aliased or not, with the empty -// string. KeyOf names none, so the view does not see the entry the parser reads -// under that key. The parser assertion pins its reading, so a release that stops -// giving those pairs the empty key is noticed. No node at all is the one row the -// parser cannot be asked, since it never hands one back. -func TestKeyOf_NamesNoKeyForAKeyThatIsNoScalar(t *testing.T) { +// string (docs/parser-library.md#non-scalar-key), so the view reads the entry +// the parser reads. No node at all is the one row the parser cannot be asked, +// since it never hands one back, and it names no key. +func TestKeyOf_ReadsAKeyThatIsNoScalarAsTheEmptyString(t *testing.T) { t.Parallel() tests := []struct { name, pair string @@ -956,7 +961,7 @@ func TestKeyOf_NamesNoKeyForAKeyThatIsNoScalar(t *testing.T) { key, own := KeyOf(raw) assert.Empty(t, key) - assert.False(t, own) + assert.True(t, own) }) } @@ -978,7 +983,7 @@ func TestRepeatedKeys_NamesEachRepeatedKeyOnce(t *testing.T) { {"two keys repeated", "{a: 1, b: 1, b: 2, a: 2}", []string{"b", "a"}}, {"a key shared with a merge is no repeat", "{<<: {a: 1}, a: 2}", nil}, {"a merge key twice is no repeat", "{<<: {a: 1}, <<: {b: 1}}", nil}, - {"a key that is no scalar names nothing", "{? [x] : 1, ? [x] : 2}", nil}, + {"two keys that are no scalar both read as the empty string", "{? [x] : 1, ? [y] : 2}", []string{""}}, {"a sequence repeats nothing", "[a, a]", nil}, {"a scalar repeats nothing", "a", nil}, } diff --git a/compilers/openapi/internal/nodeview/testdata/fuzz/FuzzMappingPairs_ReadsWhatTheModelReads/2f78d534dbf8d081 b/compilers/openapi/internal/nodeview/testdata/fuzz/FuzzMappingPairs_ReadsWhatTheModelReads/2f78d534dbf8d081 new file mode 100644 index 000000000..ecfe91f3b --- /dev/null +++ b/compilers/openapi/internal/nodeview/testdata/fuzz/FuzzMappingPairs_ReadsWhatTheModelReads/2f78d534dbf8d081 @@ -0,0 +1,2 @@ +go test fuzz v1 +uint64(696) diff --git a/compilers/openapi/internal/nodeview/walk.go b/compilers/openapi/internal/nodeview/walk.go new file mode 100644 index 000000000..8f03c330e --- /dev/null +++ b/compilers/openapi/internal/nodeview/walk.go @@ -0,0 +1,148 @@ +package nodeview + +import ( + "slices" + + yaml "gopkg.in/yaml.v3" +) + +// Step is what a Walk's visitor asks of the node it was handed. +type Step uint8 + +const ( + // Descend visits the node's children next. + Descend Step = iota + // Prune leaves the node's children unvisited. + Prune + // Stop ends the walk. + Stop +) + +// Aliases says what a Walk takes an alias node for. +type Aliases uint8 + +const ( + // AliasLeaf takes an alias for a node of its own with no children: what it + // names is written elsewhere in the tree and visited there, so the walk + // visits each node written once. + AliasLeaf Aliases = iota + // AliasTarget visits what an alias names in its place, every time, as the + // parser's model builds a copy of it at each alias. An alias whose target + // is nil is visited as a leaf. + AliasTarget +) + +// Walk is a bounded, iterative walk over a raw YAML tree as written, through +// Content and in document order, visiting each node before its children. A +// reader that visits every node written runs one rather than keeping a stack +// and a bound of its own, so a cut short walk is reported rather than silent +// wherever one runs. A walk addressing positions as the parser reads them, +// where a merged pair's value is no child of the mapping merging it, keeps a +// frame per position instead. +type Walk struct { + // Nodes bounds how many nodes the walk visits. A walk that would visit + // more stops there and reports itself Cut; a bound below one visits none. + Nodes int + // Depth, when above zero, bounds how deep the walk visits, the root at + // depth zero: a node deeper is not visited, and the walk is Cut. Nodes + // bounds the depth either way. + Depth int + // Aliases says what an alias stands for. + Aliases Aliases + // Visit is handed each node and its depth, and says whether to descend. + // Unset, the walk descends everywhere, which counts a tree's nodes. + Visit func(n *yaml.Node, depth int) Step + // Leave, when set, is handed each node Visit descended into once its + // children have all been visited, or at once for a node holding none. + Leave func(n *yaml.Node, depth int) +} + +// Outcome is how a walk ended. +type Outcome struct { + // Visited counts the nodes handed to Visit. + Visited int + // Cut reports that a bound left nodes unvisited, so whatever the walk + // answers is partial. + Cut bool + // Stopped reports that Visit ended the walk. + Stopped bool +} + +// walkFrame is one entry of a walk's stack: a node to visit, or the leaving of +// one whose children have been pushed. +type walkFrame struct { + n *yaml.Node + depth int + leave bool +} + +// Run walks the tree under root. A nil root, or a nil node in a Content list, +// which no parse produces, is not visited. +func (w Walk) Run(root *yaml.Node) Outcome { + var out Outcome + stack := []walkFrame{{n: root}} + for len(stack) > 0 { + f := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if f.leave { + w.Leave(f.n, f.depth) + continue + } + n := w.node(f.n) + if n == nil { + continue + } + if out.Visited >= w.Nodes { + out.Cut = true + return out + } + if w.Depth > 0 && f.depth > w.Depth { + out.Cut = true + continue + } + out.Visited++ + switch w.step(n, f.depth) { + case Stop: + out.Stopped = true + return out + case Prune: + continue + case Descend: + } + stack = w.push(stack, n, f.depth) + } + return out +} + +// step asks Visit what to do with n, descending where no visitor is set. +func (w Walk) step(n *yaml.Node, depth int) Step { + if w.Visit == nil { + return Descend + } + return w.Visit(n, depth) +} + +// node returns what the walk visits for n: n itself, or the node an alias +// names where aliases stand for their targets. +func (w Walk) node(n *yaml.Node) *yaml.Node { + if w.Aliases != AliasTarget { + return n + } + if target := Deref(n); target != nil { + return target + } + return n +} + +// push returns stack with the leaving of n and then its children pushed, so +// the first child is visited next. An alias visited as a leaf holds no +// children: yaml.v3 keeps its target in Alias, not in Content. +func (w Walk) push(stack []walkFrame, n *yaml.Node, depth int) []walkFrame { + if w.Leave != nil { + stack = append(stack, walkFrame{n: n, depth: depth, leave: true}) + } + for _, c := range slices.Backward(n.Content) { + stack = append(stack, walkFrame{n: c, depth: depth + 1}) + } + return stack +} diff --git a/compilers/openapi/internal/nodeview/walk_internal_test.go b/compilers/openapi/internal/nodeview/walk_internal_test.go new file mode 100644 index 000000000..778d6fe9e --- /dev/null +++ b/compilers/openapi/internal/nodeview/walk_internal_test.go @@ -0,0 +1,151 @@ +package nodeview + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/ynode" +) + +// trace records a walk: each visit as the node's value at its depth, and each +// leave marked with a slash. +type trace []string + +func (tr *trace) visit(step Step) func(n *yaml.Node, depth int) Step { + return func(n *yaml.Node, depth int) Step { + *tr = append(*tr, strings.Repeat(".", depth)+label(n)) + return step + } +} + +func (tr *trace) leave(n *yaml.Node, depth int) { + *tr = append(*tr, strings.Repeat(".", depth)+"/"+label(n)) +} + +// label names a node for a trace: a scalar by its value, anything else by its +// kind. +func label(n *yaml.Node) string { + switch n.Kind { + case yaml.ScalarNode: + return n.Value + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "seq" + default: + return "alias" + } +} + +// TestWalk_VisitsInDocumentOrderAndLeavesAfterChildren pins the order: each +// node before its children, which come in Content order, and each node's leave +// once its children are done, a node holding none left at once. +func TestWalk_VisitsInDocumentOrderAndLeavesAfterChildren(t *testing.T) { + t.Parallel() + root := ynode.Map(ynode.Scalar("k"), ynode.Seq(ynode.Scalar("a"), ynode.Scalar("b"))) + var tr trace + + out := Walk{Nodes: 10, Visit: tr.visit(Descend), Leave: tr.leave}.Run(root) + + assert.Equal(t, trace{"map", ".k", "./k", ".seq", "..a", "../a", "..b", "../b", "./seq", "/map"}, tr) + assert.Equal(t, Outcome{Visited: 5}, out) +} + +// TestWalk_PruneAndStop pins the two answers that cut a walk short on purpose: +// a pruned node's children go unvisited and it is never left, and a stop ends +// the walk there, reported as Stopped rather than Cut. +func TestWalk_PruneAndStop(t *testing.T) { + t.Parallel() + root := ynode.Seq(ynode.Seq(ynode.Scalar("hidden")), ynode.Scalar("after")) + + var pruned trace + out := Walk{Nodes: 10, Leave: pruned.leave, Visit: func(n *yaml.Node, depth int) Step { + pruned.visit(Descend)(n, depth) + if depth == 1 && n.Kind == yaml.SequenceNode { + return Prune + } + return Descend + }}.Run(root) + assert.Equal(t, trace{"seq", ".seq", ".after", "./after", "/seq"}, pruned) + assert.Equal(t, Outcome{Visited: 3}, out) + + var stopped trace + out = Walk{Nodes: 10, Visit: stopped.visit(Stop)}.Run(root) + assert.Equal(t, trace{"seq"}, stopped) + assert.Equal(t, Outcome{Visited: 1, Stopped: true}, out) +} + +// TestWalk_BoundsAreReportedNotSilent pins the two bounds: a tree of exactly +// Nodes nodes is walked whole, one node more is Cut there, and a bound below +// one visits none; a node past Depth is not visited, its siblings are, and the +// walk says it was Cut. +func TestWalk_BoundsAreReportedNotSilent(t *testing.T) { + t.Parallel() + root := ynode.Seq(ynode.Seq(ynode.Scalar("deep")), ynode.Scalar("shallow")) + + assert.Equal(t, Outcome{Visited: 4}, Walk{Nodes: 4}.Run(root)) + assert.Equal(t, Outcome{Visited: 3, Cut: true}, Walk{Nodes: 3}.Run(root)) + assert.Equal(t, Outcome{Cut: true}, Walk{Nodes: 0}.Run(root)) + + var tr trace + out := Walk{Nodes: 10, Depth: 1, Visit: tr.visit(Descend)}.Run(root) + assert.Equal(t, trace{"seq", ".seq", ".shallow"}, tr) + assert.Equal(t, Outcome{Visited: 3, Cut: true}, out) +} + +// TestWalk_AnAliasIsALeafOrItsTarget pins the alias rule: as a leaf, an alias +// is visited as itself and what it names only where that is written; standing +// for its target, the target is visited in its place each time, and an alias +// naming nothing is visited as itself. +func TestWalk_AnAliasIsALeafOrItsTarget(t *testing.T) { + t.Parallel() + target := ynode.Seq(ynode.Scalar("x")) + root := ynode.Seq(target, ynode.Alias(target), ynode.Alias(nil)) + + var leaf trace + Walk{Nodes: 10, Visit: leaf.visit(Descend)}.Run(root) + assert.Equal(t, trace{"seq", ".seq", "..x", ".alias", ".alias"}, leaf) + + var copies trace + Walk{Nodes: 10, Aliases: AliasTarget, Visit: copies.visit(Descend)}.Run(root) + assert.Equal(t, trace{"seq", ".seq", "..x", ".seq", "..x", ".alias"}, copies) +} + +// TestWalk_SkipsWhatNoParseProduces pins the walk on trees built by hand: a +// nil root and a nil child are not visited, and an unset Visit descends +// everywhere, which counts the nodes. +func TestWalk_SkipsWhatNoParseProduces(t *testing.T) { + t.Parallel() + assert.Equal(t, Outcome{}, Walk{Nodes: 10}.Run(nil)) + assert.Equal(t, Outcome{Visited: 2}, Walk{Nodes: 10}.Run(ynode.Map(ynode.Scalar("a"), nil))) +} + +// TestWalk_DeepAndWideTreesAreIterative holds the walk to the inputs a reader +// of untrusted text meets: thousands of levels, which a recursive walk would +// meet one stack frame at a time, and a mapping wide enough that a walk +// re-reading it per node would show. Both are walked whole, at their depth. +func TestWalk_DeepAndWideTreesAreIterative(t *testing.T) { + t.Parallel() + const levels = 100_000 + deep := ynode.Scalar("leaf") + for range levels { + deep = ynode.Seq(deep) + } + deepest := 0 + out := Walk{Nodes: levels + 1, Visit: func(_ *yaml.Node, depth int) Step { + deepest = max(deepest, depth) + return Descend + }}.Run(deep) + require.Equal(t, Outcome{Visited: levels + 1}, out) + assert.Equal(t, levels, deepest) + + wide := ynode.Map() + for range levels { + wide.Content = append(wide.Content, ynode.Scalar("k"), ynode.Scalar("v")) + } + assert.Equal(t, Outcome{Visited: 2*levels + 1}, Walk{Nodes: 2*levels + 1}.Run(wide)) +} diff --git a/compilers/openapi/internal/oas/mappings.go b/compilers/openapi/internal/oas/mappings.go new file mode 100644 index 000000000..478614e24 --- /dev/null +++ b/compilers/openapi/internal/oas/mappings.go @@ -0,0 +1,169 @@ +package oas + +import ( + "encoding/json/jsontext" + "slices" + "strconv" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" + "github.com/dexpace/morphic/ir" +) + +// CheckFormat holds every mapping of the tree root to the Format section's +// rule that a key is a string, at the minor declared names, as Check holds +// objects. It reports each key that is no scalar, which the parser reads as +// "" (docs/parser-library.md#non-scalar-key), and each `<<` naming no +// mapping, which merges nothing (docs/parser-library.md#merge-keys). A rule +// judging a key's spelling still judges that "" (GitHub #1108). +// +// It reads the tree as written, values and extensions included, so a node an +// alias mounts twice draws one finding, placed at its pointer as the parser +// reads the tree. It reads Locate, Budget, ReadAt and Root of opts. +func CheckFormat(root *yaml.Node, declared string, opts Options) []ir.Diagnostic { + minor, ok := ParseMinor(declared) + if !ok { + return nil + } + f := formatCheck{minor: minor, at: map[*yaml.Node]jsontext.Pointer{}, merged: map[*yaml.Node]bool{}, + locate: opts.Locate, declared: declared, readAt: opts.ReadAt, rootAt: opts.Root, said: map[ir.Diagnostic]bool{}} + if f.locate == nil { + f.locate = func(jsontext.Pointer) ir.Provenance { return ir.Provenance{} } + } + budget := opts.Budget + if budget <= 0 { + budget = DefaultBudget + } + // Placing a finding costs a pointer per node, and most documents break + // neither rule, so a first walk only looks, and stops at the first break. + look := nodeview.Walk{Nodes: budget, Visit: lookForBreak}.Run(nodeview.DocumentRoot(root)) + if !look.Stopped && !look.Cut { + return nil + } + out := nodeview.Walk{Nodes: budget, Visit: f.visit}.Run(nodeview.DocumentRoot(root)) + if out.Cut { + f.diags = append(f.diags, diag.Newf(ir.SeverityError, diag.BudgetExceeded, f.locate(""), + "the OpenAPI %s check of the document's keys stopped at its work budget; what lies past it is unchecked", minor)) + } + return f.diags +} + +// formatCheck is one CheckFormat in progress: the pointer each node to visit is +// at, and the lists a `<<` names, whose mappings merge where it is written. +type formatCheck struct { + minor Minor + at map[*yaml.Node]jsontext.Pointer + merged map[*yaml.Node]bool + reporter +} + +// visit checks n, a mapping's pairs or a merged list's elements, and notes +// where each of its children is. +func (f *formatCheck) visit(n *yaml.Node, _ int) nodeview.Step { + ptr := f.at[n] + delete(f.at, n) + switch n.Kind { + case yaml.MappingNode: + f.mapping(n, ptr) + case yaml.SequenceNode: + for i, item := range n.Content { + f.at[item] = ptr.AppendToken(strconv.Itoa(i)) + if f.merged[n] { + f.at[item] = ptr + } + } + delete(f.merged, n) + default: + } + return nodeview.Descend +} + +// lookForBreak stops a walk at the first mapping that breaks either rule +// CheckFormat holds it to. +func lookForBreak(n *yaml.Node, _ int) nodeview.Step { + for i := 0; n.Kind == yaml.MappingNode && i+1 < len(n.Content); i += 2 { + key, val := n.Content[i], n.Content[i+1] + if nodeview.IsMergeKey(key) { + if slices.ContainsFunc(mergeSources(val), notMapping) { + return nodeview.Stop + } + } else if !scalarKey(key) { + return nodeview.Stop + } + } + return nodeview.Descend +} + +// scalarKey reports whether key, followed through its alias, is a scalar, as +// the Format section requires a key to be. +func scalarKey(key *yaml.Node) bool { + k := nodeview.Deref(key) + return k == nil || k.Kind == yaml.ScalarNode +} + +// mergeSources returns what the `<<` value val names: each element of a list +// written or aliased there, or val itself. +func mergeSources(val *yaml.Node) []*yaml.Node { + if v := nodeview.Deref(val); v != nil && v.Kind == yaml.SequenceNode { + return v.Content + } + return []*yaml.Node{val} +} + +// notMapping reports whether src, followed through its alias, is no mapping, +// so a `<<` naming it merges nothing. +func notMapping(src *yaml.Node) bool { + s := nodeview.Deref(src) + return s == nil || s.Kind != yaml.MappingNode +} + +// mapping checks the pairs mapping n, at ptr, writes itself. +func (f *formatCheck) mapping(n *yaml.Node, ptr jsontext.Pointer) { + for i := 0; i+1 < len(n.Content); i += 2 { + key, val := n.Content[i], n.Content[i+1] + if nodeview.IsMergeKey(key) { + f.merge(val, ptr) + continue + } + name, _ := nodeview.KeyOf(key) + f.at[key], f.at[val] = ptr.AppendToken(name), ptr.AppendToken(name) + if !scalarKey(key) { + f.report(ir.SeverityError, classType, ptr.AppendToken(name), + "a mapping key must be a string in OpenAPI %s (Format), and this one is %s; it is read as the empty string", + f.minor, kindName(nodeview.Deref(key))) + } + } +} + +// merge checks the value of a `<<` key in the mapping at ptr, which the +// parser reads the members of as that mapping's own: a source, or a list of +// them. A list written in place is noted, so visit places its elements at ptr. +func (f *formatCheck) merge(val *yaml.Node, ptr jsontext.Pointer) { + f.at[val] = ptr + for _, src := range mergeSources(val) { + if notMapping(src) { + f.report(ir.SeverityError, classType, ptr, + "a `<<` merge key must name a mapping or a list of mappings, and this one names %s; it merges nothing", + kindName(nodeview.Deref(src))) + } + } + if v := nodeview.Deref(val); v != nil && v == val && v.Kind == yaml.SequenceNode { + f.merged[val] = true + } +} + +// kindName names the kind of a node with its article. +func kindName(n *yaml.Node) string { + switch { + case n == nil: + return "nothing" + case n.Kind == yaml.MappingNode: + return "a mapping" + case n.Kind == yaml.SequenceNode: + return "a list" + default: + return "a scalar" + } +} diff --git a/compilers/openapi/internal/oas/mappings_test.go b/compilers/openapi/internal/oas/mappings_test.go new file mode 100644 index 000000000..f5522f90a --- /dev/null +++ b/compilers/openapi/internal/oas/mappings_test.go @@ -0,0 +1,111 @@ +package oas_test + +import ( + "encoding/json/jsontext" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/oas" + "github.com/dexpace/morphic/ir" +) + +// formatFindings runs CheckFormat over doc, declaring version, each finding +// placed at the pointer it was read at. +func formatFindings(t *testing.T, version, doc string, budget int) []finding { + t.Helper() + var n yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(doc), &n)) + diags := oas.CheckFormat(&n, version, oas.Options{ + Budget: budget, + Locate: func(p jsontext.Pointer) ir.Provenance { return ir.Provenance{Pointer: p} }, + }) + out := make([]finding, 0, len(diags)) + for _, d := range diags { + out = append(out, finding{Code: d.Code, At: string(d.Provenance.Pointer)}) + } + sortFindings(out) + return out +} + +// TestCheckFormat_AKeyIsAString pins the Format section's rule on keys over +// the tree as written, objects, values and extensions alike: a key that is no +// scalar, aliased or not, is placed at the pointer the parser reads it at, +// under the empty key; a `<<` naming no mapping, by itself or in a list, is +// placed at the mapping it merges into; a pair a merge writes in place is the +// merging mapping's. A node an alias mounts twice is reported once. +func TestCheckFormat_AKeyIsAString(t *testing.T) { + t.Parallel() + cases := []struct { + name, doc string + want []finding + }{ + {"a well-formed document draws nothing", + "a: {b: 1}\nm: &m {k: v}\nn: {<<: *m}\nl: {<<: [*m, *m]}\n", []finding{}}, + {"a mapping key", "x-e:\n ? {k: v}\n : 1\n", []finding{at(mismatch, "/x-e/")}}, + {"a list key", "x-e:\n ? [k]\n : 1\n", []finding{at(mismatch, "/x-e/")}}, + {"an alias to a mapping as a key", "m: &m {k: v}\nx-e: {*m : 1}\n", []finding{at(mismatch, "/x-e/")}}, + {"a merge of a scalar", "x-e: {<<: 5}\n", []finding{at(mismatch, "/x-e")}}, + {"a merge of an alias to a scalar", "s: &s five\nx-e: {<<: *s}\n", []finding{at(mismatch, "/x-e")}}, + {"a scalar in a merge list", "m: &m {k: v}\nx-e: {<<: [*m, 5]}\n", []finding{at(mismatch, "/x-e")}}, + {"a scalar in an aliased merge list", "m: &m {k: v}\nl: &l [*m, 5]\nx-e: {<<: *l}\n", + []finding{at(mismatch, "/x-e")}}, + {"a key no scalar in a merged mapping is the merging mapping's", + "x-e: {<<: {? [k] : 1}}\n", []finding{at(mismatch, "/x-e/")}}, + {"in a merged list too", "x-e: {<<: [{? [k] : 1}]}\n", []finding{at(mismatch, "/x-e/")}}, + {"a node an alias mounts twice is reported once", "m: &m {? [k] : 1}\na: *m\nb: *m\n", + []finding{at(mismatch, "/m/")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := formatFindings(t, "3.1.0", tc.doc, 0) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("findings (-want +got):\n%s", diff) + } + }) + } +} + +// TestCheckFormat_SaysWhatItRead pins the messages: the minor and the section +// the rule comes from, what the key or the merge names, and what the parser +// makes of it. +func TestCheckFormat_SaysWhatItRead(t *testing.T) { + t.Parallel() + var n yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("a: {? [k] : 1, <<: 5}\n"), &n)) + diags := oas.CheckFormat(&n, "3.0.3", oas.Options{}) + require.Len(t, diags, 2) + assert.Equal(t, ir.SeverityError, diags[0].Severity) + assert.Equal(t, "a mapping key must be a string in OpenAPI 3.0 (Format), and this one is a list; "+ + "it is read as the empty string", diags[0].Message) + assert.Equal(t, "a `<<` merge key must name a mapping or a list of mappings, and this one names a scalar; "+ + "it merges nothing", diags[1].Message) +} + +// TestCheckFormat_IsBoundedAndKeyedToTheMinor pins the two edges: a version +// the table does not cover is held to nothing, and a tree past the budget is +// read as far as it and reported, an unset budget being the default one. +func TestCheckFormat_IsBoundedAndKeyedToTheMinor(t *testing.T) { + t.Parallel() + const doc = "x-e:\n ? [k]\n : 1\n" + assert.Empty(t, formatFindings(t, "2.0", doc, 0)) + assert.Equal(t, []finding{at(diag.BudgetExceeded, "")}, formatFindings(t, "3.1.0", doc, 2)) + assert.Equal(t, []finding{at(mismatch, "/x-e/")}, formatFindings(t, "3.1.0", doc, 0)) +} + +// TestCheckFormat_NamesEachKind pins kindName over what a merge can name: a +// mapping is no finding, and nothing at all, which no parse produces, is +// named as such. +func TestCheckFormat_NamesEachKind(t *testing.T) { + t.Parallel() + merge := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!merge", Value: "<<"} + root := &yaml.Node{Kind: yaml.MappingNode, Content: []*yaml.Node{merge, {Kind: yaml.AliasNode}}} + diags := oas.CheckFormat(root, "3.1.0", oas.Options{}) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Message, "names nothing") +} diff --git a/compilers/openapi/internal/openapitest/spec.go b/compilers/openapi/internal/openapitest/spec.go index b983d744e..62a74a7c6 100644 --- a/compilers/openapi/internal/openapitest/spec.go +++ b/compilers/openapi/internal/openapitest/spec.go @@ -1,6 +1,8 @@ package openapitest import ( + "strings" + oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" soa "github.com/speakeasy-api/openapi/openapi" "github.com/speakeasy-api/openapi/sequencedmap" @@ -42,6 +44,14 @@ func PathsSpecVer(version, paths string) string { "paths:\n" + paths } +// NestedMerges writes body, a flow mapping, merged in through levels `<<` keys +// each naming the next in place: `{<<: {<<: body}}` for two. One level past +// nodeview.MergeDepthLimit is a mapping the parser reads whole and the +// compiler's view reads short of body. +func NestedMerges(levels int, body string) string { + return strings.Repeat("{<<: ", levels) + body + strings.Repeat("}", levels) +} + // EveryKindOfReference is a document holding a resolvable reference of each // kind the library has: one in each position its walk meets one, and a schema's. // Each names the component of the same kind, whose name is its section's first diff --git a/compilers/openapi/internal/openapitest/spec_test.go b/compilers/openapi/internal/openapitest/spec_test.go index eebf40eda..78e5d4dbb 100644 --- a/compilers/openapi/internal/openapitest/spec_test.go +++ b/compilers/openapi/internal/openapitest/spec_test.go @@ -127,3 +127,13 @@ func TestInlineProbeBody_WritesEveryKeywordTheProbeAssertsOn(t *testing.T) { assert.Contains(t, openapitest.InlineProbeBody, want) } } + +// TestNestedMerges_WritesOneMergePerLevel pins the spelling: a body merged in +// through as many `<<` keys as asked, a level of none being the body itself. +func TestNestedMerges_WritesOneMergePerLevel(t *testing.T) { + t.Parallel() + assert.Equal(t, "{k: v}", openapitest.NestedMerges(0, "{k: v}")) + assert.Equal(t, "{<<: {<<: {k: v}}}", openapitest.NestedMerges(2, "{k: v}")) + var n yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(openapitest.NestedMerges(3, "{k: v}")), &n), "it parses") +} diff --git a/compilers/openapi/internal/operation/budgets_test.go b/compilers/openapi/internal/operation/budgets_test.go index 80e7413b5..abff84937 100644 --- a/compilers/openapi/internal/operation/budgets_test.go +++ b/compilers/openapi/internal/operation/budgets_test.go @@ -13,7 +13,6 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/compilers/openapi/internal/operation" "github.com/dexpace/morphic/compilers/openapi/internal/overlay" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" ) // TestLowerService_CanceledContextLowersNoPathOrWebhook holds both walk loops to @@ -37,11 +36,10 @@ webhooks: c := lowering.New(0, loadedDoc.Doc, loadedDoc.Source, lowering.GroupByTags, lowering.Limits{}, lowering.StreamingMedia{}, lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}). WithLoaded(loadedDoc) - var anchors schema.AnchorIndex ctx, cancel := context.WithCancel(t.Context()) cancel() - svc, _, diags := operation.LowerService(ctx, c, compile.NewTypes(), &anchors) + svc, _, diags := operation.LowerService(ctx, c, compile.NewTypes()) assert.Empty(t, diags) assert.Empty(t, svc.Groups, "neither the path loop nor the webhook loop lowered an operation") diff --git a/compilers/openapi/internal/operation/callbacks.go b/compilers/openapi/internal/operation/callbacks.go index dedc9b217..cb8496ea4 100644 --- a/compilers/openapi/internal/operation/callbacks.go +++ b/compilers/openapi/internal/operation/callbacks.go @@ -10,7 +10,6 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/annotation" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" ) @@ -20,7 +19,7 @@ import ( // parent.mount roots callback operation identity, so two parents sharing one // $ref'd callback keep distinct callback operations; parent.decl is the base a // $ref'd callback or path item resolves against (issue #107). -func lowerCallbacks(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, src *soa.Operation, parent opPointers, inferred string) ([]ir.Callback, []ir.Operation, ir.Unmodeled, []ir.Diagnostic) { +func lowerCallbacks(c lowering.Ctx, ts *compile.Types, claims *operationIDClaims, src *soa.Operation, parent opPointers, inferred string) ([]ir.Callback, []ir.Operation, ir.Unmodeled, []ir.Diagnostic) { cbMap := src.GetCallbacks() if cbMap == nil || cbMap.Len() == 0 { return nil, nil, nil, nil @@ -49,7 +48,7 @@ func lowerCallbacks(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd } diags = append(diags, found...) mount := parent.mount + ids.Ptr("callbacks", cbName, exprStr) - opIDs, cbOps, orphan, cbDiags := lowerCallbackOps(ts, anchors, claims, layers, mount, exprStr, inferred) + opIDs, cbOps, orphan, cbDiags := lowerCallbackOps(ts, claims, layers, mount, exprStr, inferred) ext = annotation.MergeUnmodeled(ext, orphan) diags = append(diags, cbDiags...) callbacks = append(callbacks, ir.Callback{Expression: exprStr, Operations: opIDs}) @@ -69,9 +68,9 @@ func lowerCallbacks(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd // An expression whose item mounts no operation keeps what the item wrote on the // returned map, which goes where the Callback Object's own extensions go, the // parent's HTTP binding. -func lowerCallbackOps(ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, layers []layer, mount jsontext.Pointer, expr, inferred string) ([]ir.OpID, []ir.Operation, ir.Unmodeled, []ir.Diagnostic) { +func lowerCallbackOps(ts *compile.Types, claims *operationIDClaims, layers []layer, mount jsontext.Pointer, expr, inferred string) ([]ir.OpID, []ir.Operation, ir.Unmodeled, []ir.Diagnostic) { var orphan ir.Unmodeled - mounted, diags := mountPathItem(ts, anchors, claims, layers, mount, opContext{key: expr, inferred: inferred}, &orphan) + mounted, diags := mountPathItem(ts, claims, layers, mount, opContext{key: expr, inferred: inferred}, &orphan) opIDs := make([]ir.OpID, 0, len(mounted)) ops := make([]ir.Operation, 0, len(mounted)) for _, m := range mounted { @@ -86,7 +85,7 @@ func lowerCallbackOps(ts *compile.Types, anchors *schema.AnchorIndex, claims *op // webhooks-map key, an event's name rather than a path, as WebhookName. It // stops between webhooks when ctx is done, as lowerPaths does, and svc holds // what a webhook item mounting no operation writes. -func lowerWebhooks(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service) []ir.Diagnostic { +func lowerWebhooks(ctx context.Context, c lowering.Ctx, ts *compile.Types, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service) []ir.Diagnostic { hooks := c.Doc.GetWebhooks() if hooks == nil || hooks.Len() == 0 { return nil @@ -103,7 +102,7 @@ func lowerWebhooks(ctx context.Context, c lowering.Ctx, ts *compile.Types, ancho } diags = append(diags, found...) route := opContext{key: name, isWebhook: true, withCallbacks: true} - mounted, hookDiags := mountPathItem(ts, anchors, claims, layers, hookPtr, route, &svc.Unmodeled) + mounted, hookDiags := mountPathItem(ts, claims, layers, hookPtr, route, &svc.Unmodeled) diags = append(diags, hookDiags...) for _, m := range mounted { grp := groups.group(ids.WebhookGroup(), func() ir.OperationGroup { diff --git a/compilers/openapi/internal/operation/content.go b/compilers/openapi/internal/operation/content.go index 42021589b..6961d9313 100644 --- a/compilers/openapi/internal/operation/content.go +++ b/compilers/openapi/internal/operation/content.go @@ -16,6 +16,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" ) @@ -25,7 +26,7 @@ import ( // content selection (ir-design §7.2). The pointer is the payload owner (the // response or requestBody); each Content's schema hoists under // /content//schema. -func lowerPayload(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, content *sequencedmap.Map[string, *soa.MediaType], pointer jsontext.Pointer, hint string) (*ir.Payload, []ir.Diagnostic) { +func lowerPayload(c lowering.Ctx, ts *compile.Types, content *sequencedmap.Map[string, *soa.MediaType], pointer jsontext.Pointer) (*ir.Payload, []ir.Diagnostic) { if content == nil || content.Len() == 0 { return nil, nil } @@ -35,7 +36,7 @@ func lowerPayload(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex if media == nil { continue } - one, contentDiags := lowerContent(c, ts, anchors, mt, media, pointer, hint) + one, contentDiags := lowerContent(c, ts, mt, media, pointer) diags = append(diags, contentDiags...) payload.Contents = append(payload.Contents, one) } @@ -52,7 +53,7 @@ func lowerPayload(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex // load reports them below 3.2 (diag.LaterVersionConstruct). func keepMediaTypeFields(c lowering.Ctx, u *ir.Unmodeled, media *soa.MediaType, mediaPtr jsontext.Pointer, scope string) []ir.Diagnostic { root := media.GetRootNode() - if ref := annotation.RawChildNode(root, "$ref"); ref != nil { + if ref := nodeview.Child(root, "$ref"); ref != nil { return keepMediaTypeRef(c, u, root, ref, mediaPtr, scope) } return keepMediaTypeDescription(c, u, root, mediaPtr, scope) @@ -63,7 +64,7 @@ func keepMediaTypeFields(c lowering.Ctx, u *ir.Unmodeled, media *soa.MediaType, func keepMediaTypeDescription(c lowering.Ctx, u *ir.Unmodeled, root *yaml.Node, mediaPtr jsontext.Pointer, scope string) []ir.Diagnostic { at := mediaPtr + ids.Ptr("description") kept, diags := annotation.PreserveNodeInto(u, annotation.EntryKey(scope, "description"), - annotation.RawChildNode(root, "description"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(root, "description"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) if !kept { return diags } @@ -80,7 +81,7 @@ func keepMediaTypeRef(c lowering.Ctx, u *ir.Unmodeled, root, ref *yaml.Node, med var diags []ir.Diagnostic for _, sibling := range []string{"summary", "description"} { _, siblingDiags := annotation.PreserveNodeInto(u, annotation.EntryKey(scope, sibling), - annotation.RawChildNode(root, sibling), ir.ReasonNoIRHome, c.ProvenanceAt(mediaPtr+ids.Ptr(sibling))) + nodeview.Child(root, sibling), ir.ReasonNoIRHome, c.ProvenanceAt(mediaPtr+ids.Ptr(sibling))) diags = append(diags, siblingDiags...) } at := mediaPtr + ids.Ptr("$ref") @@ -97,9 +98,9 @@ func keepMediaTypeRef(c lowering.Ctx, u *ir.Unmodeled, root, ref *yaml.Node, med // lowerContent lowers one media-type view: its type graph, examples, binary/ // form specialization, sequential-media shape, and extensions. -func lowerContent(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, mt string, media *soa.MediaType, pointer jsontext.Pointer, hint string) (ir.Content, []ir.Diagnostic) { +func lowerContent(c lowering.Ctx, ts *compile.Types, mt string, media *soa.MediaType, pointer jsontext.Pointer) (ir.Content, []ir.Diagnostic) { mediaPtr := pointer + ids.Ptr("content", mt) - mediaType, diags := schema.Ref(c, ts, anchors, schema.TopLevelDepth, media.GetSchema(), mediaPtr+ids.Ptr("schema"), hint) + mediaType, diags := schema.Ref(c, ts, schema.TopLevelDepth, media.GetSchema(), mediaPtr+ids.Ptr("schema")) content := ir.Content{ MediaType: mt, Type: mediaType, @@ -114,14 +115,14 @@ func lowerContent(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex content.File = &ir.FileInfo{IsText: false, ContentTypes: []string{mt}} content.Type = ts.PrimRef(ir.PrimBytes) case isFormContent(mt): - enc, encUnmodeled, encDiags := partEncodings(c, ts, anchors, media, mediaPtr, content.Type.Target) + enc, encUnmodeled, encDiags := partEncodings(c, ts, media, mediaPtr, content.Type.Target) diags = append(diags, encDiags...) if enc != nil { content.Encoding = enc } content.Unmodeled = annotation.MergeUnmodeled(content.Unmodeled, encUnmodeled) } - diags = append(diags, fillSequential(c, ts, anchors, &content, media, mediaPtr, hint)...) + diags = append(diags, fillSequential(c, ts, &content, media, mediaPtr)...) ext, extDiags := annotation.ExtensionsFrom(media.GetExtensions(), c.ProvenanceAt, mediaPtr) diags = append(diags, extDiags...) if len(ext) > 0 { diff --git a/compilers/openapi/internal/operation/content_internal_test.go b/compilers/openapi/internal/operation/content_internal_test.go index 3948c06b9..858be556c 100644 --- a/compilers/openapi/internal/operation/content_internal_test.go +++ b/compilers/openapi/internal/operation/content_internal_test.go @@ -14,7 +14,7 @@ func TestLowerPayload_NilMediaEntriesYieldNil(t *testing.T) { content := sequencedmap.New( sequencedmap.NewElem("application/json", (*soa.MediaType)(nil)), ) - payload, diags := lowerPayload(l.ctx, l.types, &l.anchors, content, "/p", "hint") + payload, diags := lowerPayload(l.ctx, l.types, content, "/p") assert.Nil(t, payload, "all-nil media map yields no payload") assert.Empty(t, diags) } diff --git a/compilers/openapi/internal/operation/content_test.go b/compilers/openapi/internal/operation/content_test.go index 2134d94bc..e20d9856b 100644 --- a/compilers/openapi/internal/operation/content_test.go +++ b/compilers/openapi/internal/operation/content_test.go @@ -539,7 +539,8 @@ func TestContent_MediaTypeRefIsKeptNotFollowed(t *testing.T) { // a Media Type Object, which the bundled model names no field for and // ir.Content has no docs to hold: at a body's content and at a parameter's, it // is kept verbatim with an info rather than called undefined. Below 3.2 it is -// kept the same, and load adds that the field is a later version's. +// kept the same, and load adds that the field is a later version's. One merged +// in through `<<` is kept as a written one is (GitHub #384). func TestContent_MediaTypeDescriptionIsKept(t *testing.T) { t.Parallel() const at = "/paths/~1a/get/responses/200/content/application~1json/description" @@ -560,12 +561,15 @@ func TestContent_MediaTypeDescriptionIsKept(t *testing.T) { description: ok content: application/json: {description: The body., schema: {type: string}} + text/plain: {<<: {description: Merged.}, schema: {type: string}} `)) openapitest.RequireNoErrorDiags(t, diags) op := openapitest.FirstOp(t, svc) kept := op.Responses[0].Payload.Contents[0].Unmodeled["openapi:description"] assert.Equal(t, ir.ReasonNoIRHome, kept.Reason) assert.JSONEq(t, `"The body."`, string(kept.Value)) + require.Len(t, op.Responses[0].Payload.Contents, 2) + assert.JSONEq(t, `"Merged."`, string(op.Responses[0].Payload.Contents[1].Unmodeled["openapi:description"].Value)) require.Len(t, op.Params, 1) assert.Equal(t, ir.ReasonNoIRHome, op.Params[0].Unmodeled["openapi:content/application~1json/description"].Reason) diff --git a/compilers/openapi/internal/operation/encoding.go b/compilers/openapi/internal/operation/encoding.go index 6d84f9637..396e96a81 100644 --- a/compilers/openapi/internal/operation/encoding.go +++ b/compilers/openapi/internal/operation/encoding.go @@ -11,7 +11,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -23,13 +23,13 @@ import ( // second value rather than writing it, because PartEncoding carries no // Unmodeled map of its own; scope is where that belongs on the owning Content // (see encodingUnmodeled). -func encodingConfig(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, enc *soa.Encoding, encPtr jsontext.Pointer, scope string) (ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { +func encodingConfig(c lowering.Ctx, ts *compile.Types, enc *soa.Encoding, encPtr jsontext.Pointer, scope string) (ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { pe := ir.PartEncoding{} if enc == nil { return pe, nil, nil } pe.ContentTypes = splitContentTypes(enc.GetContentTypeValue()) - headers, diags := lowerHeaders(c, ts, anchors, enc.GetHeaders(), encPtr) + headers, diags := lowerHeaders(c, ts, enc.GetHeaders(), encPtr) pe.Headers = headers if enc.Style != nil { pe.Style = string(*enc.Style) @@ -56,7 +56,7 @@ func encodingUnmodeled(c lowering.Ctx, enc *soa.Encoding, encPtr jsontext.Pointe var out ir.Unmodeled at := encPtr + ids.Ptr("allowReserved") kept, diags := annotation.PreserveNodeInto(&out, "openapi:"+scope+"/allowReserved", - annotation.RawChildNode(enc.GetRootNode(), "allowReserved"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(enc.GetRootNode(), "allowReserved"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) if kept { diags = append(diags, c.DiagAt(ir.SeverityInfo, diag.DegradedConstruct, at, "encoding allowReserved has no ir.PartEncoding home; kept verbatim under Unmodeled")) @@ -88,7 +88,7 @@ func nestedEncodings(c lowering.Ctx, out *ir.Unmodeled, enc *soa.Encoding, encPt for _, keyword := range nestedEncodingFields { at := encPtr + ids.Ptr(keyword) kept, keptDiags := annotation.PreserveNodeInto(out, annotation.EntryKey(scope, keyword), - annotation.RawChildNode(enc.GetRootNode(), keyword), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(enc.GetRootNode(), keyword), ir.ReasonNoIRHome, c.ProvenanceAt(at)) diags = append(diags, keptDiags...) if !kept { continue diff --git a/compilers/openapi/internal/operation/encoding_internal_test.go b/compilers/openapi/internal/operation/encoding_internal_test.go index 182bb2931..43afeca71 100644 --- a/compilers/openapi/internal/operation/encoding_internal_test.go +++ b/compilers/openapi/internal/operation/encoding_internal_test.go @@ -12,7 +12,7 @@ import ( func TestEncodingConfig_NilEncoding(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) - pe, unmodeled, diags := encodingConfig(l.ctx, l.types, &l.anchors, nil, "/mp", "itemEncoding") + pe, unmodeled, diags := encodingConfig(l.ctx, l.types, nil, "/mp", "itemEncoding") assert.Equal(t, ir.PartEncoding{}, pe) assert.Nil(t, unmodeled) assert.Empty(t, diags) diff --git a/compilers/openapi/internal/operation/headers.go b/compilers/openapi/internal/operation/headers.go index 22245e043..cc28c9e8d 100644 --- a/compilers/openapi/internal/operation/headers.go +++ b/compilers/openapi/internal/operation/headers.go @@ -12,6 +12,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" @@ -21,7 +22,7 @@ import ( // entry's own pointer stays its ID and Provenance (two keys $ref'ing the same // header must not collide), but its schema — and the name hint that schema is // hoisted under — follow the ref target's declaration instead (issue #107). -func lowerHeaders(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, headers *sequencedmap.Map[string, *soa.ReferencedHeader], basePtr jsontext.Pointer) ([]ir.Property, []ir.Diagnostic) { +func lowerHeaders(c lowering.Ctx, ts *compile.Types, headers *sequencedmap.Map[string, *soa.ReferencedHeader], basePtr jsontext.Pointer) ([]ir.Property, []ir.Diagnostic) { if headers == nil || headers.Len() == 0 { return nil, nil } @@ -33,7 +34,7 @@ func lowerHeaders(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex if h == nil { continue } - p, headerDiags := lowerHeader(c, dc, ts, anchors, h, name, hptr, hdecl) + p, headerDiags := lowerHeader(c, dc, ts, h, name, hptr, hdecl) diags = append(diags, headerDiags...) diags = append(diags, reservedHeaderEntryDiag(c, name, hptr)...) // A summary or description beside a $ref of the entry's chain documents @@ -70,15 +71,14 @@ func reservedHeaderEntryDiag(c lowering.Ctx, name string, hptr jsontext.Pointer) // docs, constraints, xml, examples and validation-only keywords the same way, // and ir.Property has a field for each, so the header path had no reason to drop // them (GitHub #116). -func lowerHeader(c, dc lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, h *soa.Header, name string, hptr, hdecl jsontext.Pointer) (ir.Property, []ir.Diagnostic) { +func lowerHeader(c, dc lowering.Ctx, ts *compile.Types, h *soa.Header, name string, hptr, hdecl jsontext.Pointer) (ir.Property, []ir.Diagnostic) { elected, diags := electTypeSpelling(dc, h.GetSchema(), h.GetContent(), h.GetRootNode(), hdecl, "header", "ir.Property") // The schema is named after the key the header is declared under, never // this mount's, and lowered in its declaration's document (dc), so every // response mounting one header builds the same nodes beneath it, in // whichever order they lower (GitHub #433). - headerType, headerDiags := schema.CarriedRef(dc, ts, anchors, - schema.TopLevelDepth, elected.js, elected.pointer, ids.DeclarationHint(hdecl, hdecl.LastToken())) + headerType, headerDiags := schema.CarriedRef(dc, ts, schema.TopLevelDepth, elected.js, elected.pointer) diags = append(diags, headerDiags...) p := ir.Property{ ID: c.IDs().Prop(hptr), @@ -96,7 +96,7 @@ func lowerHeader(c, dc lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIn // spelling keeps. p.Encoding = &ir.Encoding{MediaType: elected.mediaType} } - diags = append(diags, schema.FillPropertyDetail(dc, ts, anchors, &p, elected.js, elected.pointer)...) + diags = append(diags, schema.FillPropertyDetail(dc, ts, &p, elected.js, elected.pointer)...) diags = append(diags, applyHeaderAnnotations(dc, &p, h, hdecl, elected.examples)...) return p, append(diags, preserveHeaderSerialization(dc, &p, h, hdecl)...) } @@ -115,7 +115,7 @@ func preserveHeaderSerialization(c lowering.Ctx, p *ir.Property, h *soa.Header, for _, keyword := range []string{"style", "explode"} { at := hdecl + ids.Ptr(keyword) kept, keptDiags := annotation.PreserveNodeInto(&p.Unmodeled, "openapi:"+keyword, - annotation.RawChildNode(h.GetRootNode(), keyword), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(h.GetRootNode(), keyword), ir.ReasonNoIRHome, c.ProvenanceAt(at)) diags = append(diags, keptDiags...) if !kept { continue diff --git a/compilers/openapi/internal/operation/helpers_internal_test.go b/compilers/openapi/internal/operation/helpers_internal_test.go index 8ff151b30..93cfb0f95 100644 --- a/compilers/openapi/internal/operation/helpers_internal_test.go +++ b/compilers/openapi/internal/operation/helpers_internal_test.go @@ -23,14 +23,13 @@ import ( // there would reach these. // lowerer is test scaffolding, not a compiler type: every lowering is a -// function of the context, the registry and the memos (#177), and this holds -// the ones a single-lowering test wants in one place. +// function of the context and the registry (#177), and this holds what a +// single-lowering test wants in one place. type lowerer struct { - ctx lowering.Ctx - out *ir.Document - types *compile.Types - diags compile.Diags - anchors schema.AnchorIndex + ctx lowering.Ctx + out *ir.Document + types *compile.Types + diags compile.Diags } // lowererOver is the only place the fixture's fields are initialised. Both @@ -80,12 +79,12 @@ func newRawLowerer(doc *soa.OpenAPI) *lowerer { func lowerServiceSpec(t *testing.T, src string) (ir.Service, []ir.Diagnostic) { t.Helper() l, loadDiags := loweredFor(t, src) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) schemes, authDiags := auth.LowerSecuritySchemes(l.ctx) l.out.Auth = schemes l.diags.AppendAll(authDiags) - svc, _, svcDiags := LowerService(t.Context(), l.ctx.WithAuth(schemes), l.types, &l.anchors) + svc, _, svcDiags := LowerService(t.Context(), l.ctx.WithAuth(schemes), l.types) l.diags.AppendAll(svcDiags) _, finished := l.types.Finish(t.Context()) l.diags.AppendAll(finished) diff --git a/compilers/openapi/internal/operation/helpers_test.go b/compilers/openapi/internal/operation/helpers_test.go index 2c75e9e5d..23042f6e5 100644 --- a/compilers/openapi/internal/operation/helpers_test.go +++ b/compilers/openapi/internal/operation/helpers_test.go @@ -66,11 +66,10 @@ func serviceWithGrouping(t *testing.T, src string, grouping lowering.GroupingStr types := compile.NewTypes() c := lowering.New(0, loadedDoc.Doc, loadedDoc.Source, grouping, lowering.Limits{}, lowering.StreamingMedia{}, lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}).WithLoaded(loadedDoc) - var anchors schema.AnchorIndex var acc compile.Diags - acc.AppendAll(schema.LowerComponentSchemas(t.Context(), c, types, &anchors)) + acc.AppendAll(schema.LowerComponentSchemas(t.Context(), c, types)) - svc, _, svcDiags := operation.LowerService(t.Context(), c, types, &anchors) + svc, _, svcDiags := operation.LowerService(t.Context(), c, types) acc.AppendAll(svcDiags) _, finished := types.Finish(t.Context()) acc.AppendAll(finished) diff --git a/compilers/openapi/internal/operation/multipart.go b/compilers/openapi/internal/operation/multipart.go index de9471be5..31e9b06b4 100644 --- a/compilers/openapi/internal/operation/multipart.go +++ b/compilers/openapi/internal/operation/multipart.go @@ -28,7 +28,7 @@ import ( // behind body, which may be a node a $ref promised and nothing has built yet // (compile.Types.Promise), so partKeys settles both once every node exists. It // is nil where the body declares no part; one left with no key reads alike. -func partEncodings(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, media *soa.MediaType, mediaPtr jsontext.Pointer, body ir.TypeID) (map[ir.PropID]ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { +func partEncodings(c lowering.Ctx, ts *compile.Types, media *soa.MediaType, mediaPtr jsontext.Pointer, body ir.TypeID) (map[ir.PropID]ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { walk := walkBodyParts(c.Bindings(), media.GetSchema()) parts := walk.parts cutDiags := diagPartWalkCut(c, walk, mediaPtr+ids.Ptr("schema")) @@ -41,7 +41,7 @@ func partEncodings(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInde fallbackDoc, fallback := bodySchemaPointer(c, media.GetSchema(), mediaPtr+ids.Ptr("schema")) keys := partKeys{c: c, body: body, fallbackDoc: fallbackDoc, fallback: fallback} for _, part := range parts { - pe, partUnmodeled, partDiags := buildPartEncoding(c, ts, anchors, part, encMap, mediaPtr) + pe, partUnmodeled, partDiags := buildPartEncoding(c, ts, part, encMap, mediaPtr) diags = append(diags, partDiags...) // Kept whether or not the part's encoding ends up empty: an entry declaring // only allowReserved or only x-* lowers to an empty PartEncoding that is left @@ -368,13 +368,13 @@ func partTypeFlags(node func(ir.TypeID) (ir.TypeDef, bool), ref ir.TypeRef) (mul // (array part) and Filename (binary/file part), read off every schema declaring // the part. partKeys reads them off the part's merged type instead where it can // (partFlags). -func buildPartEncoding(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, part bodyPart, encMap *sequencedmap.Map[string, *soa.Encoding], mediaPtr jsontext.Pointer) (ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { +func buildPartEncoding(c lowering.Ctx, ts *compile.Types, part bodyPart, encMap *sequencedmap.Map[string, *soa.Encoding], mediaPtr jsontext.Pointer) (ir.PartEncoding, ir.Unmodeled, []ir.Diagnostic) { pe := ir.PartEncoding{} var unmodeled ir.Unmodeled var diags []ir.Diagnostic if encMap != nil { if enc, ok := encMap.Get(part.name); ok { - pe, unmodeled, diags = encodingConfig(c, ts, anchors, enc, + pe, unmodeled, diags = encodingConfig(c, ts, enc, mediaPtr+ids.Ptr("encoding", part.name), ids.Scope("encoding", part.name)) } } diff --git a/compilers/openapi/internal/operation/operationids.go b/compilers/openapi/internal/operation/operationids.go index a9c726236..05715cb12 100644 --- a/compilers/openapi/internal/operation/operationids.go +++ b/compilers/openapi/internal/operation/operationids.go @@ -8,9 +8,9 @@ import ( soa "github.com/speakeasy-api/openapi/openapi" yaml "gopkg.in/yaml.v3" - "github.com/dexpace/morphic/compilers/openapi/internal/annotation" "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -195,8 +195,9 @@ func declaringNode(n *yaml.Node) *yaml.Node { } // writtenTree reads where the document's own text writes a node. Each step -// reads only the pairs a mapping writes itself, following no $ref, alias or -// merge key, so a walk ends on a node only at the pointer its text is at. +// reads only the pairs a mapping writes itself (nodeview.WrittenChildren), +// following no $ref, alias or merge key, so a walk ends on a node only at the +// pointer its text is at. // // A mapping's pairs are indexed the first time a walk passes through it. A walk // to a declaration under paths crosses the mapping holding every path, and @@ -207,7 +208,7 @@ type writtenTree struct { } func newWrittenTree(root *yaml.Node) *writtenTree { - return &writtenTree{root: root, children: map[*yaml.Node]map[string]*yaml.Node{}} + return &writtenTree{root: nodeview.DocumentRoot(root), children: map[*yaml.Node]map[string]*yaml.Node{}} } // holds reports whether the document's text writes n at pointer. @@ -216,7 +217,7 @@ func (w *writtenTree) holds(pointer jsontext.Pointer, n *yaml.Node) bool { for token := range pointer.Tokens() { children, indexed := w.children[at] if !indexed { - children = annotation.RawChildNodes(at) + children = nodeview.WrittenChildren(at) w.children[at] = children } if at = children[token]; at == nil { diff --git a/compilers/openapi/internal/operation/operationids_internal_test.go b/compilers/openapi/internal/operation/operationids_internal_test.go index b784d689d..f033700a7 100644 --- a/compilers/openapi/internal/operation/operationids_internal_test.go +++ b/compilers/openapi/internal/operation/operationids_internal_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v3" - "github.com/dexpace/morphic/compilers/openapi/internal/annotation" "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/ir" ) @@ -39,7 +39,7 @@ const writtenFixture = `paths: func writtenOp(t *testing.T) (root, op *yaml.Node) { t.Helper() root = openapitest.YAMLNode(t, writtenFixture) - op = annotation.RawChildNode(annotation.RawChildNode(annotation.RawChildNode(root, "paths"), "/z"), "get") + op = nodeview.WrittenChildren(nodeview.WrittenChildren(nodeview.WrittenChildren(root)["paths"])["/z"])["get"] require.NotNil(t, op, "the fixture writes its operation at /z") return root, op } @@ -60,7 +60,7 @@ func TestWrittenTree_HoldsANodeOnlyWhereTheTextWritesIt(t *testing.T) { assert.False(t, w.holds("/paths/~1s/0", op), "a walk reads mappings only") assert.False(t, newWrittenTree(nil).holds("/paths/~1z/get", op), "a tree with no text holds nothing") - _, indexed := w.children[annotation.RawChildNode(root, "paths")] + _, indexed := w.children[nodeview.WrittenChildren(root)["paths"]] assert.True(t, indexed, "the paths mapping is indexed for the walks after the first") } @@ -199,8 +199,8 @@ func TestJudge_ReportsEveryDeclarationAndMountButTheFirst(t *testing.T) { func TestDeclaringNode_FollowsAnAliasOneHop(t *testing.T) { t.Parallel() root := openapitest.YAMLNode(t, "a: &op {operationId: x}\nb: *op\n") - anchored := annotation.RawChildNode(root, "a") - alias := annotation.RawChildNode(root, "b") + anchored := nodeview.WrittenChildren(root)["a"] + alias := nodeview.WrittenChildren(root)["b"] require.Equal(t, yaml.AliasNode, alias.Kind, "the fixture reuses the mapping through an alias") assert.Same(t, anchored, declaringNode(alias)) diff --git a/compilers/openapi/internal/operation/operations.go b/compilers/openapi/internal/operation/operations.go index 2f938d93a..d8f8e1e2f 100644 --- a/compilers/openapi/internal/operation/operations.go +++ b/compilers/openapi/internal/operation/operations.go @@ -12,7 +12,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -25,7 +25,7 @@ import ( // the two loops between path items and returns the groups filled so far; the // compiler's run sees ctx.Err() at the phase boundary after this and refuses, // rather than assembling a Document out of them. -func LowerService(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex) (ir.Service, []ir.TagDef, []ir.Diagnostic) { +func LowerService(ctx context.Context, c lowering.Ctx, ts *compile.Types) (ir.Service, []ir.TagDef, []ir.Diagnostic) { svc := ir.Service{ ID: ids.Service(c.SrcIndex), Provenance: c.ProvenanceAt(""), @@ -46,8 +46,8 @@ func LowerService(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchor diags = append(diags, pathsDiags...) groups := newServiceGroups(c.Doc.GetTags()) claims := newOperationIDClaims() - diags = append(diags, lowerPaths(ctx, c, ts, anchors, claims, groups, &svc)...) - diags = append(diags, lowerWebhooks(ctx, c, ts, anchors, claims, groups, &svc)...) + diags = append(diags, lowerPaths(ctx, c, ts, claims, groups, &svc)...) + diags = append(diags, lowerWebhooks(ctx, c, ts, claims, groups, &svc)...) // After both walks: whether an operationId is unique is a question about // every operation the service mounts, callbacks included. diags = append(diags, claims.report(c)...) @@ -104,7 +104,7 @@ func (o opContext) keyFields() (uriTemplate, webhookName string) { // lowerOperation lowers one source operation into the neutral core plus its HTTP // binding. It returns the operation and any callback operations that must be // registered alongside it in the same group (ir-design §7.2, §8.1). -func lowerOperation(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, src *soa.Operation, opCtx opContext) (ir.Operation, []ir.Operation, []ir.Diagnostic) { +func lowerOperation(c lowering.Ctx, ts *compile.Types, claims *operationIDClaims, src *soa.Operation, opCtx opContext) (ir.Operation, []ir.Operation, []ir.Diagnostic) { mount, decl := opCtx.ptrs.mount, opCtx.ptrs.decl // Built through the context so the source index is spelled in one place. Its // heuristic marker is filled in below, once every lowering that can add one @@ -128,10 +128,10 @@ func lowerOperation(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd if src.GetDeprecated() { op.Deprecation = &ir.Deprecation{} } - params, bindings, paramDiags := lowerParameters(c, ts, anchors, mount, opCtx.params) + params, bindings, paramDiags := lowerParameters(c, ts, mount, opCtx.params) diags = append(diags, paramDiags...) op.Params = params - diags = append(diags, lowerResponses(c, ts, anchors, &op, src, decl)...) + diags = append(diags, lowerResponses(c, ts, &op, src, decl)...) if opCtx.method == "" { diags = append(diags, c.DiagAt(ir.SeverityWarning, diag.InvalidMethodKey, decl, "additionalOperations key names no method; the operation lowers with an empty "+ @@ -145,7 +145,7 @@ func lowerOperation(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd IsWebhook: opCtx.isWebhook, ParamBindings: bindings, } - diags = append(diags, lowerRequestBody(c, ts, anchors, &op, &hb, src, decl)...) + diags = append(diags, lowerRequestBody(c, ts, &op, &hb, src, decl)...) // After both payload lowerings, which are what it reads. streaming, streamDiags := applyStreaming(c, &op, decl) diags = append(diags, streamDiags...) @@ -154,7 +154,7 @@ func lowerOperation(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd if opCtx.withCallbacks { var cbExt ir.Unmodeled var cbDiags []ir.Diagnostic - hb.Callbacks, extra, cbExt, cbDiags = lowerCallbacks(c, ts, anchors, claims, src, opCtx.ptrs, opCtx.inferred) + hb.Callbacks, extra, cbExt, cbDiags = lowerCallbacks(c, ts, claims, src, opCtx.ptrs, opCtx.inferred) hb.Unmodeled = annotation.MergeUnmodeled(hb.Unmodeled, cbExt) diags = append(diags, cbDiags...) } @@ -207,7 +207,7 @@ func applyOperationServers(c lowering.Ctx, op *ir.Operation, src *soa.Operation, return nil } kept, diags := annotation.PreserveNodeInto(&op.Unmodeled, "openapi:operationServers", - annotation.RawChildNode(src.GetRootNode(), "servers"), ir.ReasonNoIRHome, c.ProvenanceAt(declPtr+ids.Ptr("servers"))) + nodeview.Child(src.GetRootNode(), "servers"), ir.ReasonNoIRHome, c.ProvenanceAt(declPtr+ids.Ptr("servers"))) if !kept { return diags } diff --git a/compilers/openapi/internal/operation/operations_test.go b/compilers/openapi/internal/operation/operations_test.go index 0c6962b07..9a7bda0e6 100644 --- a/compilers/openapi/internal/operation/operations_test.go +++ b/compilers/openapi/internal/operation/operations_test.go @@ -439,7 +439,7 @@ func TestOperation_UnserializableExtensionStillWarns(t *testing.T) { t.Parallel() spec := openapitest.PathsSpec(` /x: get: - x-bad: {1: intkey} + x-bad: {n: .nan} responses: {"200": {description: ok}} `) doc, diags := parseFull(t, spec) diff --git a/compilers/openapi/internal/operation/params.go b/compilers/openapi/internal/operation/params.go index b68e5eb2a..792f6068f 100644 --- a/compilers/openapi/internal/operation/params.go +++ b/compilers/openapi/internal/operation/params.go @@ -13,6 +13,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" @@ -26,7 +27,7 @@ import ( // entry, else the list entry itself (issue #107). Its ID is rooted at mount, the // operation's mount pointer, not at the declaration it was copied from; a // repeat of one name and location keeps an ID of its own (paramLedger). -func lowerParameters(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, mount jsontext.Pointer, params []sourcedParam) ([]ir.Parameter, []ir.HTTPParamBinding, []ir.Diagnostic) { +func lowerParameters(c lowering.Ctx, ts *compile.Types, mount jsontext.Pointer, params []sourcedParam) ([]ir.Parameter, []ir.HTTPParamBinding, []ir.Diagnostic) { if len(params) == 0 { return nil, nil, nil } @@ -45,7 +46,7 @@ func lowerParameters(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIn } id, repeatDiags := ledger.next(pc, p.GetName(), string(p.GetIn()), sp.pointer) diags = append(diags, repeatDiags...) - param, binding, paramDiags := lowerParameter(dc, ts, anchors, id, p, pptr) + param, binding, paramDiags := lowerParameter(dc, ts, id, p, pptr) diags = append(diags, paramDiags...) // A summary or description beside a $ref of the entry's chain documents // this use of the parameter, over what its declaration says (GitHub #610). @@ -59,7 +60,7 @@ func lowerParameters(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIn // lowerParameter lowers one resolved parameter into its logical Parameter and // HTTP binding, both naming the parameter by id. Path parameters are always // required regardless of the declared flag (OpenAPI requires it). -func lowerParameter(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, id ir.ParamID, p *soa.Parameter, pptr jsontext.Pointer) (ir.Parameter, ir.HTTPParamBinding, []ir.Diagnostic) { +func lowerParameter(c lowering.Ctx, ts *compile.Types, id ir.ParamID, p *soa.Parameter, pptr jsontext.Pointer) (ir.Parameter, ir.HTTPParamBinding, []ir.Diagnostic) { name, in := p.GetName(), p.GetIn() param := ir.Parameter{ ID: id, @@ -76,7 +77,7 @@ func lowerParameter(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd Explode: explode, AllowReserved: p.GetAllowReserved(), } - media, diags := fillParamType(c, ts, anchors, ¶m, &binding, p, pptr, name) + media, diags := fillParamType(c, ts, ¶m, &binding, p, pptr) diags = append(diags, reservedHeaderParamDiag(c, name, in, pptr)...) return param, binding, append(diags, fillParamDetail(c, ¶m, p, pptr, media)...) } @@ -110,10 +111,10 @@ func reservedHeaderParamDiag(c lowering.Ctx, name string, in soa.ParameterIn, pp // parameter, whose media type goes on the binding. Constraints come from that // same schema position; the default comes from it too, falling back to its $ref // target (docs/compilers/openapi.md#ref-use-site). -func fillParamType(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, param *ir.Parameter, binding *ir.HTTPParamBinding, p *soa.Parameter, pptr jsontext.Pointer, name string) ([]ir.Example, []ir.Diagnostic) { +func fillParamType(c lowering.Ctx, ts *compile.Types, param *ir.Parameter, binding *ir.HTTPParamBinding, p *soa.Parameter, pptr jsontext.Pointer) ([]ir.Example, []ir.Diagnostic) { elected, diags := electTypeSpelling(c, p.GetSchema(), p.GetContent(), p.GetRootNode(), pptr, "parameter", "ir.Parameter") - paramType, typeDiags := schema.CarriedRef(c, ts, anchors, schema.TopLevelDepth, elected.js, elected.pointer, name) + paramType, typeDiags := schema.CarriedRef(c, ts, schema.TopLevelDepth, elected.js, elected.pointer) diags = append(diags, typeDiags...) param.Type = paramType param.Unmodeled = annotation.MergeUnmodeled(param.Unmodeled, elected.unmodeled) @@ -317,7 +318,7 @@ func fillParamDetail(c lowering.Ctx, param *ir.Parameter, p *soa.Parameter, pptr func preserveAllowEmptyValue(c lowering.Ctx, param *ir.Parameter, p *soa.Parameter, pptr jsontext.Pointer) []ir.Diagnostic { at := pptr + ids.Ptr("allowEmptyValue") kept, diags := annotation.PreserveNodeInto(¶m.Unmodeled, "openapi:allowEmptyValue", - annotation.RawChildNode(p.GetRootNode(), "allowEmptyValue"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(p.GetRootNode(), "allowEmptyValue"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) if !kept { return diags } diff --git a/compilers/openapi/internal/operation/pathitem.go b/compilers/openapi/internal/operation/pathitem.go index 9e8f66fc4..530f6cd31 100644 --- a/compilers/openapi/internal/operation/pathitem.go +++ b/compilers/openapi/internal/operation/pathitem.go @@ -13,7 +13,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -100,7 +100,7 @@ func pathOperations(pi *soa.PathItem) []pathOperation { // lowerPaths lowers every path operation in source order into groups, stopping // between path items when ctx is done. svc carries what a path item mounting no // operation writes, which has no operation of its own to hold it. -func lowerPaths(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service) []ir.Diagnostic { +func lowerPaths(ctx context.Context, c lowering.Ctx, ts *compile.Types, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service) []ir.Diagnostic { paths := c.Doc.GetPaths() if paths == nil { return nil @@ -118,7 +118,7 @@ func lowerPaths(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors continue } diags = append(diags, found...) - diags = append(diags, lowerPathItem(c, ts, anchors, claims, groups, svc, path, layers)...) + diags = append(diags, lowerPathItem(c, ts, claims, groups, svc, path, layers)...) } return diags } @@ -127,9 +127,9 @@ func lowerPaths(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors // into its group, carrying along any callback operations registered under the // same group. Each operation keeps its mount pointer (under path) as its // identity, and lowers from where its layer declares it (see pathItemLayers). -func lowerPathItem(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service, path string, layers []layer) []ir.Diagnostic { +func lowerPathItem(c lowering.Ctx, ts *compile.Types, claims *operationIDClaims, groups *serviceGroups, svc *ir.Service, path string, layers []layer) []ir.Diagnostic { route := opContext{key: path, withCallbacks: true, inferred: groupingInferred(c)} - mounted, diags := mountPathItem(ts, anchors, claims, layers, ids.Ptr("paths", path), route, &svc.Unmodeled) + mounted, diags := mountPathItem(ts, claims, layers, ids.Ptr("paths", path), route, &svc.Unmodeled) for _, m := range mounted { grp := groups.place(groups.groupFor(c, m.src, path)) grp.Operations = append(grp.Operations, m.op) @@ -155,7 +155,7 @@ type mountedOp struct { // mount lowering none keeps what its layers declare in residue instead. route // holds what the route sets on every operation; each brings its own method, // pointers and parameters. -func mountPathItem(ts *compile.Types, anchors *schema.AnchorIndex, claims *operationIDClaims, layers []layer, mount jsontext.Pointer, route opContext, residue *ir.Unmodeled) ([]mountedOp, []ir.Diagnostic) { +func mountPathItem(ts *compile.Types, claims *operationIDClaims, layers []layer, mount jsontext.Pointer, route opContext, residue *ir.Unmodeled) ([]mountedOp, []ir.Diagnostic) { ops := mountedOperations(layers) owner, diags := precedence(layers, len(ops) > 0) params := itemParams(layers) @@ -166,7 +166,7 @@ func mountPathItem(ts *compile.Types, anchors *schema.AnchorIndex, claims *opera opCtx.method = po.method opCtx.ptrs = opPointers{mount: mount + po.seg, decl: l.decl + po.seg} opCtx.params = mergeParameters(params, po.src.GetParameters(), opCtx.ptrs.decl) - op, extra, opDiags := lowerOperation(l.in, ts, anchors, claims, po.src, opCtx) + op, extra, opDiags := lowerOperation(l.in, ts, claims, po.src, opCtx) diags = append(diags, opDiags...) diags = append(diags, applyLayers(layers, owner, onOperation(&op))...) mounted = append(mounted, mountedOp{src: po.src, op: op, extra: extra}) @@ -266,9 +266,10 @@ func onNearestNode(c lowering.Ctx, u *ir.Unmodeled, mountPtr jsontext.Pointer) c // It keeps what applyPathItem keeps on a carrier holding no docs; the summary // and description are dropped (GitHub #383). // -// The announcement follows what landed, not what the item wrote: a merge-key -// construct is read by the model but not by RawChildNode, so a model predicate -// would claim a preservation that did not happen (GitHub #384). +// The announcement follows what landed, not what the item wrote: a construct +// merged in from past nodeview.MergeDepthLimit is read by the model and not by +// nodeview.Child, so a model predicate would claim a preservation that did not +// happen. func preserveUnmountedPathItem(c lowering.Ctx, into carrier, pi *soa.PathItem, mountPtr, declPtr jsontext.Pointer, cedes func(string) bool, ) []ir.Diagnostic { @@ -309,13 +310,13 @@ func undeclaredPathItemKeys(pi *soa.PathItem) []string { // The unmarshaller skips a key whose value carries a YAML anchor, so `bogus: // &a {...}` is in neither the map nor a field (GitHub #412). Anchors are // cleared before a model is built (GitHub #459, #501, #538), short of an - // external document past the hop bound that recovery follows; the raw node - // stays the one place such a key would be written. - for _, key := range annotation.RawMappingKeys(pi.GetRootNode()) { - if pathItemDeclares(pi, key) { + // external document past the hop bound that recovery follows; the mapping, + // read as the parser reads it, stays the one place such a key is found. + for _, p := range nodeview.Pairs(pi.GetRootNode()) { + if pathItemDeclares(pi, p.Key) { continue } - keys = append(keys, key) + keys = append(keys, p.Key) } return keys } @@ -362,8 +363,7 @@ var pathItemDocFields = []struct{ keyword, key string }{ // pair repeated on every operation. // // Dropped silently: an unmounted item's pair (carrier.keepsDocs is false; -// GitHub #383) and a pair supplied through a merge key or alias, which -// RawChildNode does not read (GitHub #384). +// GitHub #383). func applyPathItemDocs(c lowering.Ctx, into carrier, pi *soa.PathItem, declPtr jsontext.Pointer, cedes func(string) bool, ) []ir.Diagnostic { @@ -377,7 +377,7 @@ func applyPathItemDocs(c lowering.Ctx, into carrier, pi *soa.PathItem, declPtr j continue } wrote, fieldDiags := annotation.PreserveNodeInto(into.unmodeled, f.key, - annotation.RawChildNode(pi.GetRootNode(), f.keyword), ir.ReasonNoIRHome, + nodeview.Child(pi.GetRootNode(), f.keyword), ir.ReasonNoIRHome, c.ProvenanceAt(declPtr+ids.Ptr(f.keyword))) diags = append(diags, fieldDiags...) kept = kept || wrote @@ -405,7 +405,7 @@ func applyPathServers(c lowering.Ctx, into carrier, pi *soa.PathItem, declPtr js return nil } kept, diags := annotation.PreserveNodeInto(into.unmodeled, into.serversKey, - annotation.RawChildNode(pi.GetRootNode(), "servers"), ir.ReasonNoIRHome, c.ProvenanceAt(declPtr+ids.Ptr("servers"))) + nodeview.Child(pi.GetRootNode(), "servers"), ir.ReasonNoIRHome, c.ProvenanceAt(declPtr+ids.Ptr("servers"))) if !kept { return diags } diff --git a/compilers/openapi/internal/operation/pathitem_internal_test.go b/compilers/openapi/internal/operation/pathitem_internal_test.go index 28f1aa336..3a6372193 100644 --- a/compilers/openapi/internal/operation/pathitem_internal_test.go +++ b/compilers/openapi/internal/operation/pathitem_internal_test.go @@ -34,11 +34,10 @@ func TestApplyPathServers_WithoutRootNode(t *testing.T) { // TestApplyOperationServers_WithoutRootNode: a declared pair whose source node // cannot be read keeps nothing, and announces nothing it did not keep. // -// The input is synthetic, but the branch is not hypothetical — RawChildNode does -// not expand a merge key, so a real document supplying the pair through one -// reaches here with GetSummary non-empty and loses it silently (GitHub #384). -// This pins the current behaviour rather than endorsing it; when #384 makes the -// lookup view-aware, what changes is the node this receives, not this contract. +// The input is synthetic, but the branch is not hypothetical: a pair merged in +// through a `<<` chain deeper than nodeview.MergeDepthLimit is one the model +// reads and the compiler's raw read does not, so a real document reaches here +// with GetSummary non-empty. func TestApplyPathItemDocs_WithoutRootNode(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) diff --git a/compilers/openapi/internal/operation/pathitem_test.go b/compilers/openapi/internal/operation/pathitem_test.go index 68cc2d9f8..70e8eac0a 100644 --- a/compilers/openapi/internal/operation/pathitem_test.go +++ b/compilers/openapi/internal/operation/pathitem_test.go @@ -2,6 +2,7 @@ package operation_test import ( "encoding/json/jsontext" + "slices" "strings" "testing" @@ -10,6 +11,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/ir" ) @@ -539,27 +541,32 @@ paths: // TestPathItem_UnmountedAnnouncementFollowsWhatWasKept pins the announcement to // what landed rather than to what the source model reports. // -// A construct supplied through a YAML merge key is read by the model and not by -// RawChildNode, whose lookup is a plain mapping scan (GitHub #384). A predicate -// over the model therefore says `servers` were "kept beside it" while nothing was -// kept at all — a diagnostic asserting a preservation that did not happen, which -// is worse than the silence it replaced because it tells a reader the value is -// safe. +// Servers merged in through `<<` are read as written ones are, kept and +// announced (GitHub #384). Merged in from past nodeview.MergeDepthLimit, the +// model reads them and the compiler's raw read does not, so a predicate over +// the model would say they were "kept beside it" while nothing was kept: a +// diagnostic asserting a preservation that did not happen. func TestPathItem_UnmountedAnnouncementFollowsWhatWasKept(t *testing.T) { t.Parallel() - svc, diags := serviceWithGrouping(t, `openapi: 3.1.0 -info: {title: T, version: "1"} -x-base: &base - servers: [{url: 'https://m.example'}] -paths: - /unmounted: - <<: *base -`, lowering.GroupByTags) - - require.Empty(t, svc.Unmodeled, "the merge key is invisible to the raw-node lookup") - for _, d := range diags { - assert.NotContains(t, d.Message, "declares no operation this compiler lowers", - "nothing landed, so nothing claims it did") + const servers = "{servers: [{url: 'https://m.example'}]}" + for _, tc := range []struct { + name, item string + kept bool + }{ + {"merged in", "{<<: " + servers + "}", true}, + {"merged in past the bound", openapitest.NestedMerges(nodeview.MergeDepthLimit+1, servers), false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + svc, diags := serviceWithGrouping(t, "openapi: 3.1.0\ninfo: {title: T, version: \"1\"}\n"+ + "paths:\n /unmounted: "+tc.item+"\n", lowering.GroupByTags) + + assert.Equal(t, tc.kept, len(svc.Unmodeled) == 1, "%v", svc.Unmodeled) + announced := slices.ContainsFunc(diags, func(d ir.Diagnostic) bool { + return strings.Contains(d.Message, "declares no operation this compiler lowers") + }) + assert.Equal(t, tc.kept, announced, "the announcement follows what landed") + }) } } diff --git a/compilers/openapi/internal/operation/requestbody.go b/compilers/openapi/internal/operation/requestbody.go index 03daff656..7bd932e2c 100644 --- a/compilers/openapi/internal/operation/requestbody.go +++ b/compilers/openapi/internal/operation/requestbody.go @@ -13,7 +13,6 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" ) @@ -26,14 +25,14 @@ import ( // its content once at its declaration, not once per mount site, under the // component's name rather than the operationId hint (issue #107). A body $ref'd // from another operation has no component name, so it is named for the -// operation declaring it, never this mount's (requestBodyHint, GitHub #433). -func lowerRequestBody(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, op *ir.Operation, hb *ir.HTTPBinding, src *soa.Operation, opDeclPtr jsontext.Pointer) []ir.Diagnostic { +// operation declaring it, never this mount's (lowering.NameAt, GitHub #433). +func lowerRequestBody(c lowering.Ctx, ts *compile.Types, op *ir.Operation, hb *ir.HTTPBinding, src *soa.Operation, opDeclPtr jsontext.Pointer) []ir.Diagnostic { usePtr := opDeclPtr + ids.Ptr("requestBody") rb, dc, bodyPtr := lowering.Declared[soa.RequestBody](c, src.GetRequestBody(), usePtr) if rb == nil { return nil } - payload, diags := lowerPayload(dc, ts, anchors, rb.GetContent(), bodyPtr, requestBodyHint(dc, bodyPtr)) + payload, diags := lowerPayload(dc, ts, rb.GetContent(), bodyPtr) if payload == nil { // A body lowering no media type is still the body: whether it must be // sent, its description and its x-* stay on a Payload with no Contents. @@ -75,21 +74,3 @@ func contentTypeKeys(content *sequencedmap.Map[string, *soa.MediaType]) []string } return slices.Collect(content.Keys()) } - -// requestBodyHint is the hint the request body declared at bodyPtr, in dc's -// document, is named by: its component's name, else `_request` for -// the operation declaring it there (lowering.OperationIDAt), else -// elsewhereBodyHint. It is a -// function of where the body is written, never of which operation mounts it, -// so every operation lowering one body builds the same nodes beneath it, in -// whichever order they lower (GitHub #433, #107). -func requestBodyHint(dc lowering.Ctx, bodyPtr jsontext.Pointer) string { - if id := lowering.OperationIDAt(dc, bodyPtr.Parent()); id != "" { - return ids.DeclarationHint(bodyPtr, id+"_request") - } - return ids.DeclarationHint(bodyPtr, elsewhereBodyHint) -} - -// elsewhereBodyHint names a request body no operationId names: one an -// operation without an operationId declares, or one no operation declares. -const elsewhereBodyHint = "request" diff --git a/compilers/openapi/internal/operation/responses.go b/compilers/openapi/internal/operation/responses.go index 60f49569d..15d5b2dd7 100644 --- a/compilers/openapi/internal/operation/responses.go +++ b/compilers/openapi/internal/operation/responses.go @@ -15,8 +15,8 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/resolve" - "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" ) @@ -37,7 +37,7 @@ const ( // Default. opDeclPtr is the operation's own declaration pointer, so a $ref'd // response interns its content once at its component pointer rather than once // per mount site (issue #107). -func lowerResponses(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, op *ir.Operation, src *soa.Operation, opDeclPtr jsontext.Pointer) []ir.Diagnostic { +func lowerResponses(c lowering.Ctx, ts *compile.Types, op *ir.Operation, src *soa.Operation, opDeclPtr jsontext.Pointer) []ir.Diagnostic { // GetResponses never returns nil (it addresses an always-present map), so the // loop simply yields nothing when no responses are declared. resps := src.GetResponses() @@ -71,16 +71,16 @@ func lowerResponses(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd // statusConditions lets the success side record no status at all. // TestStatusRange_NamesNoStatus is what pins the pairing. if isErrorRange(rng) { - ec, ecDiags := lowerErrorCase(rc, ts, anchors, r, code, rng, rptr, rr) + ec, ecDiags := lowerErrorCase(rc, ts, r, code, rng, rptr, rr) diags = append(diags, ecDiags...) op.Errors = append(op.Errors, ec) } else { - resp, respDiags := lowerResponse(rc, ts, anchors, r, code, statusConditions(rng, named), rptr, rr) + resp, respDiags := lowerResponse(rc, ts, r, code, statusConditions(rng, named), rptr, rr) diags = append(diags, respDiags...) op.Responses = append(op.Responses, resp) } } - diags = append(diags, lowerDefaultResponse(c, ts, anchors, op, resps, opDeclPtr)...) + diags = append(diags, lowerDefaultResponse(c, ts, op, resps, opDeclPtr)...) return append(diags, duplicateStatusKeyDiags(c, byRange, opDeclPtr+ids.Ptr("responses"))...) } @@ -90,14 +90,14 @@ func lowerResponses(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd // (ir-design §7.2), so the compiler classes it as neither: whether it is the // success response, an error, or both is a reading of the API, which the plan // makes under its policy. -func lowerDefaultResponse(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, op *ir.Operation, +func lowerDefaultResponse(c lowering.Ctx, ts *compile.Types, op *ir.Operation, resps *soa.Responses, opDeclPtr jsontext.Pointer, ) []ir.Diagnostic { def, dc, dptr := lowering.Declared[soa.Response](c, resps.GetDefault(), opDeclPtr+ids.Ptr("responses", defaultResponseKey)) if def == nil { return nil } - resp, diags := lowerResponse(dc, ts, anchors, def, defaultResponseKey, + resp, diags := lowerResponse(dc, ts, def, defaultResponseKey, statusConditions(statusRange(defaultResponseKey)), dptr, resps.GetDefault()) op.Default = &resp return diags @@ -129,8 +129,8 @@ func duplicateStatusKeyDiags(c lowering.Ctx, byRange map[ir.StatusRange][]string // all when it named no status (see statusConditions). ref is the entry the // document wrote there, which carries the summary and description a Reference // Object may put beside its $ref. -func lowerResponse(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, r *soa.Response, code string, conds ir.ResponseConditions, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (ir.Response, []ir.Diagnostic) { - parts, diags := lowerResponseParts(c, ts, anchors, r, code, rptr, ref) +func lowerResponse(c lowering.Ctx, ts *compile.Types, r *soa.Response, code string, conds ir.ResponseConditions, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (ir.Response, []ir.Diagnostic) { + parts, diags := lowerResponseParts(c, ts, r, code, rptr, ref) resp := ir.Response{ Name: parts.name, Conditions: conds, @@ -162,9 +162,9 @@ type responseParts struct { // error interns its body once, so two naming fallbacks would rename it when // paths are reordered. The use-site docs fold over ref, the entry written here, // must agree across classes for the same reason (GitHub #610). -func lowerResponseParts(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, r *soa.Response, code string, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (responseParts, []ir.Diagnostic) { - headers, diags := lowerHeaders(c, ts, anchors, r.GetHeaders(), rptr) - payload, payloadDiags := lowerPayload(c, ts, anchors, r.GetContent(), rptr, ids.DeclarationHint(rptr, "response")) +func lowerResponseParts(c lowering.Ctx, ts *compile.Types, r *soa.Response, code string, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (responseParts, []ir.Diagnostic) { + headers, diags := lowerHeaders(c, ts, r.GetHeaders(), rptr) + payload, payloadDiags := lowerPayload(c, ts, r.GetContent(), rptr) diags = append(diags, payloadDiags...) parts := responseParts{name: responseName(code), payload: payload, headers: headers} summary, taken, summaryDiags := responseSummary(c, &parts.unmodeled, r, rptr) @@ -197,7 +197,7 @@ func responseSummary(c lowering.Ctx, p *ir.Unmodeled, r *soa.Response, rptr json // (GitHub #616). func preserveResponseExtras(c lowering.Ctx, p *ir.Unmodeled, r *soa.Response, rptr jsontext.Pointer, taken ...string) []ir.Diagnostic { _, diags := annotation.PreserveNodeInto(p, "openapi:links", - annotation.RawChildNode(r.GetRootNode(), "links"), ir.ReasonNoIRHome, c.ProvenanceAt(rptr+ids.Ptr("links"))) + nodeview.Child(r.GetRootNode(), "links"), ir.ReasonNoIRHome, c.ProvenanceAt(rptr+ids.Ptr("links"))) ext, extDiags := annotation.ExtensionsFrom(r.GetExtensions(), c.ProvenanceAt, rptr) *p = annotation.MergeUnmodeled(*p, ext) diags = append(diags, extDiags...) @@ -228,8 +228,8 @@ func responseName(code string) ir.Naming { // #422). code is the responses-map key, the only record of how the source // spelled a status its range cannot state ("4XX"); ref is the entry written // here. -func lowerErrorCase(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, r *soa.Response, code string, rng ir.StatusRange, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (ir.ErrorCase, []ir.Diagnostic) { - parts, diags := lowerResponseParts(c, ts, anchors, r, code, rptr, ref) +func lowerErrorCase(c lowering.Ctx, ts *compile.Types, r *soa.Response, code string, rng ir.StatusRange, rptr jsontext.Pointer, ref *soa.ReferencedResponse) (ir.ErrorCase, []ir.Diagnostic) { + parts, diags := lowerResponseParts(c, ts, r, code, rptr, ref) ec := ir.ErrorCase{ Name: parts.name, Conditions: ir.ResponseConditions{StatusCodes: []ir.StatusRange{rng}}, diff --git a/compilers/openapi/internal/operation/responses_internal_test.go b/compilers/openapi/internal/operation/responses_internal_test.go index bde9e2f7a..ff70b83c4 100644 --- a/compilers/openapi/internal/operation/responses_internal_test.go +++ b/compilers/openapi/internal/operation/responses_internal_test.go @@ -97,7 +97,7 @@ func TestLowerResponses_NoResponses(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) op := &ir.Operation{} - diags := lowerResponses(l.ctx, l.types, &l.anchors, op, &soa.Operation{}, "/op") + diags := lowerResponses(l.ctx, l.types, op, &soa.Operation{}, "/op") assert.Nil(t, op.Responses) assert.Nil(t, op.Errors) assert.Nil(t, op.Default) diff --git a/compilers/openapi/internal/operation/responses_test.go b/compilers/openapi/internal/operation/responses_test.go index dd96704f4..a593e8e7e 100644 --- a/compilers/openapi/internal/operation/responses_test.go +++ b/compilers/openapi/internal/operation/responses_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/dexpace/morphic/compilers/openapi/internal/diag" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/ir" ) @@ -679,11 +680,12 @@ func TestResponses_SummaryIsReadAtEveryVersion(t *testing.T) { } } -// TestResponses_SummaryNoReaderTakesIsStillReported pins the two summaries the -// 3.2 reader does not place in Docs.Summary, which must not vanish because the +// TestResponses_SummaryNoReaderTakesIsStillReported pins the 3.2 summaries the +// reader does not place in Docs.Summary, which must not vanish because the // census was told the key is the dialect's: one that is not a string is kept -// verbatim with a warning, and one merged in through `<<` keeps the census's -// report that it reached the IR in no form. +// verbatim with a warning, and one merged in from past +// nodeview.MergeDepthLimit keeps the census's report that it reached the IR in +// no form. One merged in through `<<` is read as a written one is. func TestResponses_SummaryNoReaderTakesIsStillReported(t *testing.T) { t.Parallel() doc, _, diags := lowerServiceSpec(t, openapitest.PathsSpecVer("3.2.0", ` /a: @@ -694,16 +696,18 @@ func TestResponses_SummaryNoReaderTakesIsStillReported(t *testing.T) { "201": <<: {summary: Merged} description: ok + "202": `+openapitest.NestedMerges(nodeview.MergeDepthLimit+1, "{summary: Deep, description: ok}")+` `)) responses := openapitest.FindOp(t, doc, "a").Responses - require.Len(t, responses, 2) + require.Len(t, responses, 3) assert.Empty(t, responses[0].Docs.Summary) kept, ok := responses[0].Unmodeled["openapi:summary"] require.True(t, ok, "a summary that is not a string is kept verbatim") assert.JSONEq(t, `["not","a","string"]`, string(kept.Value)) assert.True(t, openapitest.HasDiagCodeAt(diags, diag.DegradedConstruct, "/paths/~1a/get/responses/200/summary")) - assert.True(t, openapitest.HasDiagCodeAt(diags, diag.UnknownKeyUnreachable, "/paths/~1a/get/responses/201/summary"), - "a merged-in summary is still reported; got %v", diags) + assert.Equal(t, "Merged", responses[1].Docs.Summary, "a merged-in summary is read") + assert.True(t, openapitest.HasDiagCodeAt(diags, diag.UnknownKeyUnreachable, "/paths/~1a/get/responses/202/summary"), + "one past the bound is still reported; got %v", diags) } // TestResponses_SummaryUseSiteBeatsTheDeclaration holds the 3.2 field to the diff --git a/compilers/openapi/internal/operation/sequential.go b/compilers/openapi/internal/operation/sequential.go index 5349f7820..e64c1f3d8 100644 --- a/compilers/openapi/internal/operation/sequential.go +++ b/compilers/openapi/internal/operation/sequential.go @@ -10,6 +10,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/schema" "github.com/dexpace/morphic/ir" ) @@ -22,10 +23,10 @@ import ( // itemEncoding govern the items *after* them rather than every item, which a // single every-item encoding would misstate. Those documents take // positionalEncoding instead. -func fillSequential(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorIndex, content *ir.Content, media *soa.MediaType, mediaPtr jsontext.Pointer, hint string) []ir.Diagnostic { +func fillSequential(c lowering.Ctx, ts *compile.Types, content *ir.Content, media *soa.MediaType, mediaPtr jsontext.Pointer) []ir.Diagnostic { var diags []ir.Diagnostic if item := media.GetItemSchema(); item != nil { - ref, itemDiags := schema.Ref(c, ts, anchors, schema.TopLevelDepth, item, mediaPtr+ids.Ptr("itemSchema"), compile.SubHint(hint, "item")) + ref, itemDiags := schema.Ref(c, ts, schema.TopLevelDepth, item, mediaPtr+ids.Ptr("itemSchema")) diags = append(diags, itemDiags...) content.Item = &ref } @@ -36,7 +37,7 @@ func fillSequential(c lowering.Ctx, ts *compile.Types, anchors *schema.AnchorInd if enc == nil { return diags } - pe, unmodeled, encDiags := encodingConfig(c, ts, anchors, enc, mediaPtr+ids.Ptr("itemEncoding"), "itemEncoding") + pe, unmodeled, encDiags := encodingConfig(c, ts, enc, mediaPtr+ids.Ptr("itemEncoding"), "itemEncoding") pe.Multi = true content.ItemEncoding = &pe content.Unmodeled = annotation.MergeUnmodeled(content.Unmodeled, unmodeled) @@ -57,9 +58,9 @@ func positionalEncoding(c lowering.Ctx, content *ir.Content, media *soa.MediaTyp // suppress the message, and its own conversion failure reports separately. at := mediaPtr + ids.Ptr("prefixEncoding") kept, diags := annotation.PreserveNodeInto(&content.Unmodeled, "openapi:prefixEncoding", - annotation.RawChildNode(root, "prefixEncoding"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + nodeview.Child(root, "prefixEncoding"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) _, itemDiags := annotation.PreserveNodeInto(&content.Unmodeled, "openapi:itemEncoding", - annotation.RawChildNode(root, "itemEncoding"), ir.ReasonNoIRHome, c.ProvenanceAt(mediaPtr+ids.Ptr("itemEncoding"))) + nodeview.Child(root, "itemEncoding"), ir.ReasonNoIRHome, c.ProvenanceAt(mediaPtr+ids.Ptr("itemEncoding"))) diags = append(diags, itemDiags...) if !kept { // Reaching here means prefixEncoding is declared — that is the only reason diff --git a/compilers/openapi/internal/operation/sequential_internal_test.go b/compilers/openapi/internal/operation/sequential_internal_test.go index 00cd2cc1c..aea33c74c 100644 --- a/compilers/openapi/internal/operation/sequential_internal_test.go +++ b/compilers/openapi/internal/operation/sequential_internal_test.go @@ -16,7 +16,7 @@ func TestFillSequential_EmptyItemEncoding(t *testing.T) { l := newRawLowerer(&soa.OpenAPI{}) content := &ir.Content{} media := &soa.MediaType{ItemEncoding: &soa.Encoding{}} - diags := fillSequential(l.ctx, l.types, &l.anchors, content, media, "/mp", "h") + diags := fillSequential(l.ctx, l.types, content, media, "/mp") assert.Equal(t, &ir.PartEncoding{Multi: true}, content.ItemEncoding, "a config-free itemEncoding still records that the tail repeats") assert.Nil(t, content.Unmodeled, "nothing is preserved raw") @@ -33,7 +33,7 @@ func TestPositionalEncoding_WithoutRootNode(t *testing.T) { l := newRawLowerer(&soa.OpenAPI{}) content := &ir.Content{} media := &soa.MediaType{PrefixEncoding: []*soa.Encoding{{}}, ItemEncoding: &soa.Encoding{}} - diags := fillSequential(l.ctx, l.types, &l.anchors, content, media, "/mp", "h") + diags := fillSequential(l.ctx, l.types, content, media, "/mp") assert.Nil(t, content.ItemEncoding, "prefixes still block the every-item lowering") assert.Nil(t, content.Unmodeled, "a media type with no source node has nothing verbatim to keep") openapitest.AssertHasCode(t, diags, diag.UnpreservableConstruct, ir.SeverityError) diff --git a/compilers/openapi/internal/operation/sequential_test.go b/compilers/openapi/internal/operation/sequential_test.go index 9cc186b8d..9042e74ab 100644 --- a/compilers/openapi/internal/operation/sequential_test.go +++ b/compilers/openapi/internal/operation/sequential_test.go @@ -43,6 +43,26 @@ func TestFillSequential_PrefixEncodingWithoutItemEncoding(t *testing.T) { openapitest.AssertHasCode(t, diags, diag.DegradedConstruct, ir.SeverityInfo) } +// TestContent_AMergedPositionalEncodingIsPreserved pins that prefixEncoding +// and itemEncoding merged in through `<<` are kept as written ones are (GitHub +// #384). The parser reads them, so this lowering runs, and a raw read that +// missed them would report a declared prefixEncoding it could not find. +func TestContent_AMergedPositionalEncodingIsPreserved(t *testing.T) { + t.Parallel() + written := " prefixEncoding:\n - contentType: application/json\n" + + " - contentType: application/xml\n itemEncoding: {contentType: text/plain}\n" + merged := " <<: {prefixEncoding: [{contentType: application/json}, {contentType: application/xml}], " + + "itemEncoding: {contentType: text/plain}}\n" + require.Contains(t, positionalEncodingSpec, written, "sanity: the spec writes both in place") + doc, diags := parseFull(t, strings.Replace(positionalEncodingSpec, written, merged, 1)) + c := openapitest.FindOp(t, doc, "mixed").Responses[0].Payload.Contents[0] + + assert.False(t, openapitest.HasDiag(diags, diag.UnpreservableConstruct), "%+v", diags) + assert.JSONEq(t, `[{"contentType": "application/json"}, {"contentType": "application/xml"}]`, + string(c.Unmodeled["openapi:prefixEncoding"].Value)) + assert.JSONEq(t, `{"contentType": "text/plain"}`, string(c.Unmodeled["openapi:itemEncoding"].Value)) +} + // positionalEncodingSpec declares the 3.2 shape ItemEncoding cannot state: // prefixEncoding fixes the encoding of the leading items, so the itemEncoding // beside it governs only the tail after them, not every item. diff --git a/compilers/openapi/internal/operation/typespelling.go b/compilers/openapi/internal/operation/typespelling.go index 9ca687549..216d523c7 100644 --- a/compilers/openapi/internal/operation/typespelling.go +++ b/compilers/openapi/internal/operation/typespelling.go @@ -13,6 +13,7 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -88,7 +89,7 @@ func electTypeSpelling(c lowering.Ctx, js *oas3.JSONSchema[oas3.Referenceable], func passedOverSpelling(c lowering.Ctx, u *ir.Unmodeled, root *yaml.Node, passed, elected string, at jsontext.Pointer) []ir.Diagnostic { pointer := at + ids.Ptr(passed) kept, diags := annotation.PreserveNodeInto(u, "openapi:"+passed, - annotation.RawChildNode(root, passed), ir.ReasonDegradedLowering, c.ProvenanceAt(pointer)) + nodeview.Child(root, passed), ir.ReasonDegradedLowering, c.ProvenanceAt(pointer)) if !kept { return diags } diff --git a/compilers/openapi/internal/overlay/overlay.go b/compilers/openapi/internal/overlay/overlay.go index ecf05785c..ed2a8af69 100644 --- a/compilers/openapi/internal/overlay/overlay.go +++ b/compilers/openapi/internal/overlay/overlay.go @@ -278,48 +278,60 @@ func repairGrafts(root *yaml.Node, budget int) (normalized, safe bool) { // every node this compiler's other readings can see. func reachableNodes(root *yaml.Node, budget int) (map[*yaml.Node]bool, bool) { reachable := make(map[*yaml.Node]bool) - stack := []*yaml.Node{root} - for ; len(stack) > 0; budget-- { - if budget == 0 { - return nil, false - } - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if n == nil || reachable[n] { - continue + out := nodeview.Walk{Nodes: budget, Visit: func(n *yaml.Node, _ int) nodeview.Step { + if reachable[n] { + return nodeview.Prune } reachable[n] = true - stack = append(stack, n.Content...) + return nodeview.Descend + }}.Run(root) + if out.Cut { + return nil, false } return reachable, true } // substituteGrafts walks the tree and replaces each child that is an alias to a -// node outside reachable with the content that alias stands for. +// node outside reachable with the content that alias stands for, reporting +// whether every one was replaced within budget, which each child read and each +// node copied spends. // // A node is replaced in its parent's Content, which is why the walk reads // children rather than the node it is at: the tree's own root is a mapping the -// application never replaces wholesale, so no alias can sit there. +// application never replaces wholesale, so no alias can sit there. A copy +// holds no alias and is not walked. func substituteGrafts(n *yaml.Node, reachable map[*yaml.Node]bool, budget *int) bool { + safe := true + copies := map[*yaml.Node]bool{} + nodeview.Walk{Nodes: *budget + 1, Visit: func(n *yaml.Node, _ int) nodeview.Step { + if copies[n] { + return nodeview.Prune + } + if safe = substituteChildren(n, reachable, budget, copies); !safe { + return nodeview.Stop + } + return nodeview.Descend + }}.Run(n) + return safe +} + +// substituteChildren replaces each child of n that is a graft (see +// substituteGrafts), recording each copy made in copies, and reports whether +// budget held for all of them. +func substituteChildren(n *yaml.Node, reachable map[*yaml.Node]bool, budget *int, copies map[*yaml.Node]bool) bool { for i, child := range n.Content { if *budget <= 0 { return false } *budget-- - if child == nil { + if child == nil || child.Kind != yaml.AliasNode || child.Alias == nil || reachable[child.Alias] { continue } - if child.Kind == yaml.AliasNode && child.Alias != nil && !reachable[child.Alias] { - expanded, ok := expandAlias(child.Alias, budget) - if !ok { - return false - } - n.Content[i] = expanded - continue - } - if !substituteGrafts(child, reachable, budget) { + expanded, ok := expandAlias(child.Alias, budget) + if !ok { return false } + n.Content[i], copies[expanded] = expanded, true } return true } @@ -374,18 +386,12 @@ func expandAlias(target *yaml.Node, budget *int) (*yaml.Node, bool) { // overlay. func snapshot(root *yaml.Node, budget int) (map[*yaml.Node]string, bool) { before := make(map[*yaml.Node]string) - stack := []*yaml.Node{nodeview.DocumentRoot(root)} - for ; len(stack) > 0; budget-- { - if budget == 0 { - return nil, false - } - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if n == nil { - continue - } + out := nodeview.Walk{Nodes: budget, Visit: func(n *yaml.Node, _ int) nodeview.Step { before[n] = n.Value - stack = append(stack, n.Content...) + return nodeview.Descend + }}.Run(nodeview.DocumentRoot(root)) + if out.Cut { + return nil, false } return before, true } @@ -433,16 +439,20 @@ func attribute(root *yaml.Node, before map[*yaml.Node]string, budget int) (map[j } // frame is one node of the attribution walk together with the JSON pointer that -// addresses it and, for a mapping's value, the key it sits under. +// addresses it and, for a mapping's value, the key it sits under. merged marks +// a `<<` merge's value, whose members the parser reads as the merging +// mapping's own, so it and a list of them are addressed at that mapping. type frame struct { node *yaml.Node pointer jsontext.Pointer key *yaml.Node + merged bool } // children returns the frames beneath f, addressed the way the compiler -// addresses them: a mapping's values under their escaped keys, a sequence's -// elements under their positions. +// addresses them: a mapping's values under their keys as nodeview.KeyOf names +// them, a sequence's elements under their positions, and a `<<` merge's value +// at the mapping merging it. // // Mapping keys are not walked in their own right; each rides on its value's // frame, since the library appends a new key and its value together. An alias @@ -454,18 +464,22 @@ func children(f frame) []frame { case yaml.MappingNode: out := make([]frame, 0, len(f.node.Content)/2) for i := 0; i+1 < len(f.node.Content); i += 2 { - key := f.node.Content[i] - out = append(out, frame{ - node: f.node.Content[i+1], - pointer: f.pointer + ids.Ptr(key.Value), - key: key, - }) + key, own := nodeview.KeyOf(f.node.Content[i]) + at := frame{node: f.node.Content[i+1], pointer: f.pointer + ids.Ptr(key), key: f.node.Content[i]} + if !own { + at.pointer, at.merged = f.pointer, true + } + out = append(out, at) } return out case yaml.SequenceNode: out := make([]frame, 0, len(f.node.Content)) for i, child := range f.node.Content { - out = append(out, frame{node: child, pointer: f.pointer + ids.Ptr(strconv.Itoa(i))}) + at := frame{node: child, pointer: f.pointer + ids.Ptr(strconv.Itoa(i))} + if f.merged { + at.pointer, at.merged = f.pointer, true + } + out = append(out, at) } return out default: diff --git a/compilers/openapi/internal/overlay/overlay_internal_test.go b/compilers/openapi/internal/overlay/overlay_internal_test.go index e2456b939..fd906a1a5 100644 --- a/compilers/openapi/internal/overlay/overlay_internal_test.go +++ b/compilers/openapi/internal/overlay/overlay_internal_test.go @@ -298,3 +298,25 @@ func TestOrigin_IndexOfNamesTheSourceThatWroteEveryPosition(t *testing.T) { }) } } + +// TestChildren_AddressesEachNodeAsTheParserReadsIt pins the pointers the +// attribution walk gives a node: a key written through an alias at the key it +// names, a `<<` merge's value, a mapping or a list of them, at the mapping +// merging it, since the parser reads its members as that mapping's own, and a +// list that is no merge's under its indexes. +func TestChildren_AddressesEachNodeAsTheParserReadsIt(t *testing.T) { + t.Parallel() + aliased, inPlace := ynode.Scalar("v1"), ynode.Map(ynode.Scalar("m"), ynode.Scalar("v2")) + item := ynode.Map(ynode.Scalar("n"), ynode.Scalar("v3")) + list := ynode.Seq(item) + root := ynode.Map(ynode.Alias(ynode.Scalar("key")), aliased, ynode.Merge(), inPlace, ynode.Merge(), list) + + got := children(frame{node: root, pointer: "/r"}) + require.Len(t, got, 3) + assert.Equal(t, frame{node: aliased, pointer: "/r/key", key: root.Content[0]}, got[0]) + assert.Equal(t, frame{node: inPlace, pointer: "/r", key: root.Content[2], merged: true}, got[1]) + assert.Equal(t, frame{node: list, pointer: "/r", key: root.Content[4], merged: true}, got[2]) + + assert.Equal(t, []frame{{node: item, pointer: "/r", merged: true}}, children(got[2])) + assert.Equal(t, []frame{{node: item, pointer: "/s/0"}}, children(frame{node: list, pointer: "/s"})) +} diff --git a/compilers/openapi/internal/overlay/sequence.go b/compilers/openapi/internal/overlay/sequence.go index 3c5ba894c..55a0c693c 100644 --- a/compilers/openapi/internal/overlay/sequence.go +++ b/compilers/openapi/internal/overlay/sequence.go @@ -3,6 +3,7 @@ package overlay import ( "errors" "fmt" + "math" "slices" "strings" @@ -61,7 +62,7 @@ type sequence struct { func runSequence(doc *soaoverlay.Overlay, root *yaml.Node, at ir.Provenance, lax bool, maxNodes int) ([]ir.Diagnostic, bool) { s := &sequence{doc: doc, root: root, lax: lax, maxNodes: maxNodes} if s.budgeted() { - s.size = countNodes(root) + s.size = countNodes(root, s.maxNodes) } for i, action := range doc.Actions { if over, ok := s.charge(action); !ok { @@ -91,7 +92,7 @@ func (s *sequence) charge(action soaoverlay.Action) (int64, bool) { if s.size+cost > limit { // The running size only ever overstates, so take the real count before // refusing on it. - s.size = countNodes(s.root) + s.size = countNodes(s.root, s.maxNodes) if s.size+cost > limit { return s.size + cost, false } @@ -224,20 +225,11 @@ func actionType(action soaoverlay.Action) string { } } -// countNodes counts the nodes a Content walk from root reaches. -func countNodes(root *yaml.Node) int64 { - var n int64 - stack := []*yaml.Node{nodeview.DocumentRoot(root)} - for len(stack) > 0 { - node := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if node == nil { - continue - } - n++ - stack = append(stack, node.Content...) - } - return n +// countNodes counts the nodes a Content walk from root reaches, no further than +// one past limit: a tree past limit is refused at any size, and a refusal +// already says "at least". A limit of math.MaxInt counts the whole tree. +func countNodes(root *yaml.Node, limit int) int64 { + return int64(nodeview.Walk{Nodes: min(limit, math.MaxInt-1) + 1}.Run(nodeview.DocumentRoot(root)).Visited) } // weigher counts the nodes the library's clone would build from a node: every diff --git a/compilers/openapi/internal/overlay/sequence_internal_test.go b/compilers/openapi/internal/overlay/sequence_internal_test.go index 21550b48f..437c025ea 100644 --- a/compilers/openapi/internal/overlay/sequence_internal_test.go +++ b/compilers/openapi/internal/overlay/sequence_internal_test.go @@ -3,6 +3,7 @@ package overlay import ( "bytes" "fmt" + "math" "strings" "testing" @@ -226,7 +227,7 @@ func TestSequence_RefusesFanOutBeforeBuildingIt(t *testing.T) { var fresh yaml.Node require.NoError(t, yaml.Unmarshal([]byte(sequenceSpec), &fresh)) - before := countNodes(&fresh) + before := countNodes(&fresh, maxNodes) root, diags, ok := budgetedOverlay(t, ov, int(before)+300) @@ -234,7 +235,7 @@ func TestSequence_RefusesFanOutBeforeBuildingIt(t *testing.T) { last := diags[len(diags)-1] assert.Equal(t, diag.BudgetExceeded, last.Code) assert.Contains(t, last.Message, "action 1 of 1") - assert.Equal(t, before, countNodes(root), "and nothing was built: the refusal came first") + assert.Equal(t, before, countNodes(root, maxNodes), "and nothing was built: the refusal came first") _, diags, ok = budgetedOverlay(t, ov, int(before)+1000) assert.True(t, ok, "the same overlay fits a budget with room for it: %+v", diags) @@ -259,7 +260,7 @@ func TestSequence_CountsEachActionOnTheTreeItRunsAgainst(t *testing.T) { require.NoError(t, err) require.Empty(t, path.Query(&fresh), "on the source, the later filter matches nothing") - _, diags, ok := budgetedOverlay(t, head+flag+later, int(countNodes(&fresh))+300) + _, diags, ok := budgetedOverlay(t, head+flag+later, int(countNodes(&fresh, maxNodes))+300) require.False(t, ok, "on the tree the first action leaves, it matches every path item") last := diags[len(diags)-1] @@ -282,12 +283,12 @@ func TestSequence_DoesNotRefuseOnAnOverstatedCharge(t *testing.T) { var fresh yaml.Node require.NoError(t, yaml.Unmarshal([]byte(sequenceSpec), &fresh)) - before := int(countNodes(&fresh)) + before := int(countNodes(&fresh, maxNodes)) root, diags, ok := budgetedOverlay(t, b.String(), before+10) assert.True(t, ok, "twenty charges of two nodes each exceed ten, and none of them adds a node: %+v", diags) - assert.Equal(t, int64(before), countNodes(root)) + assert.Equal(t, int64(before), countNodes(root, maxNodes)) } // TestSequence_AppliesWithoutABudget pins that a zero or negative budget turns @@ -359,12 +360,12 @@ func TestSequence_NeverChargesLessThanItBuilds(t *testing.T) { s := &sequence{doc: doc, root: &root, maxNodes: 1 << 20} charged := s.cost(doc.Actions[0], 1<<20) - before := countNodes(&root) + before := countNodes(&root, maxNodes) _, diags := Apply(1, &root, Options{Data: []byte(head + action)}) require.False(t, diag.HasError(diags), "%+v", diags) - built := countNodes(&root) - before + built := countNodes(&root, maxNodes) - before assert.GreaterOrEqual(t, charged, built, "charged %d, built %d", charged, built) }) } @@ -376,7 +377,16 @@ func TestSequence_NeverChargesLessThanItBuilds(t *testing.T) { func TestCountNodes_SkipsWhatAParseNeverProduces(t *testing.T) { t.Parallel() root := &yaml.Node{Kind: yaml.MappingNode, Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "a"}, nil}} - assert.Equal(t, int64(2), countNodes(root), "the mapping and its one real child") + assert.Equal(t, int64(2), countNodes(root, maxNodes), "the mapping and its one real child") +} + +// TestCountNodes_TheLargestLimitCountsTheWholeTree pins that a limit of +// math.MaxInt, past which no node lies, counts every node rather than +// overflowing the bound one past it. +func TestCountNodes_TheLargestLimitCountsTheWholeTree(t *testing.T) { + t.Parallel() + root := &yaml.Node{Kind: yaml.MappingNode, Content: []*yaml.Node{{Kind: yaml.ScalarNode}, {Kind: yaml.ScalarNode}}} + assert.Equal(t, int64(3), countNodes(root, math.MaxInt)) } // TestActionType_NamesByTheLibrarysPrecedence holds the names the renumbering diff --git a/compilers/openapi/internal/resolve/operationid.go b/compilers/openapi/internal/resolve/operationid.go deleted file mode 100644 index faf0c76e0..000000000 --- a/compilers/openapi/internal/resolve/operationid.go +++ /dev/null @@ -1,69 +0,0 @@ -package resolve - -import ( - "encoding/json/jsontext" - - soa "github.com/speakeasy-api/openapi/openapi" - yaml "gopkg.in/yaml.v3" - - "github.com/dexpace/morphic/compilers/openapi/internal/navigation" -) - -// OperationIDAt returns the operationId of the operation written at pointer, -// and "" where none is there or it names none. raw is the document's YAML, -// read as the library's pointer walk reads it (navigation.Step): a mapping -// written there is the operation, as a reference resolves to it, siblings of a -// path item's $ref included. model, the source's, answers only a pointer raw -// holds nothing at, one that reaches its operation through a reference. Either -// reads any scalar but null as text. -func OperationIDAt(raw *yaml.Node, model *soa.OpenAPI, pointer jsontext.Pointer) string { - tokens, ok := navigation.Tokens(pointer) - if !ok { - return "" - } - if raw != nil { - if written, ok := walk(raw, tokens); ok { - return rawOperationID(written) - } - } - if model == nil { - return "" - } - written, ok := walk(model, tokens) - if !ok { - return "" - } - op, _ := written.(*soa.Operation) - return op.GetOperationID() -} - -// walk steps tokens down from root as navigation.Step does, which reads -// through a raw YAML alias, and returns what it reaches, or false where a -// token finds nothing. -func walk(root any, tokens []string) (any, bool) { - node := root - for _, token := range tokens { - next, ok := navigation.Step(node, token) - if !ok { - return nil, false - } - node = next - } - return node, true -} - -// rawOperationID is the operationId written in what raw YAML holds at an -// operation's position, and "" where it writes none: only a mapping has one. -// A value written as an alias reads as its anchored scalar, which one hop -// reaches, since yaml.v3 parses no alias of an alias. -func rawOperationID(written any) string { - id, _ := navigation.Step(written, "operationId") - scalar, ok := id.(*yaml.Node) - if ok && scalar.Kind == yaml.AliasNode { - scalar = scalar.Alias - } - if !ok || scalar.Kind != yaml.ScalarNode || scalar.ShortTag() == "!!null" { - return "" - } - return scalar.Value -} diff --git a/compilers/openapi/internal/resolve/operationid_test.go b/compilers/openapi/internal/resolve/operationid_test.go deleted file mode 100644 index 18fa85e3f..000000000 --- a/compilers/openapi/internal/resolve/operationid_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package resolve_test - -import ( - "encoding/json/jsontext" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v3" - - "github.com/dexpace/morphic/compilers" - "github.com/dexpace/morphic/compilers/openapi/internal/load" - "github.com/dexpace/morphic/compilers/openapi/internal/resolve" -) - -// TestOperationIDAt_ReadsTheOperationWrittenAtAPointer pins the read a mounted -// request body is named by: the operationId of the operation written at a -// pointer, as a reference to its body resolves there. Raw YAML answers where a -// mapping is written, a path item's $ref siblings and aliases included, and -// the model only where nothing is, through a reference. Any scalar but null -// reads as its text; nothing else names one. -func TestOperationIDAt_ReadsTheOperationWrittenAtAPointer(t *testing.T) { - t.Parallel() - const spec = "openapi: 3.1.0\ninfo: {title: t, version: '1'}\nx-op: &op {operationId: aliased}\n" + - "x-item: &item {post: {operationId: viaItem}}\nx-name: &name named\npaths:\n" + - " /a: {post: {operationId: postA}, get: {summary: s}, put: {operationId: 7}, patch: {operationId: ~}}\n" + - " /b: {$ref: '#/components/pathItems/P', put: {operationId: besidePut}, post: {operationId: besidePost}}\n" + - " /c: {$ref: '#/components/pathItems/P'}\n" + - " /d: {get: *op, post: {<<: *op}, put: {operationId: *name}}\n /f: *item\n" + - "components:\n pathItems:\n P: {put: {operationId: putP}}\n" - doc, _, err := load.Load(t.Context(), 0, compilers.Source{Path: "spec.yaml", Data: []byte(spec)}, load.Options{}) - require.NoError(t, err) - var file yaml.Node - require.NoError(t, yaml.Unmarshal([]byte(spec), &file)) - raw := file.Content[0] - for pointer, want := range map[jsontext.Pointer]string{ - "/paths/~1a/post": "postA", - "/components/pathItems/P/put": "putP", - "/paths/~1a/put": "7", - "/paths/~1b/post": "besidePost", - "/paths/~1b/put": "besidePut", - "/paths/~1c/put": "putP", - "/paths/~1d/get": "aliased", - "/paths/~1d/post": "aliased", - "/paths/~1f/post": "viaItem", - "/paths/~1d/put": "named", - "/x-name": "", - "/paths/~1a": "", - "/paths/~1a/get": "", - "/paths/~1a/patch": "", - "/paths/~1a/post/operationId": "", - "/paths/~1e/post": "", - "/paths/~1a/post/~2": "", - } { - assert.Equal(t, want, resolve.OperationIDAt(raw, doc.Doc, pointer), pointer) - } - assert.Equal(t, "putP", resolve.OperationIDAt(nil, doc.Doc, "/paths/~1c/put"), "the model alone") - assert.Equal(t, "besidePut", resolve.OperationIDAt(raw, nil, "/paths/~1b/put"), "the YAML alone") - assert.Empty(t, resolve.OperationIDAt(raw, nil, "/paths/~1c/put"), "a reference only the model follows") - assert.Empty(t, resolve.OperationIDAt(nil, nil, "/paths/~1a/post")) -} diff --git a/compilers/openapi/internal/scan/nesting_internal_test.go b/compilers/openapi/internal/scan/nesting_internal_test.go index e13273c17..a1d60cceb 100644 --- a/compilers/openapi/internal/scan/nesting_internal_test.go +++ b/compilers/openapi/internal/scan/nesting_internal_test.go @@ -77,6 +77,7 @@ func TestDeepSites_NamesWhereEachWayDownPassesTheBound(t *testing.T) { {"an alias carrying it past", "x: &x {b: {c: 1}}\ny: {z: *x}\n", []jsontext.Pointer{"/y/z"}}, {"a merge value carrying it past", "m: {<<: {b: {c: 1}}}\n", []jsontext.Pointer{"/m"}}, {"a non-scalar key carrying it past", "m: {? {b: {c: 1}} : 1}\n", []jsontext.Pointer{"/m"}}, + {"the value of a non-scalar key carrying it past", "m: {? [k] : {b: {c: 1}}}\n", []jsontext.Pointer{"/m//b"}}, {"nothing past it", "a: {b: 1}\n", nil}, } for _, tc := range tests { diff --git a/compilers/openapi/internal/scan/scan.go b/compilers/openapi/internal/scan/scan.go index 2fdbcab90..fbe731560 100644 --- a/compilers/openapi/internal/scan/scan.go +++ b/compilers/openapi/internal/scan/scan.go @@ -55,9 +55,10 @@ func InTree(srcIndex int, pointers func(*yaml.Node) (jsontext.Pointer, bool)) Lo // Pointers returns a node's JSON pointer in root's tree, its pairs keyed as // the parser keys them (nodeview.KeyOf), and whether the tree holds the node. -// A key shares its member's pointer, and an alias's target is placed where it -// is written. A nil node or root is held by none, nor is a node under a `<<` -// merge key: the document the merge spells holds that pair at another pointer. +// A scalar key shares its member's pointer, and an alias's target is placed +// where it is written. A nil node or root is held by none, nor is a node under +// a `<<` merge key, whose pair the merge's document holds at another pointer, +// nor a key that is no scalar, which no pointer reads inside. // // The tree is indexed on the first node asked about, once, so a load that // reports nothing never indexes it. @@ -137,15 +138,17 @@ func reachChildren(n *yaml.Node, reach func(*yaml.Node, step)) { // eachChild hands visit each child of a mapping or sequence n, with the token // naming it and whether a pointer reaches it there: a pair under a `<<` merge -// key, or under a key that is no scalar, is n's but has no pointer of its own. -// A scalar holds nothing, and an alias's target is written elsewhere. +// key is n's but has no pointer of its own, and a key that is no scalar has +// none either, though its value is read under the empty key the parser gives +// it. A scalar holds nothing, and an alias's target is written elsewhere. func eachChild(n *yaml.Node, visit func(c *yaml.Node, token string, held bool)) { switch n.Kind { case yaml.MappingNode: for i := 0; i+1 < len(n.Content); i += 2 { key, value := n.Content[i], n.Content[i+1] token, own := nodeview.KeyOf(key) - visit(key, token, own) + k := nodeview.Deref(key) + visit(key, token, own && k != nil && k.Kind == yaml.ScalarNode) visit(value, token, own) } case yaml.SequenceNode: diff --git a/compilers/openapi/internal/scan/site_internal_test.go b/compilers/openapi/internal/scan/site_internal_test.go index c0f2e2b51..c1c3ea5a5 100644 --- a/compilers/openapi/internal/scan/site_internal_test.go +++ b/compilers/openapi/internal/scan/site_internal_test.go @@ -130,20 +130,23 @@ func TestLeast_AgreesWithSortingEveryPointer(t *testing.T) { } // TestLeast_RanksANodeNoPointerReachesLast pins the nodes Pointers gives no -// pointer: those under a `<<` merge key or a non-scalar key. Any node with a -// pointer is less than one without, and among those without the first given -// is taken, the one place left where the caller's order shows. +// pointer: those under a `<<` merge key, and a key that is no scalar, whose +// value is held at the empty key. Any node with a pointer is less than one +// without, and among those without the first given is taken, the one place +// left where the caller's order shows. func TestLeast_RanksANodeNoPointerReachesLast(t *testing.T) { t.Parallel() root := indexOf(t, []byte("z: {}\nm: {<<: {k: {}}, ? [c] : {}}\n")).Root() z := root.Content[1] pairs := root.Content[3].Content - merged, complexValue := pairs[1].Content[1], pairs[3] - require.Equal(t, yaml.SequenceNode, pairs[2].Kind, "the second pair's key is no scalar") + merged, complexKey, complexValue := pairs[1].Content[1], pairs[2], pairs[3] + require.Equal(t, yaml.SequenceNode, complexKey.Kind, "the second pair's key is no scalar") - assert.Same(t, z, Least(root, []*yaml.Node{merged, z, complexValue})) - assert.Same(t, merged, Least(root, []*yaml.Node{merged, complexValue})) - assert.Same(t, complexValue, Least(root, []*yaml.Node{complexValue, merged})) + assert.Same(t, z, Least(root, []*yaml.Node{merged, z, complexKey})) + assert.Same(t, merged, Least(root, []*yaml.Node{merged, complexKey})) + assert.Same(t, complexKey, Least(root, []*yaml.Node{complexKey, merged})) + assert.Same(t, complexValue, Least(root, []*yaml.Node{z, complexValue}), + "the value is held at /m/, which sorts before /z") } // TestLeast_EdgeCases pins the answers that need no ordering and the node a diff --git a/compilers/openapi/internal/schema/allof.go b/compilers/openapi/internal/schema/allof.go index df8c52f5b..8252e7cad 100644 --- a/compilers/openapi/internal/schema/allof.go +++ b/compilers/openapi/internal/schema/allof.go @@ -30,20 +30,20 @@ import ( // lowerModel reads a plain object's, so they land on the same field, not // Unmodeled (GitHub #407). A composition that leaves only a catch-all is a map, // as a plain object's is (objectNode). -func lowerAllOf(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerAllOf(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { cons, consDiags := ConstraintsOf(c, &common.Unmodeled, s, pointer) diags = append(diags, consDiags...) m := &ir.Model{TypeCommon: common, Constraints: cons} comp := allOfBranches(s, pointer) // the one flattened list; the readers below take it, not s idx := propIndex(c, ts, m) - diags = append(diags, fillAllOf(c, ts, anchors, depth, m, idx, comp)...) - diags = append(diags, fillModelProperties(c, ts, anchors, depth, idx, s, pointer)...) + diags = append(diags, fillAllOf(c, ts, depth, m, idx, comp)...) + diags = append(diags, fillModelProperties(c, ts, depth, idx, s, pointer)...) requireMembers(m, compositionRequired(s.GetRequired(), comp)) - diags = append(diags, fillAdditional(c, ts, anchors, depth, m, s, pointer, hint)...) + diags = append(diags, fillAdditional(c, ts, depth, m, s, pointer)...) diags = append(diags, applyFalseBranches(c, m, comp)...) - d, discDiags := lowerDiscriminator(c, ts, anchors, s, m, nil, pointer) + d, discDiags := lowerDiscriminator(c, ts, s, m, nil, pointer) diags = append(diags, discDiags...) if d != nil { m.Discriminator = d @@ -75,8 +75,8 @@ const MaxAllOfFlattenDepth = 32 type allOfBranch struct { schema *oas3.JSONSchema[oas3.Referenceable] // pointer is where the branch was written, never rewritten to a flat index: - // property IDs, provenance, diagnostics, and the branch's Unmodeled key and - // index within its own allOf (branchKey, pointerIndex) all derive from it. + // property IDs, provenance, diagnostics, the branch's Unmodeled key + // (branchKey) and its name (lowering.NameAt) all derive from it. pointer jsontext.Pointer // wrapper marks a flattened wrapper, which contributes its properties and // required list, and is kept verbatim only for what else it writes. @@ -223,12 +223,6 @@ func (a allOfComposition) branchKey(bptr jsontext.Pointer) string { return "openapi:" + strings.TrimPrefix(string(bptr), string(a.pointer)+"/") } -// pointerIndex is the last token of bptr: a branch's index within its own list, -// which is what branchHint needs to agree with subSchemaHint. -func pointerIndex(bptr jsontext.Pointer) string { - return string(bptr)[strings.LastIndexByte(string(bptr), '/')+1:] -} - // flattensInto reports whether the allOf branch b, at nesting depth depth, is a // wrapper walkAllOf flattens, as // docs/compilers/openapi.md#allof-classification states it. What else b writes @@ -274,16 +268,16 @@ func compositionRequired(own []string, comp allOfComposition) []string { // #143). The census every $ref site runs keeps what the alias cannot model, // except `required` (compositionCensusHandled); a co-declared oneOf/anyOf is kept // too (GitHub #406). -func fillAllOf(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, m *ir.Model, idx *merge.PropIndex, comp allOfComposition) []ir.Diagnostic { +func fillAllOf(c lowering.Ctx, ts *compile.Types, depth int, m *ir.Model, idx *merge.PropIndex, comp allOfComposition) []ir.Diagnostic { var diags []ir.Diagnostic baseIdx := selectAllOfBase(c.Bindings(), comp.branches) for i, e := range comp.branches { if !isRefBranch(e.schema) { - diags = append(diags, fillModelProperties(c, ts, anchors, depth, idx, e.schema.GetSchema(), e.pointer)...) + diags = append(diags, fillModelProperties(c, ts, depth, idx, e.schema.GetSchema(), e.pointer)...) diags = append(diags, preserveUnmergedBranch(c, m, e, comp.branchKey(e.pointer))...) continue } - diags = append(diags, fillRefBranch(c, ts, anchors, m, e, i == baseIdx)...) + diags = append(diags, fillRefBranch(c, ts, m, e, i == baseIdx)...) } return diags } @@ -291,10 +285,10 @@ func fillAllOf(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth in // fillRefBranch lowers one $ref allOf branch and adds it to m as its Base, when // isBase, or as a Mixin. A $ref it cannot resolve adds nothing, and is reported // unless it lies on a loop of $refs, which load reports once (onLoop). -func fillRefBranch(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, m *ir.Model, e allOfBranch, isBase bool) []ir.Diagnostic { +func fillRefBranch(c lowering.Ctx, ts *compile.Types, m *ir.Model, e allOfBranch, isBase bool) []ir.Diagnostic { var diags []ir.Diagnostic b := e.schema - id, ok := resolveSchemaRef(c, ts, anchors, b) + id, ok := resolveSchemaRef(c, ts, b) if !ok { if !onLoop(c, b) { diags = append(diags, c.DiagAt(ir.SeverityError, diag.UnresolvedRef, e.pointer, @@ -305,7 +299,7 @@ func fillRefBranch(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, m *i bs := b.GetSchema() unhomed := refSiteUnhomedKeywords(bs, compositionCensusHandled) hasUnion := declaresUnion(bs) - ref, homeDiags := homeDeclaration(c, ts, anchors, bs, ir.TypeRef{Target: id}, e.pointer, branchHintAt(c.Bindings(), b, pointerIndex(e.pointer)), annotation.HomeOwnNode, len(unhomed) > 0 || hasUnion) + ref, homeDiags := homeDeclaration(c, ts, bs, ir.TypeRef{Target: id}, e.pointer, annotation.HomeOwnNode, len(unhomed) > 0 || hasUnion) diags = append(diags, homeDiags...) diags = append(diags, recordUnhomedKeywords(c, ts, ref.Target, bs, unhomed, refSiteShape, e.pointer)...) if hasUnion { @@ -429,7 +423,7 @@ func unmergedWhy(residue []string, e allOfBranch) string { } // unmergedBranchKeys returns the keywords the branch declares that the merge does -// not consume, in source order. +// not consume, in the order the parser's model holds them. // // The keys come off the branch's own raw mapping rather than from a list of // keywords worth preserving, which is what keeps this from rotting: a keyword a @@ -483,9 +477,10 @@ func branchExcludesObject(bs *oas3.Schema) bool { return len(types) > 0 && !slices.Contains(types, oas3.SchemaTypeObject) } -// rawMappingKeys returns the keys a YAML mapping effectively writes, in source -// order, or nil when the node is no mapping. It enumerates what -// annotation.RawChildNode looks up one key at a time. +// rawMappingKeys returns the keys a YAML mapping effectively writes, a +// document node unwrapped first, in the order the parser's model holds them, +// or nil when the node is no mapping. It enumerates what nodeview.Child looks +// up one key at a time. // // Effective, not literal: a branch spelled `*anchor` writes the anchored // mapping's keys, and `<<: *anchor` the merged-in keys. A residue derived from @@ -494,11 +489,10 @@ func branchExcludesObject(bs *oas3.Schema) bool { // silently. nodeview.View defines how a mapping is read and bounds the // expansion (nodeview.MergeDepthLimit). func rawMappingKeys(root *yaml.Node) []string { - mapping := rawMapping(root) - if mapping == nil { + pairs := nodeview.Pairs(root) + if len(pairs) == 0 { return nil } - pairs := nodeview.New().MappingPairs(mapping) keys := make([]string, 0, len(pairs)) for _, p := range pairs { keys = append(keys, p.Key) @@ -506,24 +500,6 @@ func rawMappingKeys(root *yaml.Node) []string { return keys } -// rawMapping returns the mapping node root stands for: a document wrapper is -// unwrapped and an alias followed to its anchor (bounded by nodeview.Deref). Anything -// that is not a mapping after that — a scalar, a sequence, nil — yields nil, so -// a caller needs no kind check of its own. -func rawMapping(root *yaml.Node) *yaml.Node { - if root == nil { - return nil - } - if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { - root = root.Content[0] - } - root = nodeview.Deref(root) - if root == nil || root.Kind != yaml.MappingNode { - return nil - } - return root -} - // selectAllOfBase returns the index into branches of the one that becomes // Model.Base, or -1 when none qualifies (multiple non-hierarchy refs stay // Mixins). refs says where each branch's $ref ends. diff --git a/compilers/openapi/internal/schema/allof_internal_test.go b/compilers/openapi/internal/schema/allof_internal_test.go index 034193ed9..1dc8b80ab 100644 --- a/compilers/openapi/internal/schema/allof_internal_test.go +++ b/compilers/openapi/internal/schema/allof_internal_test.go @@ -45,8 +45,8 @@ func TestRawMappingKeys_OnlyEnumeratesAMapping(t *testing.T) { useValue(t, "anchor: &a plain\nuse: *a\n"), nil}, {"a merge key yields the keys it merges in", useValue(t, "anchor: &a {b: 1, c: 2}\nuse: {<<: *a}\n"), []string{"b", "c"}}, - {"an explicit key beats the one it merges over", - useValue(t, "anchor: &a {b: 1, c: 2}\nuse: {c: 9, <<: *a}\n"), []string{"c", "b"}}, + {"an explicit key beats the one it merges over, merged keys first", + useValue(t, "anchor: &a {b: 1, c: 2}\nuse: {c: 9, <<: *a}\n"), []string{"b", "c"}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -104,13 +104,10 @@ func TestAllOfBranches_DepthCapKeepsWrapperVerbatim(t *testing.T) { } } -// TestAllOfBranches_EntriesAgreeWithThePointerWalk pins, for the inline entries -// of a nested composition, the key each is kept under, which spells its path -// from the composed schema, and the positional hint from its innermost index, -// which is subSchemaHint's for its pointer. The fixture has no $ref entry, whose -// hint reads the target's name first: -// TestAllOf_NestedBranchHintAgreesWithThePointerWalk pins that case. -func TestAllOfBranches_EntriesAgreeWithThePointerWalk(t *testing.T) { +// TestAllOfBranches_KeysSpellEachEntrysPath pins, for the entries of a nested +// composition, the key each is kept under, which spells its path from the +// composed schema, and that a wrapper follows its own branches. +func TestAllOfBranches_KeysSpellEachEntrysPath(t *testing.T) { t.Parallel() l, _ := loweredFor(t, openapitest.ComponentSpec(` M: allOf: @@ -137,11 +134,9 @@ func TestAllOfBranches_EntriesAgreeWithThePointerWalk(t *testing.T) { var inline int for i, e := range entries { assert.Equal(t, wantKeys[i], comp.branchKey(e.pointer)) - if e.wrapper { - continue + if !e.wrapper { + inline++ } - inline++ - assert.Equal(t, subSchemaHint(nil, e.schema, e.pointer), branchHintAt(nil, e.schema, pointerIndex(e.pointer)), e.pointer) } assert.Equal(t, 4, inline) assert.True(t, entries[len(entries)-1].wrapper, "a wrapper follows its own branches") diff --git a/compilers/openapi/internal/schema/allof_test.go b/compilers/openapi/internal/schema/allof_test.go index d1dfe805d..2882bf3c9 100644 --- a/compilers/openapi/internal/schema/allof_test.go +++ b/compilers/openapi/internal/schema/allof_test.go @@ -2173,15 +2173,14 @@ func TestAllOf_NestedInlineBranchKeyedByPointer(t *testing.T) { } } -// TestAllOf_NestedBranchHintAgreesWithThePointerWalk pins that a $ref branch -// inside a wrapper is named by its innermost index, as subSchemaHint names the -// same pointer: an outside $ref to that pointer lowers it from the other side, -// and the two must agree in either declaration order. The target is the -// empty-named component because branchHint returns a target's own name before it -// reads the index, so only a target with an empty hint lets the index reach the IR. -// The B before the wrapper puts the branch at flat position 2 but innermost -// index 1, so only the innermost index yields variant_1. -func TestAllOf_NestedBranchHintAgreesWithThePointerWalk(t *testing.T) { +// TestAllOf_NestedBranchIsNamedByItsInnermostIndex pins that a $ref branch +// inside a wrapper is named by its innermost index, in either declaration +// order: an outside $ref to that pointer lowers it from the other side. The +// target is the empty-named component because a branch holding a $ref takes +// its target's name first, so only a target that names nothing lets the index +// reach the IR. The B before the wrapper puts the branch at flat position 2 but +// innermost index 1, so only the innermost index yields variant_1. +func TestAllOf_NestedBranchIsNamedByItsInnermostIndex(t *testing.T) { t.Parallel() m := ` M: allOf: diff --git a/compilers/openapi/internal/schema/array.go b/compilers/openapi/internal/schema/array.go index 86139cfd4..f7f25a58e 100644 --- a/compilers/openapi/internal/schema/array.go +++ b/compilers/openapi/internal/schema/array.go @@ -17,15 +17,15 @@ import ( // lowerArray hoists an array schema as a Tuple when prefixItems is present, else // a List over its item schema, each with the collection bounds it declares. -func lowerArray(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerArray(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { if prefix := s.GetPrefixItems(); len(prefix) > 0 { - t, tupleDiags := buildTuple(c, ts, anchors, depth, s, common, pointer, hint, prefix) + t, tupleDiags := buildTuple(c, ts, depth, s, common, pointer, prefix) diags = append(diags, tupleDiags...) return t } - elem, elemDiags := Ref(c, ts, anchors, depth, s.GetItems(), pointer+ids.Ptr("items"), compile.SubHint(hint, "item")) + elem, elemDiags := Ref(c, ts, depth, s.GetItems(), pointer+ids.Ptr("items")) diags = append(diags, elemDiags...) return &ir.List{ TypeCommon: common, @@ -40,22 +40,22 @@ func lowerArray(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth i // into its Rest, and the collection bounds into its Constraints. // // 2020-12 applies items to every element past prefixItems, so `items: false` -// leaves no Rest. Any other lowers as a List's items does, under the same hint, -// since positionHint names /items alike whichever node holds it. An absent one +// leaves no Rest. Any other lowers as a List's items does, under the same name, +// since /items is named alike whichever node holds it (lowering.NameAt). An absent one // lowers to the top type as `items: true` does, though only the latter makes an // unevaluatedItems beside it inert (#862). No field is derived from another: // Elems is never trimmed to maxItems, and a closed tuple gains no maxItems. -func buildTuple(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer, hint string, prefix []*oas3.JSONSchema[oas3.Referenceable]) (ir.TypeDef, []ir.Diagnostic) { +func buildTuple(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer, prefix []*oas3.JSONSchema[oas3.Referenceable]) (ir.TypeDef, []ir.Diagnostic) { var diags []ir.Diagnostic elems := make([]ir.TypeRef, 0, len(prefix)) for i, ps := range prefix { - ref, refDiags := Ref(c, ts, anchors, depth, ps, pointer+ids.Ptr("prefixItems", strconv.Itoa(i)), compile.SubHint(hint, strconv.Itoa(i))) + ref, refDiags := Ref(c, ts, depth, ps, pointer+ids.Ptr("prefixItems", strconv.Itoa(i))) diags = append(diags, refDiags...) elems = append(elems, ref) } t := &ir.Tuple{TypeCommon: common, Elems: elems, Constraints: collectionConstraints(s)} if items := s.GetItems(); !annotation.IsFalseSchema(items) { - rest, restDiags := Ref(c, ts, anchors, depth, items, pointer+ids.Ptr("items"), compile.SubHint(hint, "item")) + rest, restDiags := Ref(c, ts, depth, items, pointer+ids.Ptr("items")) diags = append(diags, restDiags...) t.Rest = &rest } diff --git a/compilers/openapi/internal/schema/array_test.go b/compilers/openapi/internal/schema/array_test.go index 42f27a33e..c7c4ebce2 100644 --- a/compilers/openapi/internal/schema/array_test.go +++ b/compilers/openapi/internal/schema/array_test.go @@ -191,7 +191,7 @@ func TestLower_MissingArrayItems(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() l, loadDiags := loweredFor(t, openapitest.ComponentSpecVer(tc.version, tc.schemas)) - raw := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors) + raw := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types) openapitest.RequireNoErrorDiags(t, raw) var invalid []string for _, d := range loadDiags { @@ -222,7 +222,7 @@ func TestLower_MissingArrayItemsAtAReferencedPositionWarnsOnce(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l, _ := loweredFor(t, openapitest.ComponentSpecVer("3.0.3", schemas)) - raw := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors) + raw := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types) require.Len(t, missingItemsAt(t, raw), 1, "the declaration lowers the property once") _, finished := l.types.Finish(t.Context()) raw = append(raw, finished...) diff --git a/compilers/openapi/internal/schema/budgets_test.go b/compilers/openapi/internal/schema/budgets_test.go index 44cf32adb..752ecc841 100644 --- a/compilers/openapi/internal/schema/budgets_test.go +++ b/compilers/openapi/internal/schema/budgets_test.go @@ -42,7 +42,7 @@ components: l, _ := loweredFor(t, spec) l.ctx.Limits = lowering.Limits{MaxEnumMembers: tc.limit} - diags := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors) + diags := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types) node := typeByName(l.document(t), "E") require.NotNil(t, node) @@ -93,7 +93,7 @@ components: ctx, cancel := context.WithCancel(t.Context()) cancel() - diags := schema.LowerComponentSchemas(ctx, l.ctx, l.types, &l.anchors) + diags := schema.LowerComponentSchemas(ctx, l.ctx, l.types) assert.Empty(t, diags) assert.Zero(t, l.types.Len(), "a cancelled walk lowers no component") diff --git a/compilers/openapi/internal/schema/census.go b/compilers/openapi/internal/schema/census.go index 80a2630b7..a59b05e35 100644 --- a/compilers/openapi/internal/schema/census.go +++ b/compilers/openapi/internal/schema/census.go @@ -275,7 +275,7 @@ func collapsedUnhomedKeywords(s *oas3.Schema) []string { if hasUnionSiblings(s) { return nil } - if _, _, _, collapses := nullUnionCollapse(s, ""); !collapses { + if _, _, collapses := nullUnionCollapse(s, ""); !collapses { return nil } return unhomedKeywords(s, nil, nil) @@ -294,7 +294,7 @@ func collapsedUnhomedKeywords(s *oas3.Schema) []string { // // A multipart body still mints encoding keys for `properties` the node lacks, // so pass.Validate reports ir/encoding-key-unknown-property. -func preserveUnhomedKeywords(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, hint string, id ir.TypeID, d dispatch) (ir.TypeID, []ir.Diagnostic) { +func preserveUnhomedKeywords(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, id ir.TypeID, d dispatch) (ir.TypeID, []ir.Diagnostic) { td, ok, diags := registeredNode(c, ts, id, pointer) if !ok { return id, diags @@ -312,7 +312,7 @@ func preserveUnhomedKeywords(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, owner := id if owns != id { var aliasDiags []ir.Diagnostic - owner, aliasDiags = internAlias(c, ts, s, pointer, hint, ir.TypeRef{Target: id}) + owner, aliasDiags = internAlias(c, ts, s, pointer, ir.TypeRef{Target: id}) diags = append(diags, aliasDiags...) } diags = append(diags, recordUnhomedKeywords(c, ts, owner, s, unhomed, nodeShape(td.Kind()), pointer)...) diff --git a/compilers/openapi/internal/schema/census_internal_test.go b/compilers/openapi/internal/schema/census_internal_test.go index eb4ac9779..5e57fc5ce 100644 --- a/compilers/openapi/internal/schema/census_internal_test.go +++ b/compilers/openapi/internal/schema/census_internal_test.go @@ -18,7 +18,7 @@ import ( func TestPreserveUnhomedKeywords_MissingNode(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) - got, diags := preserveUnhomedKeywords(l.ctx, l.types, &oas3.Schema{}, "/p", "h", "t/anon/missing", dispatch{}) + got, diags := preserveUnhomedKeywords(l.ctx, l.types, &oas3.Schema{}, "/p", "t/anon/missing", dispatch{}) assert.Equal(t, ir.TypeID("t/anon/missing"), got, "the lowering's own ID still stands") assertInternalInvariant(t, diags) } diff --git a/compilers/openapi/internal/schema/declaration.go b/compilers/openapi/internal/schema/declaration.go index e56fda202..f413058f3 100644 --- a/compilers/openapi/internal/schema/declaration.go +++ b/compilers/openapi/internal/schema/declaration.go @@ -23,10 +23,10 @@ import ( // $ref, which reached none of it until it was routed here. A position that // skips it drops what was written at it without a word — which is what both // GitHub #116 and #143 were. -func homeDeclaration(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3.Schema, target ir.TypeRef, pointer jsontext.Pointer, hint string, home annotation.Home, unhomed bool) (ir.TypeRef, []ir.Diagnostic) { - ref, diags := hoistDeclarationHome(c, ts, s, target, pointer, hint, home, unhomed) +func homeDeclaration(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, target ir.TypeRef, pointer jsontext.Pointer, home annotation.Home, unhomed bool) (ir.TypeRef, []ir.Diagnostic) { + ref, diags := hoistDeclarationHome(c, ts, s, target, pointer, home, unhomed) if s != nil { - diags = append(diags, attachDeclaredAnnotations(c, ts, anchors, s, pointer)...) + diags = append(diags, attachDeclaredAnnotations(c, ts, s, pointer)...) } return ref, append(diags, recordDeclarationResidue(c, ts, s, pointer, home)...) } @@ -43,7 +43,7 @@ func homeDeclaration(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s // // unhomed is the census verdict of a $ref site or a collapsed union, which // declaresPositionScoped cannot judge. -func hoistDeclarationHome(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, ref ir.TypeRef, pointer jsontext.Pointer, hint string, home annotation.Home, unhomed bool) (ir.TypeRef, []ir.Diagnostic) { +func hoistDeclarationHome(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, ref ir.TypeRef, pointer jsontext.Pointer, home annotation.Home, unhomed bool) (ir.TypeRef, []ir.Diagnostic) { if home != annotation.HomeOwnNode || s == nil { return ref, nil } @@ -57,7 +57,7 @@ func hoistDeclarationHome(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, ref out.Target = id return out, nil } - id, diags := internAlias(c, ts, s, pointer, hint, ref) + id, diags := internAlias(c, ts, s, pointer, ref) out.Target = id return out, diags } @@ -129,7 +129,7 @@ func declaresValidationOnly(s *oas3.Schema) bool { // annotations stay with the declaring property (FillPropertyDetail). Ownership // is checked before conversion because callers cover a pointer in either order, // and only the one that finds a node may emit conversion diagnostics. -func attachDeclaredAnnotations(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { +func attachDeclaredAnnotations(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { // No node here means a position that reduced to a shared primitive, or one // still being built further up this walk, whose builder attaches the same // declaration once its build returns (GitHub #749). @@ -163,7 +163,7 @@ func attachDeclaredAnnotations(c lowering.Ctx, ts *compile.Types, anchors *Ancho common.Examples = a.Examples } diags = append(diags, recordUnplacedContent(c, &common.Unmodeled, s, td, pointer)...) - diags = append(diags, recordUnexpandedDynamicRef(c, anchors, &common.Unmodeled, s, pointer)...) + diags = append(diags, recordUnexpandedDynamicRef(c, &common.Unmodeled, s, pointer)...) return append(diags, PreserveUnknownKeywords(c, &common.Unmodeled, s, pointer)...) } diff --git a/compilers/openapi/internal/schema/declaration_internal_test.go b/compilers/openapi/internal/schema/declaration_internal_test.go index 8740b01bd..b2a4772de 100644 --- a/compilers/openapi/internal/schema/declaration_internal_test.go +++ b/compilers/openapi/internal/schema/declaration_internal_test.go @@ -22,13 +22,13 @@ func TestAttachDeclaredAnnotations_LeavesANodeStillBeingBuiltToItsBuilder(t *tes l, s, at := lowererDeclaringS(t) var midBuild []ir.Diagnostic - id := internNode(l.ctx, l.types, at, "s", func(cm ir.TypeCommon) ir.TypeDef { - require.NotPanics(t, func() { midBuild = attachDeclaredAnnotations(l.ctx, l.types, &l.anchors, s, at) }) + id := internNode(l.ctx, l.types, at, func(cm ir.TypeCommon) ir.TypeDef { + require.NotPanics(t, func() { midBuild = attachDeclaredAnnotations(l.ctx, l.types, s, at) }) return &ir.Model{TypeCommon: cm} }) assert.Empty(t, midBuild, "a revisit mid-build reports nothing") - assert.Empty(t, attachDeclaredAnnotations(l.ctx, l.types, &l.anchors, s, at)) + assert.Empty(t, attachDeclaredAnnotations(l.ctx, l.types, s, at)) td, ok := l.types.Node(id) require.True(t, ok) assert.Equal(t, "kept", td.Common().Docs.Description, "the builder's own pass attaches it") @@ -42,7 +42,7 @@ func TestRecordDeclarationResidue_LeavesANodeStillBeingBuiltToItsBuilder(t *test l, s, at := lowererDeclaringS(t) var midBuild []ir.Diagnostic - id := internNode(l.ctx, l.types, at, "s", func(cm ir.TypeCommon) ir.TypeDef { + id := internNode(l.ctx, l.types, at, func(cm ir.TypeCommon) ir.TypeDef { require.NotPanics(t, func() { midBuild = recordDeclarationResidue(l.ctx, l.types, s, at, annotation.HomeOwnNode) }) return &ir.Model{TypeCommon: cm} }) diff --git a/compilers/openapi/internal/schema/declaration_test.go b/compilers/openapi/internal/schema/declaration_test.go index dd23adc84..7cb5fa589 100644 --- a/compilers/openapi/internal/schema/declaration_test.go +++ b/compilers/openapi/internal/schema/declaration_test.go @@ -144,7 +144,7 @@ func TestSchema_UnserializableExtension(t *testing.T) { spec := openapitest.ComponentSpec(` S: type: object properties: {a: {type: string}} - x-bad: {1: intkey} + x-bad: {n: .nan} `) doc, diags := lowerSpec(t, spec) require.NotNil(t, doc) diff --git a/compilers/openapi/internal/schema/discriminator.go b/compilers/openapi/internal/schema/discriminator.go index c06669d5c..c711525d1 100644 --- a/compilers/openapi/internal/schema/discriminator.go +++ b/compilers/openapi/internal/schema/discriminator.go @@ -29,20 +29,20 @@ import ( // with no single model, names it by PropertyName alone. variants are a // union's, and nil for a model: what each names decides what the mapping names // (readsThrough). -func lowerDiscriminator(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3.Schema, m *ir.Model, variants []ir.Variant, pointer jsontext.Pointer) (*ir.Discriminator, []ir.Diagnostic) { +func lowerDiscriminator(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, m *ir.Model, variants []ir.Variant, pointer jsontext.Pointer) (*ir.Discriminator, []ir.Diagnostic) { d := s.GetDiscriminator() if d == nil { return nil, nil } variantTypes := variantTargets(variants) - mapping, diags := discriminatorMapping(c, ts, anchors, d, variantTypes, pointer) + mapping, diags := discriminatorMapping(c, ts, d, variantTypes, pointer) disc := &ir.Discriminator{Mapping: mapping} if pid, ok := propIDByName(m, d.GetPropertyName()); ok { disc.Property = pid } else { disc.PropertyName = d.GetPropertyName() } - defaultID, defaultDiags := discriminatorDefault(c, ts, anchors, d, variantTypes, pointer) + defaultID, defaultDiags := discriminatorDefault(c, ts, d, variantTypes, pointer) disc.Default = defaultID return disc, append(diags, defaultDiags...) } @@ -66,7 +66,7 @@ func variantTargets(variants []ir.Variant) map[ir.TypeID]bool { // error diagnostic and is dropped — never a synthesized ID that nothing backs // (issue #14). An all-dropped mapping collapses to nil, preserving infer-by-name // semantics and a clean round-trip. -func discriminatorMapping(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, d *oas3.Discriminator, variantTypes map[ir.TypeID]bool, pointer jsontext.Pointer) (map[string]ir.TypeID, []ir.Diagnostic) { +func discriminatorMapping(c lowering.Ctx, ts *compile.Types, d *oas3.Discriminator, variantTypes map[ir.TypeID]bool, pointer jsontext.Pointer) (map[string]ir.TypeID, []ir.Diagnostic) { m := d.GetMapping() if m == nil || m.Len() == 0 { return nil, nil @@ -74,7 +74,7 @@ func discriminatorMapping(c lowering.Ctx, ts *compile.Types, anchors *AnchorInde var diags []ir.Diagnostic out := make(map[string]ir.TypeID, m.Len()) for tag, target := range m.All() { - id, ok := resolveMappingTarget(c, ts, anchors, d, target, variantTypes) + id, ok := resolveMappingTarget(c, ts, d, target, variantTypes) if !ok { diags = append(diags, c.DiagAt(ir.SeverityError, diag.UnresolvedRef, pointer+ids.Ptr("discriminator", "mapping", tag), @@ -93,12 +93,12 @@ func discriminatorMapping(c lowering.Ctx, ts *compile.Types, anchors *AnchorInde // discriminatorDefault resolves an OpenAPI 3.2 defaultMapping to its target ID, // dropping it with one diagnostic when it does not resolve, as // discriminatorMapping drops an entry. -func discriminatorDefault(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, d *oas3.Discriminator, variantTypes map[ir.TypeID]bool, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { +func discriminatorDefault(c lowering.Ctx, ts *compile.Types, d *oas3.Discriminator, variantTypes map[ir.TypeID]bool, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { dm := d.GetDefaultMapping() if dm == "" { return "", nil } - id, ok := resolveMappingTarget(c, ts, anchors, d, dm, variantTypes) + id, ok := resolveMappingTarget(c, ts, d, dm, variantTypes) if !ok { return "", []ir.Diagnostic{c.DiagAt(ir.SeverityError, diag.UnresolvedRef, pointer+ids.Ptr("discriminator", "defaultMapping"), @@ -134,7 +134,7 @@ func mappingWhy(c lowering.Ctx, d *oas3.Discriminator, target string) string { // $ref written at the discriminator (resolve.Scope.MappingTarget). Hoisting // what it lands on gives the target one pointer-derived ID in either // declaration order (GitHub #530). variantTypes is as readsThrough takes it. -func resolveMappingTarget(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, d *oas3.Discriminator, target string, variantTypes map[ir.TypeID]bool) (ir.TypeID, bool) { +func resolveMappingTarget(c lowering.Ctx, ts *compile.Types, d *oas3.Discriminator, target string, variantTypes map[ir.TypeID]bool) (ir.TypeID, bool) { if c.DeclaresSchema(target) { return componentIDByName(target), true } @@ -143,7 +143,7 @@ func resolveMappingTarget(c lowering.Ctx, ts *compile.Types, anchors *AnchorInde return "", false } at, decl = typePosition(c, at, decl, variantTypes) - return resolveAt(c, ts, anchors, at, decl) + return resolveAt(c, ts, at, decl) } // maxTypePositionHops bounds how many $ref positions typePosition reads @@ -473,9 +473,10 @@ type besideUnionDiscriminator struct { // left alone: it declared none, or it was settled already, by the settlement a // reference deferred when it built the model before its declaration did. // -// It moves only what it could copy (GitHub #144). One with no raw node to copy, -// merged in through a `<<` the raw readers miss (GitHub #395), stays on the -// model rather than vanish, where pass/validate reports the routes it fails. +// It moves only what it could copy (GitHub #144). One converting to no JSON, +// such as one writing an extension `.nan`, is reported as unpreservable and +// stays on the model rather than vanish, where pass/validate reports its +// routes. // // A move leaving a map swaps in its MapT (ir.AsMap), which homes all the model // did; a later settlement leaves the map alone. diff --git a/compilers/openapi/internal/schema/discriminator_internal_test.go b/compilers/openapi/internal/schema/discriminator_internal_test.go index 25f6b627f..5bf75f1bc 100644 --- a/compilers/openapi/internal/schema/discriminator_internal_test.go +++ b/compilers/openapi/internal/schema/discriminator_internal_test.go @@ -79,7 +79,7 @@ func TestDiscriminatorDefault_ResolvesDeclaredComponent(t *testing.T) { l := newRawLowerer(openapitest.DocDeclaring("Cat")) d := &oas3.Discriminator{PropertyName: "kind", DefaultMapping: new("Cat")} - id, diags := discriminatorDefault(l.ctx, l.types, &AnchorIndex{}, d, nil, "/components/schemas/Pet") + id, diags := discriminatorDefault(l.ctx, l.types, d, nil, "/components/schemas/Pet") assert.Equal(t, ids.Document{}.NamedType("/components/schemas/Cat"), id) assert.Empty(t, diags, "a resolvable defaultMapping produces no diagnostic") } @@ -91,7 +91,7 @@ func TestDiscriminatorDefault_DroppedWhenUnresolved(t *testing.T) { // defaultMapping does not resolve and is dropped with one error diagnostic. d := &oas3.Discriminator{PropertyName: "kind", DefaultMapping: new("Missing")} - id, diags := discriminatorDefault(l.ctx, l.types, &AnchorIndex{}, d, nil, "/components/schemas/Pet") + id, diags := discriminatorDefault(l.ctx, l.types, d, nil, "/components/schemas/Pet") assert.Empty(t, id, "an unresolved defaultMapping yields no target") require.Len(t, diags, 1) assert.Equal(t, diag.UnresolvedRef, diags[0].Code) @@ -100,7 +100,7 @@ func TestDiscriminatorDefault_DroppedWhenUnresolved(t *testing.T) { func TestDiscriminatorDefault_EmptyIsNoOp(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) - id, diags := discriminatorDefault(l.ctx, l.types, &AnchorIndex{}, &oas3.Discriminator{PropertyName: "kind"}, nil, "/components/schemas/Pet") + id, diags := discriminatorDefault(l.ctx, l.types, &oas3.Discriminator{PropertyName: "kind"}, nil, "/components/schemas/Pet") assert.Empty(t, id) assert.Empty(t, diags) } @@ -130,7 +130,7 @@ func TestResolveMappingTarget_Branches(t *testing.T) { `)) d := petDiscriminator(t, l) resolveTarget := func(target string) (ir.TypeID, bool) { - return resolveMappingTarget(l.ctx, l.types, &l.anchors, d, target, nil) + return resolveMappingTarget(l.ctx, l.types, d, target, nil) } cat := ids.Document{}.NamedType(ids.Ptr("components", "schemas", "Cat")) @@ -189,7 +189,7 @@ func TestMappingTargetID_NamesAPositionByItsPointer(t *testing.T) { require.True(t, ok, "nothing is interned yet, and the position is named all the same") assert.Equal(t, want, before) - l.diags.AppendAll(LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(LowerComponentSchemas(t.Context(), l.ctx, l.types)) _, finished := l.types.Finish(t.Context()) l.diags.AppendAll(finished) after, ok := mappingTargetID(l.ctx, l.ctx.RefScope(), petDiscriminator(t, l), sub) diff --git a/compilers/openapi/internal/schema/discriminator_test.go b/compilers/openapi/internal/schema/discriminator_test.go index ccfe063b6..28f673a7a 100644 --- a/compilers/openapi/internal/schema/discriminator_test.go +++ b/compilers/openapi/internal/schema/discriminator_test.go @@ -3,6 +3,7 @@ package schema_test import ( "encoding/json/jsontext" "fmt" + "slices" "strings" "testing" @@ -1263,22 +1264,32 @@ func TestDiscriminatorBesideUnion_MultiTypedBodyIsSettled(t *testing.T) { } // TestDiscriminatorBesideUnion_MovesOnlyWhatItCopied pins that the move never -// loses a discriminator it could not keep verbatim: one merged in through a -// `<<`, which the model reads but the raw readers miss (GitHub #395), stays on -// the model, and nothing claims a copy that was never made. +// loses a discriminator it could not keep verbatim. One merged in through `<<` +// is read as a written one is (GitHub #395), so Pet lowers as it does with the +// discriminator written in place. One that converts to no JSON, through an +// extension written `.nan`, is reported as unpreservable and stays on the +// model, and nothing claims a copy never made. func TestDiscriminatorBesideUnion_MovesOnlyWhatItCopied(t *testing.T) { t.Parallel() - spec := openapitest.ComponentSpec(cat + dog + - " Shared: &shared\n discriminator: {propertyName: kind, mapping: {cat: '#/components/schemas/Cat'}}\n" + - " Pet:\n <<: *shared\n type: object\n" + petBranches) - doc, diags := parseFull(t, spec) + const disc = "{propertyName: kind, mapping: {cat: '#/components/schemas/Cat'}}" + written, _ := parseFull(t, openapitest.ComponentSpec(cat+dog+petWith(disc))) + merged, diags := parseFull(t, openapitest.ComponentSpec(cat+dog+ + " Pet:\n <<: {discriminator: "+disc+"}\n type: object\n"+petBranches)) openapitest.RequireNoErrorDiags(t, diags) + assert.Empty(t, cmp.Diff(typeByName(written, "Pet"), typeByName(merged, "Pet")), + "a merged discriminator lowers as a written one") + doc, diags := parseFull(t, openapitest.ComponentSpec(cat+dog+ + petWith("{propertyName: kind, mapping: {cat: '#/components/schemas/Cat'}, x-weight: .nan}"))) pet := petModel(t, doc) require.NotNil(t, pet.Discriminator, "the model read it, so it is not lost") assert.Equal(t, "kind", pet.Discriminator.PropertyName) assert.NotContains(t, pet.Unmodeled, "openapi:discriminator") - assert.False(t, openapitest.HasDiagCodeAt(diags, diag.DegradedConstruct, movedPointer), "no move is announced") + assert.True(t, openapitest.HasDiagCodeAt(diags, diag.UnpreservableConstruct, movedPointer), + "the failed copy is reported: %+v", diags) + assert.False(t, slices.ContainsFunc(diags, func(d ir.Diagnostic) bool { + return strings.Contains(d.Message, "beside the union it was written for") + }), "no move is announced") } // TestDiscriminatorBesideUnion_UnreachableBranchesAreNoRoute pins that a diff --git a/compilers/openapi/internal/schema/dynamicref.go b/compilers/openapi/internal/schema/dynamicref.go index a5bbad5e5..090f65920 100644 --- a/compilers/openapi/internal/schema/dynamicref.go +++ b/compilers/openapi/internal/schema/dynamicref.go @@ -5,7 +5,6 @@ import ( "fmt" "net/url" "slices" - "strconv" "strings" oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" @@ -32,7 +31,7 @@ import ( // TypeID is stable whatever lowers first (ir-design §4.3). A $dynamicRef // co-declared with a $ref, oneOf/anyOf or a shape is irreducible too, as is // one another document holds. -func dynamicExpansion(c lowering.Ctx, anchors *AnchorIndex, s *oas3.Schema, pointer jsontext.Pointer) (target ir.TypeID, why string, ok bool, diags []ir.Diagnostic) { +func dynamicExpansion(c lowering.Ctx, s *oas3.Schema, pointer jsontext.Pointer) (target ir.TypeID, why string, ok bool, diags []ir.Diagnostic) { name, why, ok := dynamicRefName(s) if !ok { return "", why, false, nil @@ -44,7 +43,7 @@ func dynamicExpansion(c lowering.Ctx, anchors *AnchorIndex, s *oas3.Schema, poin return "", "it is written in another document, whose $dynamicAnchor it names, and only the source's " + "anchors are indexed", false, nil } - at, why, ok, diags := soleAnchorSite(c, anchors, name) + at, why, ok, diags := soleAnchorSite(c, name) if !ok { return "", why, false, diags } @@ -52,7 +51,7 @@ func dynamicExpansion(c lowering.Ctx, anchors *AnchorIndex, s *oas3.Schema, poin if !handled || !resolved { return "", unnamedAnchorSiteWhy(name, at), false, diags } - chainWhy, chainOK, chainDiags := dynamicChainVerdict(c, anchors, at, pointer) + chainWhy, chainOK, chainDiags := dynamicChainVerdict(c, at, pointer) diags = append(diags, chainDiags...) if !chainOK { return "", chainWhy, false, diags @@ -136,12 +135,12 @@ func dynamicFragment(ref string) (name, why string, ok bool) { // target back under the evaluation path's control, which no static lowering can // resolve (docs/compilers/openapi.md#dynamic-ref). // -// The index is a parameter because the memo is state the caller owns. Its -// diagnostics come back on every path, including the two answering "no single -// declaration": the walk bound they report is a fact about the document, not -// about this lookup's verdict, so a failed lookup must not swallow it. -func soleAnchorSite(c lowering.Ctx, anchors *AnchorIndex, name string) (at jsontext.Pointer, why string, ok bool, diags []ir.Diagnostic) { - sites, diags := anchors.sites(c, name) +// The index's diagnostics come back on every path, including the two answering +// "no single declaration": the walk bound they report is a fact about the +// document, not about this lookup's verdict, so a failed lookup must not +// swallow it. +func soleAnchorSite(c lowering.Ctx, name string) (at jsontext.Pointer, why string, ok bool, diags []ir.Diagnostic) { + sites, diags := c.DynamicAnchorSites(name) if len(sites) == 0 { return "", fmt.Sprintf("no $dynamicAnchor %q is declared in this document", name), false, diags } @@ -161,7 +160,7 @@ func soleAnchorSite(c lowering.Ctx, anchors *AnchorIndex, name string) (at jsont // // The loop is bounded: cur only takes anchor-index values, and each turn // returns or adds one to seen. -func dynamicChainVerdict(c lowering.Ctx, anchors *AnchorIndex, at, from jsontext.Pointer) (why string, ok bool, diags []ir.Diagnostic) { +func dynamicChainVerdict(c lowering.Ctx, at, from jsontext.Pointer) (why string, ok bool, diags []ir.Diagnostic) { if declaresResourceIDAbove(c, from) { return resourceBoundaryWhy(from), false, nil } @@ -175,7 +174,7 @@ func dynamicChainVerdict(c lowering.Ctx, anchors *AnchorIndex, at, from jsontext return resourceBoundaryWhy(cur), false, diags } seen[cur] = true - next, hops, hopDiags := dynamicHop(c, anchors, cur) + next, hops, hopDiags := dynamicHop(c, cur) diags = append(diags, hopDiags...) if !hops { break @@ -201,7 +200,7 @@ func resourceBoundaryWhy(at jsontext.Pointer) string { // it carry none. Once it is consulted they come back even if the chain ends // there: whether the hop happened and whether the index was complete are // independent answers. -func dynamicHop(c lowering.Ctx, anchors *AnchorIndex, at jsontext.Pointer) (jsontext.Pointer, bool, []ir.Diagnostic) { +func dynamicHop(c lowering.Ctx, at jsontext.Pointer) (jsontext.Pointer, bool, []ir.Diagnostic) { s := componentSchemaAt(c, at) if s == nil { return "", false, nil @@ -210,7 +209,7 @@ func dynamicHop(c lowering.Ctx, anchors *AnchorIndex, at jsontext.Pointer) (json if !ok { return "", false, nil } - next, nextDiags := anchors.sites(c, name) + next, nextDiags := c.DynamicAnchorSites(name) if len(next) != 1 { return "", false, nextDiags } @@ -254,8 +253,8 @@ func declaresResourceIDAbove(c lowering.Ctx, pointer jsontext.Pointer) bool { // (the irreducible half of docs/compilers/openapi.md#dynamic-ref). An expanded // one is already the position's type and must not also be preserved — that // would tell a consumer the compiler ignored it. -func recordUnexpandedDynamicRef(c lowering.Ctx, anchors *AnchorIndex, p *ir.Unmodeled, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { - _, why, expanded, diags := dynamicExpansion(c, anchors, s, pointer) +func recordUnexpandedDynamicRef(c lowering.Ctx, p *ir.Unmodeled, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { + _, why, expanded, diags := dynamicExpansion(c, s, pointer) if expanded { return diags } @@ -274,136 +273,3 @@ func recordUnexpandedDynamicRef(c lowering.Ctx, anchors *AnchorIndex, p *ir.Unmo func declaresDynamicRef(s *oas3.Schema) bool { return annotation.RawPropertyNode(s, "$dynamicRef") != nil } - -// AnchorIndex memoizes the document's $dynamicAnchor index. -// -// It is shared and mutated, which micro-compiler-design §4 expected of nothing -// but the interning table. It is a memo, not an accumulator: the value it -// caches is a pure function of the document. It -// stays out of the immutable context (§4.1) because building it emits a -// diagnostic, which would report on documents that never write $dynamicRef, and -// the walk is a poor trade for a keyword almost no document uses. -type AnchorIndex struct { - byName map[string][]jsontext.Pointer -} - -// sites returns the pointers declaring the named $dynamicAnchor, building the -// index on first use and announcing once when a bound stopped the walk short. -// -// A truncated index can only undercount, and every verdict resting on it reads -// an undercount as "declared exactly once" — the one count that expands. So the -// warning is what tells a reader that an expansion reported below was decided -// against an index nothing verified. -func (a *AnchorIndex) sites(c lowering.Ctx, name string) ([]jsontext.Pointer, []ir.Diagnostic) { - if a.byName != nil { - return a.byName[name], nil - } - index, complete := dynamicAnchors(c.Doc.GetRootNode()) - a.byName = index - if complete { - return a.byName[name], nil - } - return a.byName[name], []ir.Diagnostic{c.DiagAt(ir.SeverityWarning, diag.DegradedConstruct, "", - "the $dynamicAnchor index stopped at its walk bounds (%d levels, %d nodes); "+ - "a $dynamicRef expanded below is not verified to name the document's only anchor of its name", - maxDynamicAnchorDepth, maxDynamicAnchorNodes)} -} - -// maxDynamicAnchorDepth bounds the raw-source walk that indexes $dynamicAnchor -// declarations (styleguide bounded-recursion rule). Document trees nest far -// shallower — a schema under a media type under a response is about ten levels — -// so the cap only ever fires on a pathological structure. -const maxDynamicAnchorDepth = 512 - -// maxDynamicAnchorNodes bounds how many nodes that walk visits in total. Depth -// stopped bounding it once the walk began following YAML aliases: an alias graph -// is a DAG, so a shallow document can present far more paths than it has nodes. -// Both bounds report the same incomplete-index warning when they fire. -const maxDynamicAnchorNodes = 1 << 20 - -// dynamicAnchors indexes every $dynamicAnchor in the raw source by name, mapping -// it to the JSON pointers that declare it, in document order, and reports -// whether the walk ran to completion. It reads the raw tree because oas3.Schema -// has no field for the keyword (docs/parser-library.md#unmodeled-keywords). -func dynamicAnchors(root *yaml.Node) (map[string][]jsontext.Pointer, bool) { - w := newAnchorWalk(maxDynamicAnchorNodes) - w.walk(root, "", 0) - return w.out, !w.truncated && !w.view.Exhausted() -} - -// anchorWalk is the state of one $dynamicAnchor index build: the source view, -// the index under construction, and what is left of the visit budget. -// -// It reads mappings through a nodeview.View, so a `<<` merge key and a YAML alias -// contribute the anchors they carry to every position that pulls them in — -// which is what the parser downstream sees, and what makes "declared exactly -// once" a count of what the document declares rather than of what it spells -// out. The count decides whether a $dynamicRef expands, so an anchor reached -// only through an alias must not be invisible to it. -type anchorWalk struct { - view *nodeview.View - out map[string][]jsontext.Pointer - budget int - truncated bool -} - -// newAnchorWalk returns a walk that will visit at most budget nodes. -func newAnchorWalk(budget int) *anchorWalk { - return &anchorWalk{view: nodeview.New(), out: map[string][]jsontext.Pointer{}, budget: budget} -} - -// walk indexes the anchors n declares, under the pointer of the mapping -// declaring each. -func (w *anchorWalk) walk(n *yaml.Node, pointer jsontext.Pointer, depth int) { - n = nodeview.Deref(n) - if n == nil || !w.charge(depth) { - return - } - switch n.Kind { - case yaml.DocumentNode: - for _, c := range n.Content { - w.walk(c, pointer, depth+1) - } - case yaml.SequenceNode: - for i, c := range n.Content { - w.walk(c, pointer+ids.Ptr(strconv.Itoa(i)), depth+1) - } - case yaml.MappingNode: - w.walkMapping(n, pointer, depth) - default: - // A scalar declares no anchor and has no children; an alias survived - // nodeview.Deref only by dangling, so it has nothing to stand in for. - } -} - -// walkMapping reads one mapping's effective pairs: a $dynamicAnchor names this -// mapping, and every other value is walked in turn. -func (w *anchorWalk) walkMapping(n *yaml.Node, pointer jsontext.Pointer, depth int) { - for _, p := range w.view.MappingPairs(n) { - if p.Key == "$dynamicAnchor" { - w.record(p.Val, pointer) - continue - } - w.walk(p.Val, pointer+ids.Ptr(p.Key), depth+1) - } -} - -// record indexes the anchor val names at pointer. A value that is not a -// non-empty scalar names nothing a $dynamicRef could spell. -func (w *anchorWalk) record(val *yaml.Node, pointer jsontext.Pointer) { - if val == nil || val.Kind != yaml.ScalarNode || val.Value == "" { - return - } - w.out[val.Value] = append(w.out[val.Value], pointer) -} - -// charge reports whether the walk may read one more node, recording the refusal -// when it may not so the caller of dynamicAnchors learns the index is partial. -func (w *anchorWalk) charge(depth int) bool { - if depth > maxDynamicAnchorDepth || w.budget == 0 { - w.truncated = true - return false - } - w.budget-- - return true -} diff --git a/compilers/openapi/internal/schema/dynamicref_bench_test.go b/compilers/openapi/internal/schema/dynamicref_bench_test.go deleted file mode 100644 index 0a3b7d2d9..000000000 --- a/compilers/openapi/internal/schema/dynamicref_bench_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package schema - -import ( - "bytes" - "os" - "testing" - - "github.com/speakeasy-api/openapi/marshaller" - soa "github.com/speakeasy-api/openapi/openapi" -) - -// BenchmarkAnchorWalk measures what deriving the $dynamicAnchor index at entry -// would cost a document that never asks for it. -// -// The index stays a lazy memo rather than living in the immutable context -// (micro-compiler-design §4.1): the walk is not free, almost no document writes -// $dynamicRef, and building it emits a diagnostic that at entry would reach -// documents that never use the keyword. -// -// Read the number against BenchmarkCompile_Petstore in compilers/openapi; the -// design's claim is a ratio. -func BenchmarkAnchorWalk(b *testing.B) { - // Four levels up, not two: this package sits at compilers/openapi/internal/ - // schema, and the corpus is at the repo root. A missing or unparseable - // fixture stops the benchmark rather than skipping it — a skip is silent - // without -v and exits 0, which is how the shorter path went unnoticed. - data, err := os.ReadFile("../../../../testdata/conformance/openapi/allof-inline-merge.yaml") - if err != nil { - b.Fatalf("corpus fixture unavailable: %v", err) - } - var doc soa.OpenAPI - if _, err := marshaller.Unmarshal(b.Context(), bytes.NewReader(data), &doc); err != nil { - b.Fatalf("fixture does not parse: %v", err) - } - root := doc.GetRootNode() - - b.ReportAllocs() - for b.Loop() { - if _, complete := dynamicAnchors(root); !complete { - b.Fatal("the fixture should not exhaust the walk bounds") - } - } -} diff --git a/compilers/openapi/internal/schema/dynamicref_internal_test.go b/compilers/openapi/internal/schema/dynamicref_internal_test.go index 24e464a94..7aaa0f4d7 100644 --- a/compilers/openapi/internal/schema/dynamicref_internal_test.go +++ b/compilers/openapi/internal/schema/dynamicref_internal_test.go @@ -2,17 +2,13 @@ package schema import ( "encoding/json/jsontext" - "strings" "testing" - "github.com/google/go-cmp/cmp" soa "github.com/speakeasy-api/openapi/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v3" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" - "github.com/dexpace/morphic/ir" ) // TestDeclaresResourceIDAbove_WithoutARawTree pins the guard on the path walk. @@ -93,168 +89,6 @@ components: } } -// TestDynamicAnchors_WalksEveryNodeShape drives the raw-tree walk over the -// shapes a YAML document can present, rather than only the mappings a schema -// happens to be written as. The walk reads the raw tree because oas3.Schema has -// no $dynamicAnchor field, so it meets whatever the source wrote — a sequence, a -// bare scalar, a non-string key — and must index anchors under a sequence while -// declining the rest without wandering off the tree. -func TestDynamicAnchors_WalksEveryNodeShape(t *testing.T) { - t.Parallel() - cases := []struct { - name string - node *yaml.Node - want map[string][]jsontext.Pointer - }{ - {"a nil node yields nothing", nil, map[string][]jsontext.Pointer{}}, - { - "a bare scalar declares no anchor", - openapitest.YAMLNode(t, `just-a-string`), - map[string][]jsontext.Pointer{}, - }, - { - "a sequence indexes its elements by ordinal", - openapitest.YAMLNode(t, "- {$dynamicAnchor: first}\n- {other: 1}\n- {$dynamicAnchor: third}\n"), - map[string][]jsontext.Pointer{"first": {"/0"}, "third": {"/2"}}, - }, - { - "a sequence element standing in for a mapping is followed", - openapitest.YAMLNode(t, "- &a {$dynamicAnchor: first}\n- *a\n"), - map[string][]jsontext.Pointer{"first": {"/0", "/1"}}, - }, - { - "a non-string key cannot name a keyword and is skipped", - openapitest.YAMLNode(t, "? [a, b]\n: {$dynamicAnchor: buried}\n$dynamicAnchor: reached\n"), - map[string][]jsontext.Pointer{"reached": {""}}, - }, - { - "an empty anchor name is not indexed", - openapitest.YAMLNode(t, `{$dynamicAnchor: ""}`), - map[string][]jsontext.Pointer{}, - }, - { - "a non-scalar anchor value is not indexed", - openapitest.YAMLNode(t, `{$dynamicAnchor: [a]}`), - map[string][]jsontext.Pointer{}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - got, complete := dynamicAnchors(tc.node) - assert.Empty(t, cmp.Diff(tc.want, got)) - assert.True(t, complete, "nothing here reaches a walk bound") - }) - } -} - -// TestDynamicAnchors_CountsWhatAnAliasBringsIn pins the count the whole -// expansion rests on. A YAML alias and a `<<` merge key each hand a second -// position the anchor the first one wrote, and the parser downstream reads both -// positions as declaring it — so an index that walked only the spelled-out tree -// would report one declaration where the document has three, and "declared -// exactly once" would expand a reference that is genuinely ambiguous. -func TestDynamicAnchors_CountsWhatAnAliasBringsIn(t *testing.T) { - t.Parallel() - cases := []struct { - name, source string - want []jsontext.Pointer - }{ - { - name: "an alias re-declares the anchor at its own position", - source: "A: &base {$dynamicAnchor: tail}\nB: *base\n", - want: []jsontext.Pointer{"/A", "/B"}, - }, - { - name: "a merge key does too", - source: "A: &base {$dynamicAnchor: tail}\nB: {<<: *base, description: merged}\n", - want: []jsontext.Pointer{"/A", "/B"}, - }, - { - name: "an explicit key still beats the merged one", - source: "A: &base {$dynamicAnchor: tail}\nB: {<<: *base, $dynamicAnchor: own}\n", - want: []jsontext.Pointer{"/A"}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - got, complete := dynamicAnchors(openapitest.YAMLNode(t, tc.source)) - assert.True(t, complete) - assert.Equal(t, tc.want, got["tail"]) - }) - } -} - -// TestDynamicAnchors_DocumentNodeUnwraps pins that the walk accepts a whole -// document as well as a content node, since dynamicAnchors is handed whatever -// the loader kept. -func TestDynamicAnchors_DocumentNodeUnwraps(t *testing.T) { - t.Parallel() - var doc yaml.Node - require.NoError(t, yaml.Unmarshal([]byte("$dynamicAnchor: top\n"), &doc)) - require.Equal(t, yaml.DocumentNode, doc.Kind) - - got, complete := dynamicAnchors(&doc) - assert.Equal(t, map[string][]jsontext.Pointer{"top": {""}}, got) - assert.True(t, complete) -} - -// TestDynamicAnchors_StopsAtTheDepthCap pins the bound. A raw tree deeper than -// the cap is a legal document, so the walk must stop rather than recur to -// exhaustion; anchors above the cap are still indexed, and the truncation is -// reported rather than passed off as a complete index. -func TestDynamicAnchors_StopsAtTheDepthCap(t *testing.T) { - t.Parallel() - deep, shallow := nestedAnchor(maxDynamicAnchorDepth+2), nestedAnchor(0) - - deepIndex, deepComplete := dynamicAnchors(deep) - assert.NotContains(t, deepIndex, "toodeep", - "the walk stops at the cap instead of recurring to exhaustion") - assert.False(t, deepComplete, "and says so, since an unseen anchor undercounts every name") - - shallowIndex, shallowComplete := dynamicAnchors(shallow) - assert.Contains(t, shallowIndex, "toodeep", - "an anchor within the cap is still indexed, so the cap is what excluded the other") - assert.True(t, shallowComplete) -} - -// TestAnchorWalk_StopsAtTheNodeBudget pins the other bound. Depth alone stopped -// bounding the walk once it began following aliases, since an alias DAG presents -// many more paths than the tree has nodes; the budget is what caps the total. -func TestAnchorWalk_StopsAtTheNodeBudget(t *testing.T) { - t.Parallel() - source := openapitest.YAMLNode(t, "a: {$dynamicAnchor: first}\nb: {$dynamicAnchor: second}\n") - - w := newAnchorWalk(2) // the root mapping and its first value, and no more - w.walk(source, "", 0) - - assert.True(t, w.truncated, "the budget ran out mid-walk") - assert.NotContains(t, w.out, "second", "so the anchors past it are missing") -} - -// TestDynamicAnchorIndex_ReportsATruncatedWalk pins what a reader is told when -// the index is partial. Truncation only ever drops declarations, and a dropped -// duplicate reads as "declared exactly once" — the count that expands — so an -// unreported truncation would turn an ambiguous reference into a confident one. -func TestDynamicAnchorIndex_ReportsATruncatedWalk(t *testing.T) { - t.Parallel() - // The walk descends into every key, so an extension at the document root - // nests the tree past the cap without the schema lowering ever seeing it. - deep := strings.Repeat("{a: ", maxDynamicAnchorDepth) + "1" + strings.Repeat("}", maxDynamicAnchorDepth) - l, diags := loweredFor(t, openapitest.ComponentSpec(" A: {type: string}\n")+"x-deep: "+deep+"\n") - openapitest.RequireNoErrorDiags(t, diags) - - _, got := l.anchors.sites(l.ctx, "absent") - require.NotNil(t, l.anchors.byName, "the index is built even when partial") - require.Len(t, got, 1, "the truncation is announced exactly once: %+v", got) - assert.Equal(t, ir.SeverityWarning, got[0].Severity) - assert.Contains(t, got[0].Message, "not verified") - - _, again := l.anchors.sites(l.ctx, "absent") - assert.Empty(t, again, "and the cached index does not announce it again") -} - // TestComponentSchemaAt_OnlyATopLevelComponentPointerHasABody pins the split // the function exists for: a component pointer has a body, and a pointer into // that same component does not. @@ -352,12 +186,12 @@ func TestDynamicHop_HopsOnlyWhenExactlyOneAnchorSiteIsNamed(t *testing.T) { l, diags := loweredFor(t, openapitest.ComponentSpec(tc.schemas)) openapitest.RequireNoErrorDiags(t, diags) if tc.anchor != "" { - sites, siteDiags := l.anchors.sites(l.ctx, tc.anchor) + sites, siteDiags := l.ctx.DynamicAnchorSites(tc.anchor) require.Len(t, sites, tc.wantSites, "the fixture must set up the case it is named for") require.Empty(t, siteDiags) } - next, hops, hopDiags := dynamicHop(l.ctx, &l.anchors, tc.at) + next, hops, hopDiags := dynamicHop(l.ctx, tc.at) assert.Equal(t, tc.wantNext != "", hops) assert.Equal(t, tc.wantNext, next) diff --git a/compilers/openapi/internal/schema/enum.go b/compilers/openapi/internal/schema/enum.go index 92609ee73..426b61a0f 100644 --- a/compilers/openapi/internal/schema/enum.go +++ b/compilers/openapi/internal/schema/enum.go @@ -31,9 +31,9 @@ import ( // (ir-design §3.3) when schemaAdmitsNull holds, the predicate every Nullable // site shares. A sibling `type: string` forbids that null, so it stays and the // set becomes a Union. -func lowerEnum(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerEnum(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { // The degenerate list first, then the budget. They cannot both hold — a // member count of zero exceeds no positive budget — so the order decides // nothing, but reading the empty case beside the members it lacks is @@ -52,7 +52,7 @@ func lowerEnum(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsonte diags = append(diags, enumRepeats(c, s.GetEnum(), pointer)...) members, memberPrim, ok := enumMembers(s.GetEnum(), schemaAdmitsNull(c, s), c.IDs(), pointer) if !ok { - def, enumDiags := enumAsUnion(c, ts, s, common, pointer, hint) + def, enumDiags := enumAsUnion(c, ts, s, common, pointer) diags = append(diags, enumDiags...) return def } @@ -136,18 +136,17 @@ func enumMembers(nodes []values.Value, dropNull bool, d ids.Document, pointer js // enumAsUnion lowers a heterogeneous or non-scalar enum to an exclusive Union of // hoisted Literals, emitting one info diagnostic. -func enumAsUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer, hint string) (ir.TypeDef, []ir.Diagnostic) { +func enumAsUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer) (ir.TypeDef, []ir.Diagnostic) { diags := []ir.Diagnostic{c.DiagAt(ir.SeverityInfo, diag.DegradedConstruct, pointer, "heterogeneous or non-scalar enum lowered as a union of literals")} nodes := s.GetEnum() variants := make([]ir.Variant, 0, len(nodes)) for i, node := range nodes { - vh := compile.SubHint(hint, strconv.Itoa(i)) lptr := pointer + ids.Ptr("enum", strconv.Itoa(i)) - litID, litDiags := hoistLiteral(c, ts, node, lptr, vh) + litID, litDiags := hoistLiteral(c, ts, node, lptr) diags = append(diags, litDiags...) variants = append(variants, ir.Variant{ - Name: compile.NamingHint(vh), + Name: compile.NamingHint(lowering.NameAt(c, lptr)), Type: ir.TypeRef{Target: litID}, }) } @@ -168,7 +167,7 @@ func enumAsUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, common ir.Ty // // It returns the interned ID, plus the diagnostic an unconvertible node // produces — none when the node converts. -func hoistLiteral(c lowering.Ctx, ts *compile.Types, node values.Value, pointer jsontext.Pointer, hint string, +func hoistLiteral(c lowering.Ctx, ts *compile.Types, node values.Value, pointer jsontext.Pointer, ) (ir.TypeID, []ir.Diagnostic) { // Captured from inside the build rather than reported around it, which keeps // the report tied to the node actually being constructed. Reporting eagerly @@ -176,7 +175,7 @@ func hoistLiteral(c lowering.Ctx, ts *compile.Types, node values.Value, pointer // diagnostic, which Diags.Append drops — so nothing observable rests on this; // it is the honest place for it, not a load-bearing one. var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { val, err := value.FromNode(node) if err != nil { diags = append(diags, c.DiagAt(ir.SeverityWarning, diag.DegradedConstruct, pointer, diff --git a/compilers/openapi/internal/schema/helpers_internal_test.go b/compilers/openapi/internal/schema/helpers_internal_test.go index d11224626..f40ccdf5e 100644 --- a/compilers/openapi/internal/schema/helpers_internal_test.go +++ b/compilers/openapi/internal/schema/helpers_internal_test.go @@ -8,10 +8,10 @@ import ( yaml "gopkg.in/yaml.v3" "github.com/dexpace/morphic/compilers/compile" - "github.com/dexpace/morphic/compilers/openapi/internal/annotation" "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/load" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/compilers/openapi/internal/overlay" "github.com/dexpace/morphic/ir" @@ -26,41 +26,25 @@ import ( // exercises the interning-lookup paths rather than the named-component path. const deepPointer = "/components/schemas/Obj/properties/inner" -// nestedAnchor builds a mapping chain of the given depth with a $dynamicAnchor -// at the bottom. It is built rather than parsed because nesting this deep in -// source text is indentation arithmetic, and duplicate keys at one level would -// not nest at all. -func nestedAnchor(depth int) *yaml.Node { - n := &yaml.Node{ - Kind: yaml.MappingNode, - Content: []*yaml.Node{openapitest.StrNode("$dynamicAnchor"), openapitest.StrNode("toodeep")}, - } - for range depth + 1 { - n = &yaml.Node{Kind: yaml.MappingNode, Content: []*yaml.Node{openapitest.StrNode("k"), n}} - } - return n -} - // useValue parses src — an anchor plus a `use:` key written against it — and // returns the raw node at `use` undereferenced, so a value spelled `*anchor` // arrives as the alias node a branch reader is really handed rather than as the // mapping it stands for. func useValue(t *testing.T, src string) *yaml.Node { t.Helper() - node := annotation.RawChildNode(openapitest.YAMLNode(t, src), "use") + node := nodeview.WrittenChildren(openapitest.YAMLNode(t, src))["use"] require.NotNil(t, node, `src writes no "use" key`) return node } // lowerer is test scaffolding, not a compiler type. The compiler has no such -// struct: every lowering is a function of the context, the registry and the -// memos, and the caller owns the document being built (#177). +// struct: every lowering is a function of the context and the registry, and +// the caller owns the document being built (#177). type lowerer struct { - ctx lowering.Ctx - out *ir.Document - types *compile.Types - diags compile.Diags - anchors AnchorIndex + ctx lowering.Ctx + out *ir.Document + types *compile.Types + diags compile.Diags } // lowererOver is the only place the fixture's fields are initialised. Both @@ -92,7 +76,7 @@ func loweredFor(t *testing.T, src string) (*lowerer, []ir.Diagnostic) { func lowerSpec(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { t.Helper() l, diags := loweredFor(t, src) - l.diags.AppendAll(LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(LowerComponentSchemas(t.Context(), l.ctx, l.types)) doc := l.document(t) return doc, append(diags, l.diags.List()...) } diff --git a/compilers/openapi/internal/schema/helpers_test.go b/compilers/openapi/internal/schema/helpers_test.go index 34d9273c8..238769970 100644 --- a/compilers/openapi/internal/schema/helpers_test.go +++ b/compilers/openapi/internal/schema/helpers_test.go @@ -52,15 +52,14 @@ func parseFull(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { } // lowerer is test scaffolding, not a compiler type. The compiler has no such -// struct: every lowering is a function of the context, the registry and the -// memos, and the caller owns the document being built (#177). Tests that drive -// one lowering in isolation still want those in one place. +// struct: every lowering is a function of the context and the registry, and +// the caller owns the document being built (#177). Tests that drive one +// lowering in isolation still want those in one place. type lowerer struct { - ctx lowering.Ctx - out *ir.Document - types *compile.Types - diags compile.Diags - anchors schema.AnchorIndex + ctx lowering.Ctx + out *ir.Document + types *compile.Types + diags compile.Diags } // lowererOver is the only place the fixture's fields are initialised. Both @@ -92,7 +91,7 @@ func loweredFor(t *testing.T, src string) (*lowerer, []ir.Diagnostic) { func lowerSpec(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { t.Helper() l, diags := loweredFor(t, src) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) doc := l.document(t) return doc, append(diags, l.diags.List()...) } @@ -107,7 +106,7 @@ func lowerSpec(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { func lowerSpecOnly(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) { t.Helper() l, _ := loweredFor(t, src) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) doc := l.document(t) return doc, l.diags.List() } diff --git a/compilers/openapi/internal/schema/hoist.go b/compilers/openapi/internal/schema/hoist.go index a91b7adba..14baeec46 100644 --- a/compilers/openapi/internal/schema/hoist.go +++ b/compilers/openapi/internal/schema/hoist.go @@ -1,23 +1,15 @@ package schema import ( - "cmp" "encoding/json/jsontext" - "net/url" - "slices" - "strconv" - "strings" - "unicode/utf8" oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" "github.com/dexpace/morphic/compilers/compile" "github.com/dexpace/morphic/compilers/openapi/internal/annotation" - "github.com/dexpace/morphic/compilers/openapi/internal/bind/typed" "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" - "github.com/dexpace/morphic/compilers/openapi/internal/resolve" "github.com/dexpace/morphic/ir" ) @@ -45,24 +37,22 @@ func registeredNode(c lowering.Ctx, ts *compile.Types, id ir.TypeID, pointer jso // internNode is the single hoisting entry point: it derives pointer's stable // TypeID (ids.Document.ForPointer) and shared TypeCommon (commonFor) once, then // interns build's result under that ID. build receives the built TypeCommon, so -// it never re-derives pointer, hint or id. +// it never re-derives pointer, name or id. // // A $ref promises the node rather than building it (resolvePointer), so every // lowering reaching pointer does so in the context its position gives it, and -// the first build stands for all. A node a $ref hoists where no lowering -// interned one takes the hint the position's own lowering recorded -// (hoistSubSchema), so hint is always the position's. -func internNode(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer, hint string, - build func(common ir.TypeCommon) ir.TypeDef, +// the first build stands for all. +func internNode(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer, build func(common ir.TypeCommon) ir.TypeDef, ) ir.TypeID { id := c.IDs().ForPointer(pointer) - return ts.Intern(c.IDs().Coordinate(pointer), id, func() ir.TypeDef { return build(commonFor(c, id, pointer, hint)) }) + return ts.Intern(c.IDs().Coordinate(pointer), id, func() ir.TypeDef { return build(commonFor(c, id, pointer)) }) } // commonFor builds the TypeCommon shared by every hoisted node at pointer. A // top-level component schema is named (source + canonical words); any deeper -// inline position is anonymous and carries only the context-derived hint. -func commonFor(c lowering.Ctx, id ir.TypeID, pointer jsontext.Pointer, hint string) ir.TypeCommon { +// inline position is anonymous and carries only the hint its coordinate names +// it by (lowering.NameAt), whichever lowering builds it. +func commonFor(c lowering.Ctx, id ir.TypeID, pointer jsontext.Pointer) ir.TypeCommon { common := ir.TypeCommon{ ID: id, Provenance: c.ProvenanceAt(pointer), @@ -71,7 +61,7 @@ func commonFor(c lowering.Ctx, id ir.TypeID, pointer jsontext.Pointer, hint stri common.Name = compile.NamingFor(name) } else { common.Anonymous = true - common.Name = compile.NamingHint(hint) + common.Name = compile.NamingHint(lowering.NameAt(c, pointer)) } return common } @@ -81,222 +71,13 @@ func commonFor(c lowering.Ctx, id ir.TypeID, pointer jsontext.Pointer, hint stri // target still owns a resolvable node at its own TypeID. It carries the value // constraints s writes, and the exclusive modifier no Constraints field holds, // for the reason hoistScalar records. -func internAlias(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, hint string, - target ir.TypeRef, +func internAlias(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, target ir.TypeRef, ) (ir.TypeID, []ir.Diagnostic) { var kept ir.Unmodeled cons, diags := ConstraintsOf(c, &kept, s, pointer) - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { common.Unmodeled = annotation.MergeUnmodeled(common.Unmodeled, kept) return &ir.Scalar{TypeCommon: common, Base: new(target), Constraints: cons} }) return id, diags } - -// subSchemaHint names the node a $ref'd sub-schema pointer owns: the hint the -// position takes for where it is (ownHint), or, at a position that takes its -// target's name, the hint of the node its own $ref resolves to (targetHint). -// -// It names a position no declaration reached: where one did, hoistSubSchema -// takes the hint that declaration recorded. Beneath /components/schemas -// positionHint replays the declaration's hint exactly; elsewhere it names the -// position by the tokens it can read. -func subSchemaHint(refs *typed.Bindings, decl *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) string { - hint, follow := ownHint(decl, pointer) - if !follow { - return hint - } - return cmp.Or(targetHint(refs, decl), hint) -} - -// ownHint returns the hint the position at pointer takes for where it is, and -// whether a $ref written there names it after its target instead. decl is the -// schema written there, if known. -// -// Only a composition branch holding a $ref is named after its target, as the -// composition names it (branchHint); every other position is named for where it -// is, as positionHint replays it. -func ownHint(decl *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) (hint string, follow bool) { - hint, branch := positionHint(pointer) - return hint, branch && decl != nil && decl.IsReference() -} - -// targetHint returns the hint of the node a $ref resolves to, so a union -// variant or allOf branch is named as the type it holds is (GitHub #521). -// -// Each position on the way is read as subSchemaHint names it (ownHint); a -// composition branch holding a $ref moves the walk on to its target, keeping -// the last name found. The fragment is percent-decoded, then RFC 6901 -// unescaped, so `#/components/schemas/Cat~1Dog` suggests "Cat/Dog" (GitHub -// #505); one spelling no pointer ends the walk with refHint's name. A position -// under /paths keeps ownHint's guess, which the declaration replaces on the -// node (GitHub #729). -func targetHint(refs *typed.Bindings, b *oas3.JSONSchema[oas3.Referenceable]) string { - hint := "" - for range maxTargetHintHops { - ref := b.GetRef().String() - pointer, ok := resolve.FragmentPointer(ref) - if !ok { - return cmp.Or(refHint(ref), hint) - } - decl := refs.Declared(b) - own, follow := ownHint(decl, pointer) - hint = cmp.Or(own, hint) - if !follow { - return hint - } - b = decl - } - return hint -} - -// maxTargetHintHops bounds targetHint's walk along composition branches that -// hold a $ref: a loop of them, and any longer chain, stops there. An acyclic -// chain longer than the cap is legal, though no real document writes one: past -// it the walk keeps the name it has reached, so a union variant naming the -// chain's first branch can be named apart from the node it holds. -const maxTargetHintHops = 64 - -// refHint returns the name a reference suggests when its fragment spells no -// pointer — a whole-document URI, a $anchor, a fragment that decodes to bytes -// that are not UTF-8 (GitHub #520): what follows its last '/', percent-decoded -// when that decodes to UTF-8, so `./Fish%2DTank.yaml` suggests "Fish-Tank.yaml". -func refHint(ref string) string { - name := ref - if _, after, ok := strings.CutLast(ref, "/"); ok { - name = after - } - if decoded, err := url.PathUnescape(name); err == nil && utf8.ValidString(decoded) { - return decoded - } - return name -} - -// branchHint derives a composition branch's naming hint from its $ref target's -// name, falling back to a positional hint for inline branches. It names both a -// union variant and an allOf branch, which reach it as the same question. -// -// A $ref branch's answer is also subSchemaHint's, and must stay so: an outside -// $ref can name the branch pointer, and hoistSubSchema names it by -// subSchemaHint where no lowering of the composition recorded this hint, so -// the branch takes one name whether or not its composition is lowered. -func branchHint(refs *typed.Bindings, b *oas3.JSONSchema[oas3.Referenceable], i int) string { - return branchHintAt(refs, b, strconv.Itoa(i)) -} - -// branchHintAt is branchHint for a branch whose index is already a token, as the -// last one of its pointer is. -func branchHintAt(refs *typed.Bindings, b *oas3.JSONSchema[oas3.Referenceable], index string) string { - // Only a true reference carries a target name: IsReference() is precisely - // GetSchema().Ref != "" for a non-bool branch, so a schema whose Ref pointer is - // set but empty (IsReference() false) suggests no name. - if b != nil && b.IsReference() { - if name := targetHint(refs, b); name != "" { - return name - } - } - return positionalBranchHint(index) -} - -// positionalBranchHint names an inline composition branch by its position, which -// is all there is to name it by: it has no $ref target to take a name from. -func positionalBranchHint(index string) string { return "variant_" + index } - -// componentSchemaTokens is the length of /components/schemas/, the -// pointer positionHint roots a component's walk at: the component itself. -const componentSchemaTokens = 3 - -// positionHint returns the hint the structural lowering gives the schema -// position at pointer, and whether it is a composition branch. -// -// It replays the lowering because the hint is composed: items is -// compile.SubHint(enclosing, "item"), so only a walk from the root tells the -// keyword from a property named items (GitHub #518). Beneath /components/schemas -// the replay is exact. Elsewhere the enclosing hint is not in the pointer, so -// the walk resets at every token it does not know, the name a position only -// references reach takes (GitHub #529). Each step consumes a token, bounding -// the walk. -func positionHint(pointer jsontext.Pointer) (hint string, branch bool) { - tokens := slices.Collect(pointer.Tokens()) - start := min(1, len(tokens)) - if len(tokens) >= componentSchemaTokens && tokens[0] == "components" && tokens[1] == "schemas" { - start = componentSchemaTokens - } - if start > 0 { - hint = tokens[start-1] - } - for i := start; i < len(tokens); { - var step int - hint, branch, step = positionStep(hint, tokens[i:]) - i += step - } - return hint, branch -} - -// positionStep applies the keyword at rest[0] to the enclosing hint, returning -// the hint of the position it leads to, whether that position is a composition -// branch, and how many tokens the step consumed: two for a keyword whose -// children are keyed or indexed, one otherwise. A keyword the lowering does not -// walk (not, if, then, else, an unknown one) is named after its own token: only -// a reference reaches it, so any stable spelling keeps its name independent of -// declaration order. -func positionStep(enclosing string, rest []string) (hint string, branch bool, step int) { - keyword := rest[0] - if role, ok := structuralRoles[keyword]; ok { - return compile.SubHint(enclosing, role), false, 1 - } - if len(rest) < 2 { - return keyword, false, 1 - } - child := rest[1] - switch { - case keyedSchemaMaps[keyword]: - return child, false, 2 - case keyword == "patternProperties": - return compile.SubHint(enclosing, "pattern"), false, 2 - case keyword == "prefixItems" && isDecimalIndex(child): - return compile.SubHint(enclosing, child), false, 2 - case compositionKeywords[keyword] && isDecimalIndex(child): - return positionalBranchHint(child), true, 2 - } - return keyword, false, 1 -} - -// structuralRoles maps each keyword whose one schema the structural lowering -// names by role to that role: the suffixes its compile.SubHint call sites pass. -// A change to one of them has to be made here too. -// TestInlinePosition_HintIsTheSameInBothOrders pins the spelling the call sites -// give, and TestPositionHint_Shapes the one this table replays, so a drift on -// either side fails one of them. -var structuralRoles = map[string]string{ - "items": "item", - "additionalProperties": "value", - "contentSchema": "content", -} - -// keyedSchemaMaps are the keywords whose value maps a key to a schema that is -// named by the key alone: a property by its name, and the $defs, definitions, -// dependentSchemas and dependencies entries by theirs, as the pointer's last -// token always named them. patternProperties is keyed too but names by role, so -// positionStep takes it separately. -var keyedSchemaMaps = map[string]bool{ - "properties": true, "$defs": true, "definitions": true, "dependentSchemas": true, "dependencies": true, -} - -// compositionKeywords are the schema keywords whose numbered children are -// composition branches. -var compositionKeywords = map[string]bool{"allOf": true, "oneOf": true, "anyOf": true} - -// isDecimalIndex reports whether s is a non-empty run of ASCII digits — the -// shape a composition branch's pointer segment takes. -func isDecimalIndex(s string) bool { - if s == "" { - return false - } - for _, r := range s { - if r < '0' || r > '9' { - return false - } - } - return true -} diff --git a/compilers/openapi/internal/schema/hoist_internal_test.go b/compilers/openapi/internal/schema/hoist_internal_test.go index 7858ecc10..ebc9980dd 100644 --- a/compilers/openapi/internal/schema/hoist_internal_test.go +++ b/compilers/openapi/internal/schema/hoist_internal_test.go @@ -1,13 +1,10 @@ package schema import ( - "encoding/json/jsontext" "strconv" "strings" "testing" - oas3 "github.com/speakeasy-api/openapi/jsonschema/oas3" - "github.com/speakeasy-api/openapi/references" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,29 +12,30 @@ import ( "github.com/dexpace/morphic/compilers/openapi/internal/diag" "github.com/dexpace/morphic/compilers/openapi/internal/ids" "github.com/dexpace/morphic/compilers/openapi/internal/lowering" + "github.com/dexpace/morphic/compilers/openapi/internal/naming" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" "github.com/dexpace/morphic/ir" ) // TestCommonFor_NamesOnlyAComponent pins the split every hoisted node carries. // A top-level component schema is named from the source; any deeper position is -// anonymous and carries only its context-derived hint. Getting this backwards +// anonymous and carries only the hint its coordinate names it by. Getting this backwards // gives an emitter a name to render where the source declared none, or drops the // one it did declare. func TestCommonFor_NamesOnlyAComponent(t *testing.T) { t.Parallel() c := lowering.Ctx{SrcIndex: 2} - named := commonFor(c, "t/x", "/components/schemas/User", "ignored_hint") + named := commonFor(c, "t/x", "/components/schemas/User") assert.False(t, named.Anonymous, "a component schema is a declaration") assert.Equal(t, "User", named.Name.Source) assert.Equal(t, "user", named.Name.Canonical, "the framework supplies the neutral words") assert.Empty(t, named.Name.Hint, "a named node needs no hint") - anon := commonFor(c, "t/y", "/components/schemas/User/properties/id", "user_id") + anon := commonFor(c, "t/y", "/components/schemas/User/properties/id") assert.True(t, anon.Anonymous, "an inline position declares no name") assert.Empty(t, anon.Name.Source) - assert.Equal(t, "user_id", anon.Name.Hint, "only the hint survives") + assert.Equal(t, "id", anon.Name.Hint, "only the hint survives: the property's key") } // TestCommonFor_StampsTheDeclaringPointer pins that the node records where it @@ -45,7 +43,7 @@ func TestCommonFor_NamesOnlyAComponent(t *testing.T) { // traces a type back through. func TestCommonFor_StampsTheDeclaringPointer(t *testing.T) { t.Parallel() - got := commonFor(lowering.Ctx{SrcIndex: 5}, "t/x", "/components/schemas/User", "") + got := commonFor(lowering.Ctx{SrcIndex: 5}, "t/x", "/components/schemas/User") assert.Equal(t, ir.TypeID("t/x"), got.ID) assert.Equal(t, ir.Provenance{Source: 5, Pointer: "/components/schemas/User"}, got.Provenance) @@ -67,8 +65,8 @@ func TestInternNode_DerivesTheIDAndInternsOnce(t *testing.T) { return &ir.Model{TypeCommon: common} } - first := internNode(c, ts, at, "", build) - second := internNode(c, ts, at, "", build) + first := internNode(c, ts, at, build) + second := internNode(c, ts, at, build) assert.Equal(t, ids.Document{}.ForPointer(at), first, "the ID is derived from the pointer") assert.Equal(t, first, second, "a second visit returns the interned ID") @@ -96,7 +94,7 @@ func TestRegisteredNode_ReportsAMissRatherThanDroppingIt(t *testing.T) { assert.Equal(t, diag.InternalInvariant, diags[0].Code) assert.Equal(t, ir.Provenance{Source: 4, Pointer: "/components/schemas/Ghost"}, diags[0].Provenance) - id := internNode(c, ts, "/components/schemas/User", "", func(cm ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, "/components/schemas/User", func(cm ir.TypeCommon) ir.TypeDef { return &ir.Model{TypeCommon: cm} }) td, ok, diags := registeredNode(c, ts, id, "/components/schemas/User") @@ -117,7 +115,7 @@ func TestRegisteredNode_LeavesANodeStillBeingBuiltToItsBuilder(t *testing.T) { var midBuild bool var midBuildDiags []ir.Diagnostic - id := internNode(c, ts, at, "", func(cm ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, at, func(cm ir.TypeCommon) ir.TypeDef { _, midBuild, midBuildDiags = registeredNode(c, ts, cm.ID, at) return &ir.Model{TypeCommon: cm} }) @@ -129,109 +127,20 @@ func TestRegisteredNode_LeavesANodeStillBeingBuiltToItsBuilder(t *testing.T) { assert.Empty(t, diags) } -// TestTargetHint_Shapes pins what a reference suggests from its text alone: -// each row is unresolved, so the walk reads the pointer, never the schema -// behind it. A fragment spelling a pointer is decoded at both layers (GitHub -// #505) and named by positionHint, so a branch or structural position is -// suggested as the node it holds (GitHub #521), whether at a component or -// elsewhere (GitHub #529). Any other fragment (non-UTF-8 included, GitHub -// #520) is named by the text after its last '/', percent-decoded when that is -// valid UTF-8, else as written. -func TestTargetHint_Shapes(t *testing.T) { - t.Parallel() - tests := []struct { - name string - ref string - want string - }{ - {name: "plain component", ref: "#/components/schemas/Pet", want: "Pet"}, - {name: "RFC 6901 escape decodes", ref: "#/components/schemas/Cat~1Dog", want: "Cat/Dog"}, - {name: "percent escape decodes", ref: "#/components/schemas/Fish%2DTank", want: "Fish-Tank"}, - {name: "another document, still a pointer", ref: "other.yaml#/components/schemas/Foo", want: "Foo"}, - {name: `a lone slash names the member keyed ""`, ref: "#/", want: ""}, - {name: "a oneOf branch is suggested as the variant, not its ordinal", - ref: "#/components/schemas/X/oneOf/0", want: "variant_0"}, - {name: "a structural position is suggested by its role, not the keyword", - ref: "#/components/schemas/A/items", want: compile.SubHint("A", "item")}, - {name: "a property whose key reads like a keyword is suggested by the key", - ref: "#/components/schemas/A/properties/items", want: "items"}, - {name: "a branch outside components is suggested as the variant too", - ref: "#/paths/~1x/get/responses/200/content/application~1json/schema/oneOf/0", want: "variant_0"}, - {name: "any other position outside components is named by positionHint's replay, not its last token", - ref: "#/paths/~1x/get/responses/200/content/application~1json/schema/items", want: compile.SubHint("schema", "item")}, - {name: "no fragment at all", ref: "bare", want: "bare"}, - {name: "another document, no fragment", ref: "other.yaml", want: "other.yaml"}, - {name: "a document in a directory, no fragment", ref: "./schemas/Pet.yaml", want: "Pet.yaml"}, - {name: "a $anchor is not a pointer", ref: "#anchor", want: "#anchor"}, - {name: "a fragment that is not UTF-8 is not a pointer", ref: "#/components/schemas/%FF", want: "%FF"}, - {name: "a percent escape in a reference with no fragment decodes", - ref: "./Fish%2DTank.yaml", want: "Fish-Tank.yaml"}, - {name: "an invalid percent escape is kept as written", - ref: "./bad%ZZ.yaml", want: "bad%ZZ.yaml"}, - {name: "a percent escape decoding to non-UTF-8 bytes is kept as written", - ref: "./x%FF.yaml", want: "x%FF.yaml"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tc.want, targetHint(nil, oas3.NewJSONSchemaFromReference(references.Reference(tc.ref)))) - }) - } -} - -// TestTargetHint_FollowsABranchToItsTarget pins where targetHint's walk goes on -// and where it stops, over the bindings a load gives. A reference to a -// composition branch holding a $ref (at, holding to) is named after that -// branch's target (GitHub #521), inside a component or not; a target that -// names nothing leaves the branch's own name. Any other position holding a -// $ref is named for the position, as positionHint replays it: a guess under -// /paths that the declaration replaces (GitHub #729). -func TestTargetHint_FollowsABranchToItsTarget(t *testing.T) { - t.Parallel() - const under = "/paths/~1x/get/responses/200/content/application~1json/schema" - tests := []struct { - name string - at string - to string - want string - }{ - {"a branch beneath a component", "/components/schemas/X/oneOf/0", "#/components/schemas/Y", "Y"}, - {"a branch elsewhere", under + "/oneOf/0", "#/components/schemas/Y", "Y"}, - {"a branch holding a reference to another document", "/components/schemas/X/oneOf/0", - "./Fish%2DTank.yaml", "Fish-Tank.yaml"}, - {"a branch whose target names nothing", "/components/schemas/X/oneOf/0", "#/components/schemas/", "variant_0"}, - {"a branch whose other document names nothing", "/components/schemas/X/oneOf/0", "./schemas/", "variant_0"}, - {"any other position beneath a component", "/components/schemas/X/items", "#/components/schemas/Y", - compile.SubHint("X", "item")}, - {"any other position elsewhere", under + "/items", "#/components/schemas/Y", compile.SubHint("schema", "item")}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - l, _ := loweredFor(t, hintSpec(tc.at, tc.to)) - u, ok := l.ctx.Doc.Components.GetSchemas().Get("U") - require.True(t, ok) - assert.Equal(t, tc.want, targetHint(l.ctx.Bindings(), u)) - }) - } -} - -// TestTargetHint_StopsAtTheHopCap pins maxTargetHintHops at its boundary, -// through a compile because only the resolver attaches a resolution to every -// branch of a long chain. Each branch holds a $ref to the next and the last to -// T. A variant naming the first branch is named after T while the walk reaches -// T within the cap, and keeps the last branch's name once it cannot. The node -// that variant holds starts its own walk a hop further on, so it still reaches -// T: past the cap the two are named apart, as the cap's doc says. -func TestTargetHint_StopsAtTheHopCap(t *testing.T) { +// TestUnionVariant_StopsAtTheHopCap pins naming.MaxHops through a compile. +// Each branch holds a $ref to the next and the last to T. A variant naming the +// first branch is named after T while T is within the cap, and keeps its own +// name once it is not. The node that variant holds is T's, so past the cap the +// two are named apart, as the cap's doc says. +func TestUnionVariant_StopsAtTheHopCap(t *testing.T) { t.Parallel() for _, tc := range []struct { name string branches int variant string }{ - {"T within reach", maxTargetHintHops - 1, "t"}, - {"T one hop past the cap", maxTargetHintHops, "variant_0"}, + {"T within reach", naming.MaxHops - 1, "t"}, + {"T one hop past the cap", naming.MaxHops, "variant_0"}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -258,308 +167,3 @@ func TestTargetHint_StopsAtTheHopCap(t *testing.T) { }) } } - -// TestSubSchemaHint_Shapes pins the name subSchemaHint gives a position, one -// row per answer ownHint gives, each for a schema that is a $ref and for one -// that is not. A branch holding a $ref takes its target's name, falling back to -// its own when the target names nothing (an empty-named component); every other -// position is named for where it is, whatever it holds: positionHint's name. -// -// The $ref rows resolve nothing, so a target is named from its pointer alone. -func TestSubSchemaHint_Shapes(t *testing.T) { - t.Parallel() - const under = "/paths/~1x/get/responses/200/content/application~1json/schema" - ref := func(to string) *oas3.JSONSchema[oas3.Referenceable] { - return oas3.NewJSONSchemaFromReference(references.Reference(to)) - } - inline := oas3.NewJSONSchemaFromSchema[oas3.Referenceable](&oas3.Schema{}) - tests := []struct { - name string - decl *oas3.JSONSchema[oas3.Referenceable] - pointer jsontext.Pointer - want string - }{ - {"a position beneath a component, inline", inline, "/components/schemas/A/items", compile.SubHint("A", "item")}, - {"a position beneath a component, holding a $ref", ref("#/components/schemas/T"), - "/components/schemas/A/items", compile.SubHint("A", "item")}, - {"a branch beneath a component, inline", inline, "/components/schemas/A/oneOf/0", "variant_0"}, - {"a branch beneath a component, holding a $ref", ref("#/components/schemas/T"), - "/components/schemas/A/oneOf/0", "T"}, - {"a branch holding a $ref to a target that names nothing", ref("#/components/schemas/"), - "/components/schemas/A/oneOf/0", "variant_0"}, - {"a branch elsewhere, inline", inline, under + "/oneOf/0", "variant_0"}, - {"a branch elsewhere, holding a $ref", ref("#/components/schemas/T"), under + "/oneOf/0", "T"}, - {"any other position elsewhere, inline", inline, under + "/items", compile.SubHint("schema", "item")}, - {"any other position elsewhere, holding a $ref", ref("#/components/schemas/T"), under + "/items", - compile.SubHint("schema", "item")}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tc.want, subSchemaHint(nil, tc.decl, tc.pointer)) - }) - } -} - -// TestPositionHint_BranchShapes pins which pointers name a composition branch. -// -// The walk decides whether hoistSubSchema answers what the composition would, -// so a pointer it misreads either reintroduces the disagreement (GitHub #181) or -// invents a variant hint for something that is not a branch. The rows that are -// not branches matter as much as the rows that are: `oneOf` is a legal property -// name, and a schema keyword taking numbered children is not necessarily a -// composition. -func TestPositionHint_BranchShapes(t *testing.T) { - t.Parallel() - tests := []struct { - name string - pointer jsontext.Pointer - want string - }{ - {name: "a oneOf branch", pointer: "/components/schemas/S/oneOf/0", want: "variant_0"}, - {name: "an anyOf branch", pointer: "/components/schemas/S/anyOf/12", want: "variant_12"}, - {name: "an allOf branch", pointer: "/components/schemas/S/allOf/1", want: "variant_1"}, - { - name: "a branch of a schema reached through a property named oneOf", - pointer: "/components/schemas/S/properties/oneOf/anyOf/0", want: "variant_0", - }, - - {name: "prefixItems takes indices but composes nothing", pointer: "/components/schemas/S/prefixItems/0"}, - {name: "items is not indexed at all", pointer: "/components/schemas/S/items"}, - {name: "a property named oneOf is not a branch", pointer: "/components/schemas/S/properties/oneOf"}, - {name: "a non-numeric child of a composition keyword", pointer: "/components/schemas/S/oneOf/x"}, - {name: "a negative index is not a segment the compiler writes", pointer: "/components/schemas/S/oneOf/-1"}, - {name: "an empty index", pointer: "/components/schemas/S/oneOf/"}, - {name: "a branch outside components", pointer: "/paths/~1x/get/responses/200/content/application~1json/schema/oneOf/0", want: "variant_0"}, - {name: "a bare segment", pointer: "oneOf"}, - {name: "the empty pointer", pointer: ""}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - hint, branch := positionHint(tc.pointer) - got := "" - if branch { - got = hint - } - assert.Equal(t, tc.want != "", branch, "pointer %q", tc.pointer) - assert.Equal(t, tc.want, got, "pointer %q", tc.pointer) - }) - } -} - -// TestBranchHint_AgreesWithThePointerWalk states the property the two -// derivations exist to share, at the functions rather than through a compile: for -// an inline branch, what the composition names the node and what a pointer walk -// names it are the same string, so the node takes one hint whether or not its -// composition is lowered. -func TestBranchHint_AgreesWithThePointerWalk(t *testing.T) { - t.Parallel() - for i := range 3 { - inline := oas3.NewJSONSchemaFromSchema[oas3.Referenceable](&oas3.Schema{}) - pointer := jsontext.Pointer("/components/schemas/Host/oneOf/" + strconv.Itoa(i)) - - fromComposition := branchHint(nil, inline, i) - fromPointer, branch := positionHint(pointer) - require.True(t, branch, "the branch pointer is recognized as one") - assert.Equal(t, fromComposition, fromPointer, - "branch %d is named the same by the composition and by a pointer walk", i) - assert.Equal(t, fromComposition, subSchemaHint(nil, inline, pointer), - "and subSchemaHint is the path that actually asks") - } -} - -// TestPositionHint_Shapes pins the left-to-right walk down a schema: each step -// recomposes the enclosing hint by role (items, additionalProperties, -// contentSchema, a patternProperties entry, a prefixItems slot, a composition -// branch) or, for a keyed map (properties, $defs, definitions, -// dependentSchemas, dependencies), takes the key itself. A key is never read as -// a keyword (GitHub #518), and a branch ordinal names "variant_0", never "0". -// Beneath /components/schemas the walk roots at the component; elsewhere at the -// pointer's first token, resetting at every token it does not know (GitHub -// #529). -func TestPositionHint_Shapes(t *testing.T) { - t.Parallel() - tests := []struct { - name string - pointer jsontext.Pointer - want string - branch bool - }{ - { - name: "a bare component", pointer: "/components/schemas/Foo", - want: "Foo", - }, - { - name: "items under a component", pointer: "/components/schemas/Foo/items", - want: compile.SubHint("Foo", "item"), - }, - { - name: "nested roles", - pointer: "/components/schemas/Foo/items/additionalProperties", - want: compile.SubHint(compile.SubHint("Foo", "item"), "value"), - }, - { - name: "a property literally named items is not the items keyword", - pointer: "/components/schemas/Foo/properties/items", - want: "items", - }, - { - name: "a property named items holding an array names its own items by role", - pointer: "/components/schemas/Foo/properties/items/items", - want: compile.SubHint("items", "item"), - }, - { - name: "a property literally named properties holding items names its own items by role", - pointer: "/components/schemas/Foo/properties/properties/items", - want: compile.SubHint("properties", "item"), - }, - { - name: "additionalProperties holding an object whose own additionalProperties is a role", - pointer: "/components/schemas/Foo/properties/additionalProperties/additionalProperties", - want: compile.SubHint("additionalProperties", "value"), - }, - { - name: "a patternProperties entry whose pattern holds ~1", - pointer: "/components/schemas/Foo/patternProperties/a~1b", - want: compile.SubHint("Foo", "pattern"), - }, - { - name: "a patternProperties entry keyed items is not the items keyword", - pointer: "/components/schemas/Foo/patternProperties/items", - want: compile.SubHint("Foo", "pattern"), - }, - { - name: "an items entry inside a patternProperties entry keyed items", - pointer: "/components/schemas/Foo/patternProperties/items/items", - want: compile.SubHint(compile.SubHint("Foo", "pattern"), "item"), - }, - { - name: "a $defs entry keyed items is not the items keyword", - pointer: "/components/schemas/Foo/$defs/items", - want: "items", - }, - { - name: "a dependentSchemas entry keyed items is not the items keyword", - pointer: "/components/schemas/Foo/dependentSchemas/items", - want: "items", - }, - { - name: "a definitions entry keyed items is not the items keyword", - pointer: "/components/schemas/Foo/definitions/items", - want: "items", - }, - { - name: "a dependencies entry keyed items is not the items keyword", - pointer: "/components/schemas/Foo/dependencies/items", - want: "items", - }, - { - name: "a contentSchema", - pointer: "/components/schemas/Foo/contentSchema", - want: compile.SubHint("Foo", "content"), - }, - { - name: "a prefixItems slot", - pointer: "/components/schemas/Foo/prefixItems/2", - want: compile.SubHint("Foo", "2"), - }, - { - name: "a prefixItems member that is no slot is named after its own token", - pointer: "/components/schemas/Foo/prefixItems/x", - want: "x", - }, - { - name: "a component literally named items is named items", - pointer: "/components/schemas/items", - want: "items", - }, - { - name: "a oneOf branch is named variant_N, not its ordinal", - pointer: "/components/schemas/Foo/oneOf/0", - want: "variant_0", branch: true, - }, - { - name: "items beneath a oneOf branch is named off the branch's hint, not the ordinal", - pointer: "/components/schemas/Foo/oneOf/0/items", - want: compile.SubHint("variant_0", "item"), - }, - { - name: "a branch reached through a property named oneOf", - pointer: "/components/schemas/Foo/properties/oneOf/anyOf/0", - want: "variant_0", branch: true, - }, - { - name: "a property named oneOf with no index is not a branch", - pointer: "/components/schemas/Foo/properties/oneOf", - want: "oneOf", - }, - { - name: "a non-numeric child of a composition keyword is not a branch", - pointer: "/components/schemas/Foo/oneOf/x", - want: "x", - }, - { - name: "an unrecognised keyword is named after its own token, and what it holds hangs off that", - pointer: "/components/schemas/Foo/not/items", - want: compile.SubHint("not", "item"), - }, - { - name: "an extension keyword is treated the same as any other unrecognised token", - pointer: "/components/schemas/Foo/x-ext/items", - want: compile.SubHint("x-ext", "item"), - }, - { - name: "a keyed keyword with no key names itself", - pointer: "/components/schemas/Foo/properties", - want: "properties", - }, - { - name: "a position under /paths resets at every token the walk does not know, so items is named off the schema's own hint, not the response or media type that led there", - pointer: "/paths/~1x/get/responses/200/content/application~1json/schema/items", - want: compile.SubHint("schema", "item"), - }, - { - name: "a component that is not a schema replays the same way", - pointer: "/components/responses/R/content/application~1json/schema/items", - want: compile.SubHint("schema", "item"), - }, - { - name: `the component keyed ""`, - pointer: "/components/schemas//items", - want: compile.SubHint("", "item"), - }, - { - name: "the schemas map has too few tokens to root at a component, so it replays like any other pointer", - pointer: "/components/schemas", - want: "schemas", - }, - {name: "the empty pointer has no tokens to replay, so it is named \"\" and is never a branch", pointer: ""}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - got, branch := positionHint(tc.pointer) - assert.Equal(t, tc.want, got, "pointer %q", tc.pointer) - assert.Equal(t, tc.branch, branch, "pointer %q", tc.pointer) - }) - } -} - -// hintSpec is a document whose schema U is a $ref to at, a composition branch -// or an items position, which holds a $ref to to; Y is a schema to name. -func hintSpec(at, to string) string { - holder := "{oneOf: [{$ref: '" + to + "'}]}" - if strings.HasSuffix(at, "/items") { - holder = "{type: array, items: {$ref: '" + to + "'}}" - } - doc := "openapi: 3.1.0\ninfo: {title: t, version: '1'}\npaths:" - x := "" - if strings.HasPrefix(at, "/components/schemas/X") { - doc += " {}\n" - x = " X: " + holder + "\n" - } else { - doc += "\n /x: {get: {responses: {'200': {description: ok, content: {application/json: {schema: " + holder + "}}}}}}\n" - } - return doc + "components:\n schemas:\n" + x + " Y: {type: object}\n U: {$ref: '#" + at + "'}\n" -} diff --git a/compilers/openapi/internal/schema/hoist_test.go b/compilers/openapi/internal/schema/hoist_test.go index 83b642e05..078250b8d 100644 --- a/compilers/openapi/internal/schema/hoist_test.go +++ b/compilers/openapi/internal/schema/hoist_test.go @@ -148,8 +148,8 @@ func TestLower_InlineSchemaHoistedOnce(t *testing.T) { // // refAt says where that $ref aims, at the node asserted or above it. A // reference only promises a node, so the declaration names each one in either -// order; the rows pin the spelling the SubHint call sites give, and -// TestPositionHint_Shapes the one structuralRoles replays. +// order; the rows pin the spelling each position takes here, and +// naming.TestAt_NamesASchemaByItsStructure every spelling the namer gives. func TestInlinePosition_HintIsTheSameInBothOrders(t *testing.T) { t.Parallel() for _, tc := range []struct { @@ -169,8 +169,8 @@ func TestInlinePosition_HintIsTheSameInBothOrders(t *testing.T) { "t/anon/components/schemas/A/prefixItems/0", "a_0", ""}, {"tuple rest", " A: {type: array, prefixItems: [{type: string}], items: " + openapitest.InlineProbeBody + "}\n", "t/anon/components/schemas/A/items", "a_item", ""}, - // Nested, because the derivation replays the whole chain rather than one - // step: the outside $ref used to name this "items", losing both levels. + // Nested, because the name is composed along the whole chain rather than + // one step: the outside $ref used to name this "items", losing both levels. {"items under items", " A: {type: array, items: {type: array, items: " + openapitest.InlineProbeBody + "}}\n", "t/anon/components/schemas/A/items/items", "a_item_item", ""}, // Rooted at a property rather than at the component, so the enclosing hint @@ -188,7 +188,7 @@ func TestInlinePosition_HintIsTheSameInBothOrders(t *testing.T) { "t/anon/components/schemas/A/contentSchema/items", "a_content_item", "t/anon/components/schemas/A/contentSchema"}, // Aimed at a tuple's rest: a rest named by a role of its own would part - // from the /items a pointer walk replays. + // from the name its /items pointer gives it. {"items under a tuple rest", " A: {type: array, prefixItems: [{type: string}], items: {type: array, items: " + openapitest.InlineProbeBody + "}}\n", "t/anon/components/schemas/A/items/items", "a_item_item", @@ -327,9 +327,8 @@ func TestPureRefPosition_IsNamedByItsDeclarationInBothOrders(t *testing.T) { "t/anon/components/schemas/A/prefixItems/0", "a_0"}, {"patternProperties", " A: {type: object, patternProperties: {\"^x\": {$ref: '#/components/schemas/T'}}}\n" + tDecl, "t/anon/components/schemas/A/patternProperties/^x", "a_pattern"}, - // A $ref branch is the one position the composition names after its - // target rather than its own location, in either order — branchHint - // and subSchemaHint agree on that (GitHub #521) — so this row's + // A $ref branch is the one position named after its target rather + // than its own location (GitHub #521), in either order, so this row's // expectation does not flip between the two orders below. {"a union branch", " A: {oneOf: [{$ref: '#/components/schemas/T'}, {type: integer}]}\n" + tDecl, "t/anon/components/schemas/A/oneOf/0", "t"}, @@ -354,11 +353,10 @@ func TestPureRefPosition_IsNamedByItsDeclarationInBothOrders(t *testing.T) { // TestPureRefPosition_UnderPathsIsNamedByItsDeclaration is // TestPureRefPosition_IsNamedByItsDeclarationInBothOrders's counterpart under // /paths, modelled on TestInlinePosition_UnderPathsIsNamedByItsDeclaration: -// positionHint cannot replay the enclosing hint there, so it offers -// "schema_item" where the declaration says "response_item". /b's declaration, -// which interns nothing at the pure $ref, records the name it gives the -// position, and the alias a reference hoists there once every declaration has -// lowered takes it, whichever of /b or /a lowers first. +// the pointer alone would name the position "schema_item" where its owner, a +// response, names it "response_item". /b's declaration interns nothing at the +// pure $ref, and the alias a reference hoists there takes the owner's name +// (lowering.NameAt), whichever of /b or /a lowers first. func TestPureRefPosition_UnderPathsIsNamedByItsDeclaration(t *testing.T) { t.Parallel() const id = ir.TypeID("t/anon/paths/~1b/get/responses/200/content/application~1json/schema/items") @@ -393,7 +391,7 @@ func TestPureRefPosition_UnderPathsIsNamedByItsDeclaration(t *testing.T) { openapitest.RequireNoErrorDiags(t, diags) assert.Equal(t, "response_item", bFirst.Types[id].Common().Name.Hint, - "the declaration's hint is recorded and survives even though it interns nothing itself") + "the position is named for its owner though its declaration interns nothing") assert.Equal(t, "response_item", aFirst.Types[id].Common().Name.Hint, "and in the other order too") } diff --git a/compilers/openapi/internal/schema/model.go b/compilers/openapi/internal/schema/model.go index 0bbaddcf3..343c1aa5f 100644 --- a/compilers/openapi/internal/schema/model.go +++ b/compilers/openapi/internal/schema/model.go @@ -24,16 +24,16 @@ import ( // Cardinality is read here rather than on lowerComponentSchema's internAlias // fallback because an object-shaped schema owns its node before that fallback // would run, so the fallback never fires for one (GitHub #129). -func lowerModel(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerModel(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { cons, consDiags := ConstraintsOf(c, &common.Unmodeled, s, pointer) diags = append(diags, consDiags...) m := &ir.Model{TypeCommon: common, Constraints: cons} - diags = append(diags, fillModelProperties(c, ts, anchors, depth, propIndex(c, ts, m), s, pointer)...) + diags = append(diags, fillModelProperties(c, ts, depth, propIndex(c, ts, m), s, pointer)...) requireMembers(m, s.GetRequired()) - diags = append(diags, fillAdditional(c, ts, anchors, depth, m, s, pointer, hint)...) - d, discDiags := lowerDiscriminator(c, ts, anchors, s, m, nil, pointer) + diags = append(diags, fillAdditional(c, ts, depth, m, s, pointer)...) + d, discDiags := lowerDiscriminator(c, ts, s, m, nil, pointer) diags = append(diags, discDiags...) if d != nil { m.Discriminator = d @@ -87,7 +87,7 @@ func (s settledMerge) Settle(reg ir.TypeRegistry) []ir.Diagnostic { // fillModelProperties lowers a model's own properties in source order, each with // its full property-level detail (constraints, visibility, defaults, docs, ...). // Which are required is requireMembers' to say, once every property is in. -func fillModelProperties(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, idx *merge.PropIndex, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { +func fillModelProperties(c lowering.Ctx, ts *compile.Types, depth int, idx *merge.PropIndex, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { props := s.GetProperties() if props == nil { return nil @@ -95,7 +95,7 @@ func fillModelProperties(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex var diags []ir.Diagnostic for name, js := range props.All() { ppointer := pointer + ids.Ptr("properties", name) - ref, refDiags := CarriedRef(c, ts, anchors, depth, js, ppointer, name) + ref, refDiags := CarriedRef(c, ts, depth, js, ppointer) diags = append(diags, refDiags...) p := ir.Property{ ID: c.IDs().Prop(ppointer), @@ -104,7 +104,7 @@ func fillModelProperties(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex Type: ref, Provenance: c.ProvenanceAt(ppointer), } - diags = append(diags, FillPropertyDetail(c, ts, anchors, &p, js, ppointer)...) + diags = append(diags, FillPropertyDetail(c, ts, &p, js, ppointer)...) idx.MergeProperty(p, redeclarationSource(js)) } return diags @@ -160,7 +160,7 @@ func requireMembers(m *ir.Model, required []string) { // beyond the declared ones are allowed. Only an absent keyword writes no // catch-all. Neither spelling touches Additional, which this sets only for // `false` and `unevaluatedProperties: false`. -func fillAdditional(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, m *ir.Model, s *oas3.Schema, pointer jsontext.Pointer, hint string) []ir.Diagnostic { +func fillAdditional(c lowering.Ctx, ts *compile.Types, depth int, m *ir.Model, s *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { var diags []ir.Diagnostic ap := s.GetAdditionalProperties() switch { @@ -169,11 +169,11 @@ func fillAdditional(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, dep case annotation.IsFalseSchema(ap): m.Additional = ir.AdditionalClosed default: - ref, refDiags := Ref(c, ts, anchors, depth, ap, pointer+ids.Ptr("additionalProperties"), compile.SubHint(hint, "value")) + ref, refDiags := Ref(c, ts, depth, ap, pointer+ids.Ptr("additionalProperties")) diags = append(diags, refDiags...) m.AdditionalProps = &ir.AdditionalProps{Value: ref} } - patterns, patternDiags := patternProps(c, ts, anchors, depth, s, pointer, hint) + patterns, patternDiags := patternProps(c, ts, depth, s, pointer) diags = append(diags, patternDiags...) if len(patterns) > 0 { if m.AdditionalProps == nil { @@ -189,7 +189,7 @@ func fillAdditional(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, dep // patternProps lowers patternProperties into pattern/value bindings in source // order. -func patternProps(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) ([]ir.PatternProps, []ir.Diagnostic) { +func patternProps(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) ([]ir.PatternProps, []ir.Diagnostic) { pp := s.GetPatternProperties() if pp == nil || pp.Len() == 0 { return nil, nil @@ -197,7 +197,7 @@ func patternProps(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth var diags []ir.Diagnostic out := make([]ir.PatternProps, 0, pp.Len()) for pattern, js := range pp.All() { - ref, refDiags := Ref(c, ts, anchors, depth, js, pointer+ids.Ptr("patternProperties", pattern), compile.SubHint(hint, "pattern")) + ref, refDiags := Ref(c, ts, depth, js, pointer+ids.Ptr("patternProperties", pattern)) diags = append(diags, refDiags...) out = append(out, ir.PatternProps{Pattern: pattern, Value: ref}) } diff --git a/compilers/openapi/internal/schema/property.go b/compilers/openapi/internal/schema/property.go index e83d2b626..a59461bb8 100644 --- a/compilers/openapi/internal/schema/property.go +++ b/compilers/openapi/internal/schema/property.go @@ -21,7 +21,7 @@ import ( // has, while its schema may lower to a node of a kind with no Constraints field // or to no node of its own at all. Restating them where the node also carries // them is the safe half of that trade. -func FillPropertyDetail(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, p *ir.Property, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) []ir.Diagnostic { +func FillPropertyDetail(c lowering.Ctx, ts *compile.Types, p *ir.Property, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) []ir.Diagnostic { ref := js.GetSchema() if ref == nil { return nil @@ -31,7 +31,7 @@ func FillPropertyDetail(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, p.Default = def diags = append(diags, fillPropertyVisibility(c, p, ref, tgt, pointer)...) diags = append(diags, fillPropertyConstraints(c, p, ref, pointer)...) - diags = append(diags, fillPropertyAnnotations(c, ts, anchors, p, ref, tgt, pointer)...) + diags = append(diags, fillPropertyAnnotations(c, ts, p, ref, tgt, pointer)...) return append(diags, PreserveRefSiteKeywords(c, ts, &p.Unmodeled, js, p.Type, pointer)...) } @@ -82,7 +82,7 @@ func fillPropertyConstraints(c lowering.Ctx, p *ir.Property, ref *oas3.Schema, p // // A $ref position never hoists a node, so annotations written beside a // property's $ref land here (GitHub #114). -func fillPropertyAnnotations(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, p *ir.Property, ref, tgt *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { +func fillPropertyAnnotations(c lowering.Ctx, ts *compile.Types, p *ir.Property, ref, tgt *oas3.Schema, pointer jsontext.Pointer) []ir.Diagnostic { if LoweredToOwnNode(c, ts, pointer, p.Type) { return nil } @@ -106,7 +106,7 @@ func fillPropertyAnnotations(c lowering.Ctx, ts *compile.Types, anchors *AnchorI // nil node: this arm runs only when the schema lowered to no node of its own, // so nothing here can be carrying an Encoding. diags = append(diags, recordUnplacedContent(c, &p.Unmodeled, ref, nil, pointer)...) - diags = append(diags, recordUnexpandedDynamicRef(c, anchors, &p.Unmodeled, ref, pointer)...) + diags = append(diags, recordUnexpandedDynamicRef(c, &p.Unmodeled, ref, pointer)...) return append(diags, PreserveUnknownKeywords(c, &p.Unmodeled, ref, pointer)...) } diff --git a/compilers/openapi/internal/schema/refsite.go b/compilers/openapi/internal/schema/refsite.go index feee0bf81..f659cf414 100644 --- a/compilers/openapi/internal/schema/refsite.go +++ b/compilers/openapi/internal/schema/refsite.go @@ -26,11 +26,11 @@ import ( // // At an annotation.HomeCarrier position no alias is hoisted: the carrier keeps // them through PreserveRefSiteKeywords (GitHub #114). -func refSiteRef(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, js *oas3.JSONSchema[oas3.Referenceable], s *oas3.Schema, pointer jsontext.Pointer, hint string, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { - target, diags := refTypeRef(c, ts, anchors, js, pointer) +func refSiteRef(c lowering.Ctx, ts *compile.Types, js *oas3.JSONSchema[oas3.Referenceable], s *oas3.Schema, pointer jsontext.Pointer, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { + target, diags := refTypeRef(c, ts, js, pointer) unhomed := refSiteUnhomedKeywords(s, nil) hasUnion := declaresUnion(s) - ref, homeDiags := homeDeclaration(c, ts, anchors, s, target, pointer, hint, home, len(unhomed) > 0 || hasUnion) + ref, homeDiags := homeDeclaration(c, ts, s, target, pointer, home, len(unhomed) > 0 || hasUnion) diags = append(diags, homeDiags...) if home != annotation.HomeOwnNode { return ref, diags @@ -81,9 +81,9 @@ func PreserveRefSiteKeywords(c lowering.Ctx, ts *compile.Types, p *ir.Unmodeled, // one landing on no schema, is diagnosed and dropped to any. A $ref on a loop of $refs is // dropped to any too, with nothing of its own to report: the load phase reports // the loop once. -func refTypeRef(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { +func refTypeRef(c lowering.Ctx, ts *compile.Types, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { ref := js.GetRef().String() - id, ok := resolveSchemaRef(c, ts, anchors, js) + id, ok := resolveSchemaRef(c, ts, js) if ok { return refNullRef(c, id, js, siteAt(pointer)) } @@ -135,22 +135,22 @@ func hopWhy(h bind.Hop) string { // names, in whichever document (resolveAt). It returns ok=false for a $ref that // lands on no schema (resolve.Scope.Target) and for a reference to an // undeclared component. -func resolveSchemaRef(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, js *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { +func resolveSchemaRef(c lowering.Ctx, ts *compile.Types, js *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { at, ok := c.RefScope().Target(js) if !ok { return "", false } - return resolveAt(c, ts, anchors, at, c.Bindings().Declared(js)) + return resolveAt(c, ts, at, c.Bindings().Declared(js)) } // resolveAt is resolvePointer at a position of the bindings' table, read at // its own document's coordinates (lowering.InDocument). -func resolveAt(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, at bind.Position, decl *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { +func resolveAt(c lowering.Ctx, ts *compile.Types, at bind.Position, decl *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { c, ok := lowering.InDocument(c, at.Doc) if !ok { return "", false } - return resolvePointer(c, ts, anchors, at.Pointer, decl) + return resolvePointer(c, ts, at.Pointer, decl) } // resolvePointer resolves a pointer in c's document to the ID of the schema it @@ -163,7 +163,7 @@ func resolveAt(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, at bind. // // A $ref and a discriminator mapping target meet here once each has its // pointer, so they cannot drift apart. -func resolvePointer(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, pointer jsontext.Pointer, decl *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { +func resolvePointer(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer, decl *oas3.JSONSchema[oas3.Referenceable]) (ir.TypeID, bool) { // The source's components alone are lowered whole and named by name; // another document's are hoisted where a reference reaches them, as any // other position of it is. @@ -175,7 +175,7 @@ func resolvePointer(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, poi if !resolve.Hoistable(decl) { return "", false } - ts.Promise(c.IDs().Coordinate(pointer), func() []ir.Diagnostic { return hoistSubSchema(c, ts, anchors, decl, pointer) }) + ts.Promise(c.IDs().Coordinate(pointer), func() []ir.Diagnostic { return hoistSubSchema(c, ts, decl, pointer) }) return c.IDs().ForPointer(pointer), true } @@ -187,26 +187,21 @@ func resolvePointer(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, poi // and examples. // // It lowers in the position's own context, whichever reference promised it: -// from TopLevelDepth, as a walk of its own, in decl's document, and under the -// hint the position's lowering recorded, else subSchemaHint. A $ref with +// from TopLevelDepth, as a walk of its own, in decl's document. A $ref with // siblings is aliased, keeping them. -func hoistSubSchema(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, decl *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) []ir.Diagnostic { +func hoistSubSchema(c lowering.Ctx, ts *compile.Types, decl *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) []ir.Diagnostic { s := annotation.At(decl) // c reads the document that declares pointer already (resolveAt), which is // where the bindings place decl. - hint, declared := c.HintAt(pointer) - if !declared { - hint = subSchemaHint(c.Bindings(), decl, pointer) - } - ref, diags := Ref(c, ts, anchors, TopLevelDepth, decl, pointer, hint) + ref, diags := Ref(c, ts, TopLevelDepth, decl, pointer) if _, ok := ts.Lookup(c.IDs().Coordinate(pointer)); ok { return diags } - _, aliasDiags := internAlias(c, ts, s.Node, pointer, hint, ref) + _, aliasDiags := internAlias(c, ts, s.Node, pointer, ref) diags = append(diags, aliasDiags...) // As in lowerComponentSchema: this alias is the first node the pointer owns, // so the annotations Ref had nowhere to put now have a home. - return append(diags, attachDeclaredAnnotations(c, ts, anchors, s.Node, pointer)...) + return append(diags, attachDeclaredAnnotations(c, ts, s.Node, pointer)...) } // onLoop reports whether the $ref at b lies on a loop of $refs, which the load diff --git a/compilers/openapi/internal/schema/refsite_internal_test.go b/compilers/openapi/internal/schema/refsite_internal_test.go index 9ef4ea44e..c386b5898 100644 --- a/compilers/openapi/internal/schema/refsite_internal_test.go +++ b/compilers/openapi/internal/schema/refsite_internal_test.go @@ -29,7 +29,7 @@ func TestRefTypeRef_SaysWhyAnUnboundRefIsUnresolved(t *testing.T) { t.Parallel() message := func(t *testing.T, l *lowerer, js *oas3.JSONSchema[oas3.Referenceable]) string { t.Helper() - ref, diags := refTypeRef(l.ctx, l.types, &l.anchors, js, "/components/schemas/A") + ref, diags := refTypeRef(l.ctx, l.types, js, "/components/schemas/A") assert.Equal(t, ir.TypeID("t/prim/any"), ref.Target) require.Len(t, diags, 1) return diags[0].Message @@ -75,7 +75,7 @@ func TestResolvePointer_PromisesWhatItNames(t *testing.T) { object := oas3.NewJSONSchemaFromSchema[oas3.Referenceable]( &oas3.Schema{Type: oas3.NewTypeFromString(oas3.SchemaTypeObject)}) - id, ok := resolvePointer(l.ctx, l.types, &l.anchors, deepPointer, object) + id, ok := resolvePointer(l.ctx, l.types, deepPointer, object) require.True(t, ok) assert.Equal(t, ids.Document{}.AnonType(deepPointer), id) _, built := l.types.Lookup(deepPointer) @@ -87,7 +87,7 @@ func TestResolvePointer_PromisesWhatItNames(t *testing.T) { require.True(t, built, "Finish builds what no lowering interned") assert.Equal(t, id, seeded) - _, ok = resolvePointer(l.ctx, l.types, &l.anchors, deepPointer, nil) + _, ok = resolvePointer(l.ctx, l.types, deepPointer, nil) assert.False(t, ok, "a target the bindings hold no schema at cannot be hoisted") } @@ -98,7 +98,7 @@ func TestResolveSchemaRef_UnboundDeepRefDropped(t *testing.T) { // nothing the bindings know it lands on, so the reference is dropped. js := oas3.NewJSONSchemaFromReference("#" + deepPointer) - _, ok := resolveSchemaRef(l.ctx, l.types, &l.anchors, js) + _, ok := resolveSchemaRef(l.ctx, l.types, js) assert.False(t, ok, "an unresolved deep sub-schema $ref is dropped, not synthesized") } @@ -110,7 +110,7 @@ func TestHoistSubSchema_BodyInternsAtPointer(t *testing.T) { object := oas3.NewJSONSchemaFromSchema[oas3.Referenceable]( &oas3.Schema{Type: oas3.NewTypeFromString(oas3.SchemaTypeObject)}) - assert.Empty(t, hoistSubSchema(l.ctx, l.types, &l.anchors, object, deepPointer), "an object body hoists cleanly") + assert.Empty(t, hoistSubSchema(l.ctx, l.types, object, deepPointer), "an object body hoists cleanly") seeded, _ := l.types.Lookup(deepPointer) assert.Equal(t, ids.Document{}.AnonType(deepPointer), seeded) td, ok := l.types.NodeAt(deepPointer) diff --git a/compilers/openapi/internal/schema/refsite_test.go b/compilers/openapi/internal/schema/refsite_test.go index d8c8948cb..ae4da6f28 100644 --- a/compilers/openapi/internal/schema/refsite_test.go +++ b/compilers/openapi/internal/schema/refsite_test.go @@ -193,7 +193,7 @@ components: l := lowererOver(lowering.New(0, unlisted.Doc, unlisted.Source, lowering.GroupByTags, lowering.Limits{}, lowering.StreamingMedia{}, lowering.ExtensionPromotions{}, lowering.NullableWidening{}, overlay.Origin{}).WithLoaded(&unlisted)) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) doc := l.document(t) unresolved := func(pointer jsontext.Pointer, message string) ir.Diagnostic { @@ -310,8 +310,9 @@ func TestSchema_EmptyFragmentRef(t *testing.T) { func TestSchema_EmptyStringRefMirrorBranches(t *testing.T) { t.Parallel() // An empty-string $ref has IsReference()==false (the ref value is "") yet a - // non-nil oas3 Ref pointer, exercising that mirror path in schema.Ref and the - // branchHint fallback. + // non-nil oas3 Ref pointer, exercising that mirror path in schema.Ref and a + // branch named by its position, since a $ref that suggests no name leaves it + // its own. spec := openapitest.ComponentSpec(` Owner: type: object properties: @@ -527,7 +528,8 @@ func TestCrossPathReference_SubtreeIsNamedByItsDeclaration(t *testing.T) { // a dependentSchemas entry (each kept verbatim in Unmodeled, ir-design §4.7), is // reached only by references. Two references into one such region, one aimed at // the outer position and one beneath it, name the inner position alike in -// either order, because positionHint names it from the pointer alone. +// either order, because its name is a function of its coordinate +// (lowering.NameAt). func TestUndeclaredRegion_TwoReferencesAgreeInBothOrders(t *testing.T) { t.Parallel() const preamble = "openapi: 3.1.0\ninfo: {title: O, version: \"1.0.0\"}\npaths:\n" @@ -951,10 +953,9 @@ func TestSchemaRef_ToAPointerEndingInSlashNamesAnEmptyKey(t *testing.T) { } // crossPathSchemaRoot names one shape /paths (or /webhooks) can hold a schema -// at, for TestCrossPathReference_SubtreeIsNamedByItsDeclaration. positionHint -// cannot replay the enclosing hint under any of them: it comes from an -// operationId, a response, a parameter or a webhook, none of which the -// pointer records. +// at, for TestCrossPathReference_SubtreeIsNamedByItsDeclaration. The pointer +// alone names none of them: each is named for an operationId, a response, a +// parameter or a webhook, which the namer reads from the document. type crossPathSchemaRoot struct { name string // block returns /b's (or the webhook's) top-level path-item entry with diff --git a/compilers/openapi/internal/schema/scalar.go b/compilers/openapi/internal/schema/scalar.go index 87f8869fc..ee0b6efec 100644 --- a/compilers/openapi/internal/schema/scalar.go +++ b/compilers/openapi/internal/schema/scalar.go @@ -20,13 +20,13 @@ import ( // wrapping the base primitive with an Encoding, so what the position wrote // never leaks onto the shared primitive every other declaration of that type // also resolves to. -func scalarTypeID(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, st oas3.SchemaType, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func scalarTypeID(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, st oas3.SchemaType, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { format := s.GetFormat() if annotation.DeclaresRedaction(s) { format = "" // a redaction, not a shape: the annotation readers hold it } if st == oas3.SchemaTypeString && format == "byte" { - return hoistScalar(c, ts, anchors, depth, s, ir.PrimBytes, "base64", new(ir.PrimString), pointer, hint) + return hoistScalar(c, ts, depth, s, ir.PrimBytes, "base64", new(ir.PrimString), pointer) } key := string(st) if format != "" { @@ -34,12 +34,12 @@ func scalarTypeID(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth } prim, known := spelling.Lookup(key) if !known { - return hoistScalar(c, ts, anchors, depth, s, baseForType(st), format, nil, pointer, hint) + return hoistScalar(c, ts, depth, s, baseForType(st), format, nil, pointer) } if !declaresContentVocabulary(s) { return ts.PrimID(prim), nil } - return hoistScalar(c, ts, anchors, depth, s, prim, "", nil, pointer, hint) + return hoistScalar(c, ts, depth, s, prim, "", nil, pointer) } // baseForType returns the base primitive for an unknown-format scalar of type st. @@ -65,11 +65,11 @@ func baseForType(st oas3.SchemaType) ir.PrimKind { // otherwise carry them: that fallback resolves to whatever node the pointer // already owns and returns early. A scalar that hoisted because it wrote a // format must not lose the bounds it wrote beside it (invariant 2). -func hoistScalar(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, base ir.PrimKind, formatName string, wire *ir.PrimKind, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func hoistScalar(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, base ir.PrimKind, formatName string, wire *ir.PrimKind, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { baseRef := ts.PrimRef(base) - enc, encDiags := scalarEncoding(c, ts, anchors, depth, s, formatName, &common, pointer, hint) + enc, encDiags := scalarEncoding(c, ts, depth, s, formatName, &common, pointer) diags = append(diags, encDiags...) if wire != nil { wireRef := ts.PrimRef(*wire) @@ -92,11 +92,11 @@ func hoistScalar(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth // name. formatName is the encoding name the format contributes — "base64" for // format: byte, an unrecognized format verbatim, "" when the pairing is already // captured by the primitive kind. -func scalarEncoding(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, - formatName string, common *ir.TypeCommon, pointer jsontext.Pointer, hint string, +func scalarEncoding(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, + formatName string, common *ir.TypeCommon, pointer jsontext.Pointer, ) (*ir.Encoding, []ir.Diagnostic) { name, diags := encodingName(c, s, formatName, common, pointer) - content, contentDiags := contentSchemaRef(c, ts, anchors, depth, s, pointer, hint) + content, contentDiags := contentSchemaRef(c, ts, depth, s, pointer) enc := &ir.Encoding{Name: name, MediaType: s.GetContentMediaType(), Schema: content} return enc, append(diags, contentDiags...) } @@ -134,12 +134,12 @@ func encodingName(c lowering.Ctx, s *oas3.Schema, formatName string, common *ir. // // The pointer it hoists at is the one the source wrote it at, which only this // declaration can name, so the node needs no namespace of its own (§4.3). -func contentSchemaRef(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (*ir.TypeRef, []ir.Diagnostic) { +func contentSchemaRef(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (*ir.TypeRef, []ir.Diagnostic) { cs := s.GetContentSchema() if !annotation.Written(cs) { return nil, nil } - ref, diags := Ref(c, ts, anchors, depth, cs, pointer+ids.Ptr("contentSchema"), compile.SubHint(hint, "content")) + ref, diags := Ref(c, ts, depth, cs, pointer+ids.Ptr("contentSchema")) return &ref, diags } diff --git a/compilers/openapi/internal/schema/schema.go b/compilers/openapi/internal/schema/schema.go index aea51785b..6f72f3fde 100644 --- a/compilers/openapi/internal/schema/schema.go +++ b/compilers/openapi/internal/schema/schema.go @@ -26,7 +26,7 @@ import ( // returns what was lowered so far; the caller — run — sees ctx.Err() at the // phase boundary immediately after and refuses the document there, so a partial // registry never becomes a Document. -func LowerComponentSchemas(ctx context.Context, c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex) []ir.Diagnostic { +func LowerComponentSchemas(ctx context.Context, c lowering.Ctx, ts *compile.Types) []ir.Diagnostic { comps := c.Doc.Components if comps == nil { return nil @@ -43,7 +43,7 @@ func LowerComponentSchemas(ctx context.Context, c lowering.Ctx, ts *compile.Type if ctx.Err() != nil { return diags } - diags = append(diags, lowerComponentSchema(c, ts, anchors, js, ids.Ptr("components", "schemas", name), name)...) + diags = append(diags, lowerComponentSchema(c, ts, js, ids.Ptr("components", "schemas", name))...) } return diags } @@ -53,20 +53,20 @@ func LowerComponentSchemas(ctx context.Context, c lowering.Ctx, ts *compile.Type // shared primitive/any or aliases another type. Without this, a component like // `MyId: {type: string, format: uuid}` would leave nothing at its component // pointer and every $ref to it would dangle (invariants 1 and 2). -func lowerComponentSchema(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer, name string) []ir.Diagnostic { +func lowerComponentSchema(c lowering.Ctx, ts *compile.Types, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) []ir.Diagnostic { s := annotation.At(js) - ref, diags := Ref(c, ts, anchors, TopLevelDepth, js, pointer, name) + ref, diags := Ref(c, ts, TopLevelDepth, js, pointer) // A body that interned the component's own node at its component ID needs no // alias, and its annotations were attached where it was lowered. if _, owned := ts.Lookup(c.IDs().Coordinate(pointer)); owned { return diags } - _, aliasDiags := internAlias(c, ts, s.Node, pointer, name, ref) + _, aliasDiags := internAlias(c, ts, s.Node, pointer, ref) diags = append(diags, aliasDiags...) // This alias is the first node the pointer owns, so the annotations // schemaBody had nowhere to put now have a home. if s.Node != nil { - diags = append(diags, attachDeclaredAnnotations(c, ts, anchors, s.Node, pointer)...) + diags = append(diags, attachDeclaredAnnotations(c, ts, s.Node, pointer)...) } return diags } @@ -94,21 +94,21 @@ const maxSchemaDepth = 256 // It defaults to annotation.HomeOwnNode deliberately: a position added later // inherits the lossless behaviour, and only a caller that can prove it already // carries the annotations opts out through CarriedRef. -func Ref(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer, hint string) (ir.TypeRef, []ir.Diagnostic) { - return schemaRefHomed(c, ts, anchors, depth, js, pointer, hint, annotation.HomeOwnNode) +func Ref(c lowering.Ctx, ts *compile.Types, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { + return schemaRefHomed(c, ts, depth, js, pointer, annotation.HomeOwnNode) } // CarriedRef lowers a schema whose annotations the calling position // already carries — a model property, a header, a parameter. Those callers // copy the declaration onto their own ir.Property/ir.Parameter, so the pointer // must not also hoist a node to hold it: one home per declaration. -func CarriedRef(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer, hint string) (ir.TypeRef, []ir.Diagnostic) { - return schemaRefHomed(c, ts, anchors, depth, js, pointer, hint, annotation.HomeCarrier) +func CarriedRef(c lowering.Ctx, ts *compile.Types, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { + return schemaRefHomed(c, ts, depth, js, pointer, annotation.HomeCarrier) } // schemaRefHomed is the shared body of the two entry points above; home only // reaches the two places that can hoist an annotation-holding alias. -func schemaRefHomed(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer, hint string, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { +func schemaRefHomed(c lowering.Ctx, ts *compile.Types, depth int, js *oas3.JSONSchema[oas3.Referenceable], pointer jsontext.Pointer, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { // depth counts the active frames of this function, which is where every // recursive descent re-enters. Incrementing on entry and passing the result // down is the parameter form of a counter that used to live on a shared @@ -118,9 +118,6 @@ func schemaRefHomed(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, dep return ts.PrimRef(ir.PrimAny), []ir.Diagnostic{c.DiagAt(ir.SeverityError, diag.DegradedConstruct, pointer, "schema nesting exceeds %d; lowered as any", maxSchemaDepth)} } - // The hint this lowering names the position by stands for any node a $ref - // later hoists here, where the position itself interned none (hoistSubSchema). - c.DeclareHint(pointer, hint) if !annotation.Written(js) { // None, or a null, which states nothing: an entry written so — a // property, a branch, a component, items — lowers as the empty schema. @@ -130,7 +127,7 @@ func schemaRefHomed(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, dep } if js.IsBool() { if b := js.GetBool(); b != nil && !*b { - id, diags := falseSchema(c, ts, pointer, hint) + id, diags := falseSchema(c, ts, pointer) return ir.TypeRef{Target: id}, diags } return ts.PrimRef(ir.PrimAny), nil @@ -139,22 +136,22 @@ func schemaRefHomed(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, dep // either reads as a bool), so GetSchema never returns nil here. schema := js.GetSchema() if resolve.IsRefSite(js, schema) { - return refSiteRef(c, ts, anchors, js, schema, pointer, hint, home) + return refSiteRef(c, ts, js, schema, pointer, home) } - return schemaBody(c, ts, anchors, depth, schema, pointer, hint, home) + return schemaBody(c, ts, depth, schema, pointer, home) } // falseSchema hoists a boolean `false` schema as a closed empty model (it // matches nothing), returning the interned ID and the one info diagnostic that // announces it. -func falseSchema(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func falseSchema(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { // Captured from inside the build rather than reported around it, which keeps // the report tied to the node actually being constructed. Reporting eagerly // would be indistinguishable today — a second visit produces the identical // diagnostic, which Diags.Append drops — so nothing observable rests on this; // it is the honest place for it, not a load-bearing one. var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { // Kept verbatim beside the approximation, which is what §4.8 asks of // every degraded lowering that loses something, and what the composition // half of this same rule already did. Without it a `false` schema and a @@ -184,15 +181,15 @@ func falseSchema(c lowering.Ctx, ts *compile.Types, pointer jsontext.Pointer, hi // hoisting (hoistSubSchema) arrives through Ref. A rule about the // declaration itself is checked here, above the dispatch that picks what it // lowers to (missingArrayItems). -func schemaBody(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, schema *oas3.Schema, pointer jsontext.Pointer, hint string, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { +func schemaBody(c lowering.Ctx, ts *compile.Types, depth int, schema *oas3.Schema, pointer jsontext.Pointer, home annotation.Home) (ir.TypeRef, []ir.Diagnostic) { diags := missingArrayItems(c, schema, pointer) - target, bodyDiags := lowerSchemaBody(c, ts, anchors, depth, schema, pointer, hint) + target, bodyDiags := lowerSchemaBody(c, ts, depth, schema, pointer) diags = append(diags, bodyDiags...) // A body's census ran inside lower(), against the node the walk actually // built, which is the only thing that can answer it. A collapsed union built // none, so it takes a $ref site's census, and its verdict homes an alias. unhomed := collapsedUnhomedKeywords(schema) - ref, homeDiags := homeDeclaration(c, ts, anchors, schema, target, pointer, hint, home, len(unhomed) > 0) + ref, homeDiags := homeDeclaration(c, ts, schema, target, pointer, home, len(unhomed) > 0) diags = append(diags, homeDiags...) if home != annotation.HomeOwnNode { return ref, diags // the carrier keeps them (PreserveRefSiteKeywords) @@ -202,8 +199,8 @@ func schemaBody(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth i // lowerSchemaBody lowers the body itself, handling the $dynamicRef expansion and // the oneOf/anyOf dispatch that precede structural lowering. -func lowerSchemaBody(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, schema *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeRef, []ir.Diagnostic) { - target, _, expanded, diags := dynamicExpansion(c, anchors, schema, pointer) +func lowerSchemaBody(c lowering.Ctx, ts *compile.Types, depth int, schema *oas3.Schema, pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { + target, _, expanded, diags := dynamicExpansion(c, schema, pointer) if expanded { diags = append(diags, c.DiagAt(ir.SeverityInfo, diag.DynamicRefExpanded, pointer+ids.Ptr("$dynamicRef"), "$dynamicRef expanded to %q, the one matching $dynamicAnchor in this document", target)) @@ -212,14 +209,14 @@ func lowerSchemaBody(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, de } if len(schema.GetOneOf()) > 0 || len(schema.GetAnyOf()) > 0 { if hasUnionSiblings(schema) { - id, unionDiags := lowerCoDeclaredUnion(c, ts, anchors, depth, schema, pointer, hint) + id, unionDiags := lowerCoDeclaredUnion(c, ts, depth, schema, pointer) ref, nullDiags := schemaNullRef(c, id, schema, siteAt(pointer)) return ref, slices.Concat(diags, unionDiags, nullDiags) } - ref, unionDiags := lowerOneOfAnyOf(c, ts, anchors, depth, schema, pointer, hint) + ref, unionDiags := lowerOneOfAnyOf(c, ts, depth, schema, pointer) return ref, append(diags, unionDiags...) } - id, bodyDiags := lower(c, ts, anchors, depth, schema, pointer, hint) + id, bodyDiags := lower(c, ts, depth, schema, pointer) ref, nullDiags := schemaNullRef(c, id, schema, siteAt(pointer)) return ref, slices.Concat(diags, bodyDiags, nullDiags) } @@ -234,19 +231,19 @@ func lowerSchemaBody(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, de // keyword the elected lowering never reads, such as `type: string` beside an // allOf, goes through preserveUnhomedKeywords, which asks the built node // whether it has a field for it. -func lower(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lower(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { d := dispatchOf(s) unhomed := func(id ir.TypeID, diags []ir.Diagnostic) (ir.TypeID, []ir.Diagnostic) { - owner, ownDiags := preserveUnhomedKeywords(c, ts, s, pointer, hint, id, d) + owner, ownDiags := preserveUnhomedKeywords(c, ts, s, pointer, id, d) return owner, append(diags, ownDiags...) } switch d.won { case "const": - return unhomed(hoistLiteral(c, ts, s.GetConst(), pointer, hint)) + return unhomed(hoistLiteral(c, ts, s.GetConst(), pointer)) case "enum": - return unhomed(lowerEnum(c, ts, s, pointer, hint)) + return unhomed(lowerEnum(c, ts, s, pointer)) case "allOf": - return unhomed(lowerAllOf(c, ts, anchors, depth, s, pointer, hint)) + return unhomed(lowerAllOf(c, ts, depth, s, pointer)) } // won is "" — the schema declares no family, so the type set decides. types := effectiveTypes(s) @@ -259,32 +256,32 @@ func lower(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s // not the Union's, so the applicator check does not apply. A keyword no // declared type's variant holds, such as a discriminator with no object // variant, is dropped (GitHub #852). - return lowerUnion(c, ts, anchors, depth, s, pointer, hint, types) + return lowerUnion(c, ts, depth, s, pointer, types) case len(types) == 1: - return unhomed(lowerTyped(c, ts, anchors, depth, s, pointer, hint, types[0])) + return unhomed(lowerTyped(c, ts, depth, s, pointer, types[0])) default: - return unhomed(lowerUntyped(c, ts, anchors, depth, s, pointer, hint)) + return unhomed(lowerUntyped(c, ts, depth, s, pointer)) } } // lowerTyped dispatches a single-typed schema to its structural or scalar form. -func lowerTyped(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string, st oas3.SchemaType) (ir.TypeID, []ir.Diagnostic) { +func lowerTyped(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer, st oas3.SchemaType) (ir.TypeID, []ir.Diagnostic) { switch st { case oas3.SchemaTypeObject: - return lowerModel(c, ts, anchors, depth, s, pointer, hint) + return lowerModel(c, ts, depth, s, pointer) case oas3.SchemaTypeArray: - return lowerArray(c, ts, anchors, depth, s, pointer, hint) + return lowerArray(c, ts, depth, s, pointer) default: - return scalarTypeID(c, ts, anchors, depth, s, st, pointer, hint) + return scalarTypeID(c, ts, depth, s, st, pointer) } } // lowerUntyped handles a schema with no declared type and no const, enum or // allOf, which lower() elects first: a property set makes it a model; anything // else is schemaless. -func lowerUntyped(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerUntyped(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { if props := s.GetProperties(); props != nil && props.Len() > 0 { - return lowerModel(c, ts, anchors, depth, s, pointer, hint) + return lowerModel(c, ts, depth, s, pointer) } return ts.PrimID(ir.PrimAny), nil } diff --git a/compilers/openapi/internal/schema/schema_test.go b/compilers/openapi/internal/schema/schema_test.go index 94ad28f8e..ed49718fd 100644 --- a/compilers/openapi/internal/schema/schema_test.go +++ b/compilers/openapi/internal/schema/schema_test.go @@ -94,7 +94,7 @@ func TestSchemaRef_BooleanAndUntypedShapes(t *testing.T) { func TestSchemaRef_EmptyEitherIsAny(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) - ref, diags := schema.Ref(l.ctx, l.types, &l.anchors, schema.TopLevelDepth, openapitest.EmptyEitherSchema(), "/p", "h") + ref, diags := schema.Ref(l.ctx, l.types, schema.TopLevelDepth, openapitest.EmptyEitherSchema(), "/p") assert.Equal(t, ir.TypeID("t/prim/any"), ref.Target) assert.Empty(t, diags, "an empty either lowers to any without complaint") } @@ -167,7 +167,7 @@ func vocabularyCore() []vocabCase { }, { keyword: "$dynamicAnchor", write: "$dynamicAnchor: m", schemas: stringSchema, - excluded: "read as a reference target by dynamicAnchors, which is what lets a " + + excluded: "read as a reference target by anchors.Build, which is what lets a " + "$dynamicRef expand; declaring one says nothing about the shape", }, { @@ -804,7 +804,7 @@ func TestNullSchema_AnEntryIsTheEmptySchemaAndAKeywordIsLeftOut(t *testing.T) { " - {type: object, properties: {p: {type: string, enum: [a, b]}}}\n"+ " - {type: object, properties: {p: null}}\n")) l.ctx.Limits = lowering.Limits{MaxEnumMembers: 1} - diags := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors) + diags := schema.LowerComponentSchemas(t.Context(), l.ctx, l.types) m := asKind[*ir.Model](t, typeByName(l.document(t), "A")) require.Len(t, m.Properties, 1) diff --git a/compilers/openapi/internal/schema/union.go b/compilers/openapi/internal/schema/union.go index e16045a1a..4c9c8b850 100644 --- a/compilers/openapi/internal/schema/union.go +++ b/compilers/openapi/internal/schema/union.go @@ -24,22 +24,22 @@ import ( // instead (lowerNullOnlyUnion, GitHub #416); everything else becomes a Union // with one Variant per non-null branch (oneOf exclusive, anyOf not), never // collapsing a union into optional fields. -func lowerOneOfAnyOf(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeRef, []ir.Diagnostic) { - if inner, ip, i, ok := nullUnionCollapse(s, pointer); ok { - ref, diags := Ref(c, ts, anchors, depth, inner, ip, branchHint(c.Bindings(), inner, i)) +func lowerOneOfAnyOf(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { + if inner, ip, ok := nullUnionCollapse(s, pointer); ok { + ref, diags := Ref(c, ts, depth, inner, ip) // The null branch declares the null, so no heuristic's marker the // branch's own reference carried says anything of it. ref.Nullable, ref.Inferred = true, "" return ref, diags } if allNullUnion(s) { - return lowerNullOnlyUnion(c, ts, s, pointer, hint) + return lowerNullOnlyUnion(c, ts, s, pointer) } var diags []ir.Diagnostic - tid := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { - def, unionDiags := buildUnion(c, ts, anchors, s, common, pointer, - func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, vhint string) ir.TypeRef { - ref, refDiags := Ref(c, ts, anchors, depth, b, vptr, vhint) + tid := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { + def, unionDiags := buildUnion(c, ts, s, common, pointer, + func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer) ir.TypeRef { + ref, refDiags := Ref(c, ts, depth, b, vptr) diags = append(diags, refDiags...) return ref }) @@ -51,20 +51,15 @@ func lowerOneOfAnyOf(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, de } // nullUnionCollapse detects a oneOf/anyOf with exactly one non-null branch -// beside `type: null` branches, and returns that branch's schema, pointer and -// index, so it lowers as nullable X, not a union node (ir-design §3.3). -// -// It is named by the branch's own hint (branchHint), which an outside $ref -// naming the same pointer also derives. Whichever lowers first interns the -// node, so any other hint would make the document depend on declaration order -// (GitHub #281). +// beside `type: null` branches, and returns that branch's schema and pointer, +// so it lowers as nullable X, not a union node (ir-design §3.3). // // A schema declaring both combinators collapses neither: the co-declared anyOf // conjoins, so the position is not nullable X. The Union keeps the unused // combinator (preserveUnusedCombinator), which X's shared node cannot. -func nullUnionCollapse(s *oas3.Schema, pointer jsontext.Pointer) (*oas3.JSONSchema[oas3.Referenceable], jsontext.Pointer, int, bool) { +func nullUnionCollapse(s *oas3.Schema, pointer jsontext.Pointer) (*oas3.JSONSchema[oas3.Referenceable], jsontext.Pointer, bool) { if len(s.GetOneOf()) > 0 && len(s.GetAnyOf()) > 0 { - return nil, "", 0, false + return nil, "", false } variants, key, _ := unionBranches(s) var nonNull *oas3.JSONSchema[oas3.Referenceable] @@ -78,9 +73,9 @@ func nullUnionCollapse(s *oas3.Schema, pointer jsontext.Pointer) (*oas3.JSONSche nonNullCount++ } if nullCount == 0 || nonNullCount != 1 { - return nil, "", 0, false + return nil, "", false } - return nonNull, pointer + ids.Ptr(key, strconv.Itoa(nonNullIdx)), nonNullIdx, true + return nonNull, pointer + ids.Ptr(key, strconv.Itoa(nonNullIdx)), true } // isNullSchema reports whether a variant schema is the bare null-typed schema. @@ -135,14 +130,14 @@ func oneOfAnyOfHasNull(s *oas3.Schema) bool { // it, and the other one too when both are declared, carries no shape a Union // or a Scalar's fields could hold, so it is kept verbatim under Unmodeled // instead of dropped (GitHub #416; docs/compilers/openapi.md#union-null-only). -func lowerNullOnlyUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeRef, []ir.Diagnostic) { +func lowerNullOnlyUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeRef, []ir.Diagnostic) { // inner is the shared `any` primitive: no schema pointer ever interns at // its ID, so this position never already owns it. Hoist an alias // unconditionally, rather than testing for a case that cannot occur, so // the preserved union attaches to a node this pointer owns and not to the // primitive every other unrelated position also resolves to. inner := ts.PrimID(ir.PrimAny) - owner, diags := internAlias(c, ts, s, pointer, hint, ir.TypeRef{Target: inner, Nullable: true}) + owner, diags := internAlias(c, ts, s, pointer, ir.TypeRef{Target: inner, Nullable: true}) return ir.TypeRef{Target: owner, Nullable: true}, append(diags, preserveNullOnlyUnion(c, ts, owner, s, pointer)...) } @@ -252,7 +247,7 @@ func preserveUnusedCombinator(c lowering.Ctx, p *ir.Unmodeled, s *oas3.Schema, w // is a parameter so the distributed lowering can conjoin the sibling composition // into each branch while sharing every other rule about union shape — null-branch // stripping, variant hints, exclusivity, pointer derivation. -type variantTypeFunc func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, vhint string) ir.TypeRef +type variantTypeFunc func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer) ir.TypeRef // buildUnion assembles the Union node for a oneOf/anyOf schema, attaching a // discriminator when one is declared. common is already built by the caller @@ -262,7 +257,7 @@ type variantTypeFunc func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext. // knowing what each variant names (readsThrough). A union the discriminator // routes is WireTagged, an anyOf as much as a oneOf: the tag property's value // names its variant (ir-design §4.4). -func buildUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer, variantType variantTypeFunc) (ir.TypeDef, []ir.Diagnostic) { +func buildUnion(c lowering.Ctx, ts *compile.Types, s *oas3.Schema, common ir.TypeCommon, pointer jsontext.Pointer, variantType variantTypeFunc) (ir.TypeDef, []ir.Diagnostic) { branches, key, exclusive := unionBranches(s) diags := preserveUnusedCombinator(c, &common.Unmodeled, s, key, pointer) variants := make([]ir.Variant, 0, len(branches)) @@ -270,11 +265,10 @@ func buildUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3 if isNullSchema(b) { continue // null branches lift to the enclosing ref's Nullable bit } - vh := branchHint(c.Bindings(), b, i) vptr := pointer + ids.Ptr(key, strconv.Itoa(i)) - t := variantType(b, vptr, vh) + t := variantType(b, vptr) variants = append(variants, ir.Variant{ - Name: compile.NamingHint(decoratedBranchHint(c, ts, b, t, vh)), + Name: compile.NamingHint(variantHint(c, ts, b, vptr, t)), Type: t, }) } @@ -283,35 +277,37 @@ func buildUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, s *oas3 Variants: variants, Exclusive: exclusive, } - disc, discDiags := lowerDiscriminator(c, ts, anchors, s, nil, variants, pointer) + disc, discDiags := lowerDiscriminator(c, ts, s, nil, variants, pointer) u.Discriminator = disc u.WireTagged = disc != nil return u, append(diags, discDiags...) } -// decoratedBranchHint is the hint a union variant takes for b, which lowered -// to t: hint, unless b is written inline and t is a decorator (ir.Decorated) -// over the $ref b's allOf names, when it is the hint that $ref takes as a -// branch of its own (branchHint). 3.0 annotates a branch only by wrapping its -// $ref, so `{allOf: [{$ref: X}], description: …}` names its variant as 3.1's -// `{$ref: X, description: …}` does (GitHub #1018). The node t keeps hint: a -// decorator collapses into its reference wherever it is used. -func decoratedBranchHint(c lowering.Ctx, ts *compile.Types, b *oas3.JSONSchema[oas3.Referenceable], t ir.TypeRef, hint string) string { +// variantHint is the hint the union variant for the branch b written at vptr, +// which lowered to t, takes: the branch's own (lowering.NameAt), unless b is +// written inline and t is a decorator (ir.Decorated) over the $ref b's allOf +// names, when it is the name that $ref suggests (lowering.NameThrough), as a +// branch holding it would be named. 3.0 annotates a branch only by wrapping +// its $ref, so `{allOf: [{$ref: X}], description: …}` names its variant as +// 3.1's `{$ref: X, description: …}` does (GitHub #1018). The node t keeps the +// branch's own: a decorator collapses into its reference wherever it is used. +func variantHint(c lowering.Ctx, ts *compile.Types, b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, t ir.TypeRef) string { + own := lowering.NameAt(c, vptr) s := annotation.SchemaOf(b) if s == nil || isRefBranch(b) { - return hint + return own } // The flattened list, as b lowers: a $ref a nested wrapper holds names the // variant as it would written at the top. - branches := allOfBranches(s, "").branches + branches := allOfBranches(s, vptr).branches base := selectAllOfBase(c.Bindings(), branches) if base < 0 { - return hint + return own } if td, ok := ts.Node(t.Target); !ok || !decorates(td) { - return hint + return own } - return cmp.Or(targetHint(c.Bindings(), branches[base].schema), hint) + return cmp.Or(lowering.NameThrough(c, branches[base].pointer), own) } // decorates reports whether td is a decorator (ir.Decorated). @@ -326,13 +322,13 @@ func decorates(td ir.TypeDef) bool { // KNOWN GAP (GitHub #868): every integer is a number, so a type array naming // both admits values that match two variants, which Exclusive rejects; with an // integer width both variants are the same primitive. -func lowerUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string, types []oas3.SchemaType) (ir.TypeID, []ir.Diagnostic) { +func lowerUnion(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer, types []oas3.SchemaType) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { variants := make([]ir.Variant, 0, len(types)) for i, st := range types { vptr := pointer + ids.Ptr("type", strconv.Itoa(i)) - vid, vDiags := lowerTyped(c, ts, anchors, depth, s, vptr, hint, st) + vid, vDiags := lowerTyped(c, ts, depth, s, vptr, st) diags = append(diags, vDiags...) variants = append(variants, ir.Variant{ Name: compile.NamingHint(string(st)), @@ -496,16 +492,16 @@ const coDeclaredUnionWhyPrefix = "oneOf/anyOf co-declared with structural keywor // lowerCoDeclaredUnion lowers a schema whose oneOf/anyOf sits beside structural // keywords, per classifyUnionSiblings. -func lowerCoDeclaredUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerCoDeclaredUnion(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { beside := func(reason ir.UnmodeledReason, why string) (ir.TypeID, []ir.Diagnostic) { if why != "" { why = coDeclaredUnionWhyPrefix + why } - return lowerBesideUnmodeledUnion(c, ts, anchors, depth, s, pointer, hint, reason, why) + return lowerBesideUnmodeledUnion(c, ts, depth, s, pointer, reason, why) } switch classifyUnionSiblings(c, s) { case unionDistributed: - return lowerDistributedUnion(c, ts, anchors, depth, s, pointer, hint) + return lowerDistributedUnion(c, ts, depth, s, pointer) case unionValidationOnly: return beside(ir.ReasonValidationOnly, "") case unionUncomposableBody: @@ -554,15 +550,15 @@ func diagUnresolvedBranches(c lowering.Ctx, s *oas3.Schema, pointer jsontext.Poi // is and why says what stopped a classified lowering; lowerCoDeclaredUnion picks // both from classifyUnionSiblings' verdict. A discriminator the body's model // holds is settled once every node is interned (deferDiscriminatorBesideUnion). -func lowerBesideUnmodeledUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string, reason ir.UnmodeledReason, why string) (ir.TypeID, []ir.Diagnostic) { - inner, diags := lower(c, ts, anchors, depth, s, pointer, hint) +func lowerBesideUnmodeledUnion(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer, reason ir.UnmodeledReason, why string) (ir.TypeID, []ir.Diagnostic) { + inner, diags := lower(c, ts, depth, s, pointer) owner := inner if got, _ := ts.Lookup(c.IDs().Coordinate(pointer)); got != inner { // The structural body reduced to a shared/aliased target; hoist an alias // so the preserved union attaches to a node this pointer owns, never to a // shared primitive. var aliasDiags []ir.Diagnostic - owner, aliasDiags = internAlias(c, ts, s, pointer, hint, ir.TypeRef{Target: inner}) + owner, aliasDiags = internAlias(c, ts, s, pointer, ir.TypeRef{Target: inner}) diags = append(diags, aliasDiags...) } deferDiscriminatorBesideUnion(c, ts, owner, s, pointer) @@ -618,13 +614,13 @@ const refSiteUnionWhy = "oneOf/anyOf beside a $ref conjoin with it rather than c // `(S ∧ X) | (S ∧ Y)`. Each variant is a Model classifying S's Base/Mixins and // own properties (ir-design §4.3) alongside its branch, so the composition is // carried on every variant rather than merged into one or dropped. -func lowerDistributedUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, s *oas3.Schema, pointer jsontext.Pointer, hint string) (ir.TypeID, []ir.Diagnostic) { +func lowerDistributedUnion(c lowering.Ctx, ts *compile.Types, depth int, s *oas3.Schema, pointer jsontext.Pointer) (ir.TypeID, []ir.Diagnostic) { var diags []ir.Diagnostic - id := internNode(c, ts, pointer, hint, func(common ir.TypeCommon) ir.TypeDef { - body := composedBody{schema: s, comp: allOfBranches(s, pointer), hint: hint, id: common.ID} - def, unionDiags := buildUnion(c, ts, anchors, s, common, pointer, - func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, vhint string) ir.TypeRef { - ref, variantDiags := composedVariant(c, ts, anchors, depth, body, b, vptr, vhint) + id := internNode(c, ts, pointer, func(common ir.TypeCommon) ir.TypeDef { + body := composedBody{schema: s, comp: allOfBranches(s, pointer), id: common.ID} + def, unionDiags := buildUnion(c, ts, s, common, pointer, + func(b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer) ir.TypeRef { + ref, variantDiags := composedVariant(c, ts, depth, body, b, vptr) diags = append(diags, variantDiags...) return ref }) @@ -642,7 +638,6 @@ func lowerDistributedUnion(c lowering.Ctx, ts *compile.Types, anchors *AnchorInd type composedBody struct { schema *oas3.Schema comp allOfComposition // schema's flattened allOf, built once for every variant - hint string id ir.TypeID } @@ -656,16 +651,19 @@ type composedBody struct { // which already denotes the branch schema. Interning there made the two race // for one pointer, so a $ref to the branch could find either the bare branch or // the full variant, depending on lowering order (§4.3). -func composedVariant(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, body composedBody, - b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, vhint string, +func composedVariant(c lowering.Ctx, ts *compile.Types, depth int, body composedBody, + b *oas3.JSONSchema[oas3.Referenceable], vptr jsontext.Pointer, ) (ir.TypeRef, []ir.Diagnostic) { // The branch lowers through the ordinary schema path, at its own pointer, so // it keeps whatever it declares beside the $ref and a reference to that // pointer still finds the branch rather than the variant. - branch, diags := Ref(c, ts, anchors, depth, b, vptr, vhint) + branch, diags := Ref(c, ts, depth, b, vptr) id := c.IDs().ComposedType(vptr) - common := commonFor(c, id, vptr, compile.SubHint(body.hint, vhint)) - def, variantDiags := buildComposedVariant(c, ts, anchors, depth, body, branch.Target, common) + common := commonFor(c, id, vptr) + // The Model conjoins the union's body with the branch, so it is named for + // both. + common.Name = compile.NamingHint(compile.SubHint(lowering.NameAt(c, body.comp.pointer), lowering.NameAt(c, vptr))) + def, variantDiags := buildComposedVariant(c, ts, depth, body, branch.Target, common) ts.Register(id, def) return composedVariantRef(c, id, body.schema, branch), append(diags, variantDiags...) } @@ -696,14 +694,14 @@ func composedVariantRef(c lowering.Ctx, id ir.TypeID, body *oas3.Schema, branch // (GitHub #407). Each variant carries a copy, since a bound on the schema // constrains both sides of the disjunction and this path never runs the // pointer's own census. -func buildComposedVariant(c lowering.Ctx, ts *compile.Types, anchors *AnchorIndex, depth int, body composedBody, branch ir.TypeID, common ir.TypeCommon) (ir.TypeDef, []ir.Diagnostic) { +func buildComposedVariant(c lowering.Ctx, ts *compile.Types, depth int, body composedBody, branch ir.TypeID, common ir.TypeCommon) (ir.TypeDef, []ir.Diagnostic) { cons, diags := ConstraintsOf(c, &common.Unmodeled, body.schema, body.comp.pointer) m := &ir.Model{TypeCommon: common, Constraints: cons} idx := propIndex(c, ts, m) - diags = append(diags, fillAllOf(c, ts, anchors, depth, m, idx, body.comp)...) - diags = append(diags, fillModelProperties(c, ts, anchors, depth, idx, body.schema, body.comp.pointer)...) + diags = append(diags, fillAllOf(c, ts, depth, m, idx, body.comp)...) + diags = append(diags, fillModelProperties(c, ts, depth, idx, body.schema, body.comp.pointer)...) requireMembers(m, compositionRequired(body.schema.GetRequired(), body.comp)) - diags = append(diags, fillAdditional(c, ts, anchors, depth, m, body.schema, body.comp.pointer, body.hint)...) + diags = append(diags, fillAdditional(c, ts, depth, m, body.schema, body.comp.pointer)...) diags = append(diags, applyFalseBranches(c, m, body.comp)...) conjoinBranch(m, branch) // The tag is the enclosing schema's: it is what a base's mapping names, and diff --git a/compilers/openapi/internal/schema/union_internal_test.go b/compilers/openapi/internal/schema/union_internal_test.go index 6a718fdb5..48e8fce5a 100644 --- a/compilers/openapi/internal/schema/union_internal_test.go +++ b/compilers/openapi/internal/schema/union_internal_test.go @@ -54,7 +54,7 @@ func TestPreserveNullOnlyUnion_MissingNode(t *testing.T) { func TestPreserveNullOnlyUnion_NothingToKeep(t *testing.T) { t.Parallel() l := newRawLowerer(&soa.OpenAPI{}) - id := internNode(l.ctx, l.types, "/components/schemas/S", "", func(cm ir.TypeCommon) ir.TypeDef { + id := internNode(l.ctx, l.types, "/components/schemas/S", func(cm ir.TypeCommon) ir.TypeDef { return &ir.Scalar{TypeCommon: cm} }) diags := preserveNullOnlyUnion(l.ctx, l.types, id, &oas3.Schema{}, "/components/schemas/S") diff --git a/compilers/openapi/internal/schema/union_test.go b/compilers/openapi/internal/schema/union_test.go index f5947f249..6b91b64b0 100644 --- a/compilers/openapi/internal/schema/union_test.go +++ b/compilers/openapi/internal/schema/union_test.go @@ -114,7 +114,7 @@ func TestDistributedUnion_VariantCarriesTheBranchNullability(t *testing.T) { // or keyword. Choice/oneOf/0 targets X's inline first branch and Choice/oneOf/1 // a plain structural position (A's items). The other variants target a branch // that is itself a $ref, so naming them chases it on to its own target -// (targetHint): Choice/oneOf/2 and Z/oneOf/0 take two hops to reach Y's items, +// (naming.Namer.Through): Choice/oneOf/2 and Z/oneOf/0 take two hops to reach Y's items, // and Choice/oneOf/3 three, so a hop cap below three fails that row. func TestUnionVariant_IsNamedAsTheNodeItHolds(t *testing.T) { t.Parallel() @@ -153,16 +153,12 @@ func TestUnionVariant_IsNamedAsTheNodeItHolds(t *testing.T) { } // TestUnionVariant_UnderPathsIsNamedAsTheNodeItHolds is the same property for -// variants naming positions in a response body, where the pointer does not spell -// the enclosing hint. A branch is still named by its ordinal, a branch holding -// a $ref after its target — through a second branch here, for two hops — and a -// property by its key, so each variant still agrees with the node it holds. -// Before this held outside /components/schemas the first three variants read -// 0, 1 and 2. -// -// A position the response itself names — the body, or its items — is not -// covered: its hint comes from the response, which the pointer does not spell -// (GitHub #729). +// variants naming positions under /paths, where the pointer does not spell the +// enclosing hint. A branch is named by its ordinal, a branch holding a $ref +// after its target — through a second branch here, for two hops — and a +// property by its key. A position its response or a parameter names, the body +// itself or a parameter's schema, is named for them (GitHub #729). Each variant +// agrees with the node it holds. func TestUnionVariant_UnderPathsIsNamedAsTheNodeItHolds(t *testing.T) { t.Parallel() const body = "#/paths/~1x/get/responses/200/content/application~1json/schema/properties/" @@ -172,6 +168,8 @@ paths: /x: get: operationId: getX + parameters: + - {name: filterBy, in: query, schema: {type: object, properties: {f: {type: string}}}} responses: "200": description: ok @@ -195,28 +193,67 @@ components: - $ref: '`+body+`choice/oneOf/1' - $ref: '`+body+`choice/oneOf/2' - $ref: '`+body+`held' + - $ref: '#/paths/~1x/get/responses/200/content/application~1json/schema' + - $ref: '#/paths/~1x/get/parameters/0/schema' `) openapitest.RequireNoErrorDiags(t, diags) choice, ok := typeByName(doc, "Choice").(*ir.Union) require.True(t, ok, "Choice should be a union") - require.Len(t, choice.Variants, 4) + require.Len(t, choice.Variants, 6) got := variantsNamingTheirNodes(t, doc, map[string]ir.Variant{ "an inline branch": choice.Variants[0], "a branch holding a $ref": choice.Variants[1], "a branch holding a $ref to one": choice.Variants[2], "a property holding a $ref": choice.Variants[3], + "the response's body": choice.Variants[4], + "a parameter's schema": choice.Variants[5], }) want := map[string]string{ "an inline branch": "variant_0", "a branch holding a $ref": "t", "a branch holding a $ref to one": "t", "a property holding a $ref": "held", + "the response's body": "response", + "a parameter's schema": "filter_by", } assert.Empty(t, cmp.Diff(want, got)) } +// TestUnionVariant_AWholeDocumentIsNamedForIt is the same property for a +// branch whose $ref names another document whole: the document's root, which +// no owner holds, is named for the document, so the variant and the node it +// holds are both named after the file, as is a wrapped branch's variant +// (GitHub #1018). A position inside the document keeps its own name. +func TestUnionVariant_AWholeDocumentIsNamedForIt(t *testing.T) { + t.Parallel() + doc, diags, _ := compileBeside(t, map[string]string{ + "Cat.yaml": "type: object\nproperties:\n toy: {type: object, properties: {name: {type: string}}}\n", + }, `openapi: 3.0.3 +info: {title: T, version: "1"} +paths: {} +components: + schemas: + Pet: + oneOf: + - $ref: './Cat.yaml' + - {allOf: [{$ref: './Cat.yaml'}], description: a cat} + - $ref: './Cat.yaml#/properties/toy' +`) + openapitest.RequireNoErrorDiags(t, diags) + + pet, ok := typeByName(doc, "Pet").(*ir.Union) + require.True(t, ok, "Pet should be a union") + require.Len(t, pet.Variants, 3) + got := variantsNamingTheirNodes(t, doc, map[string]ir.Variant{ + "the document": pet.Variants[0], + "a position in it": pet.Variants[2], + }) + assert.Empty(t, cmp.Diff(map[string]string{"the document": "cat_yaml", "a position in it": "toy"}, got)) + assert.Equal(t, "cat_yaml", pet.Variants[1].Name.Hint, "the wrapped branch is named for its $ref") +} + func TestOneOf_WithDiscriminator(t *testing.T) { t.Parallel() spec := openapitest.ComponentSpec(` Cat: @@ -750,7 +787,7 @@ func TestOneOf_CoDeclaredDistributionIsOrderIndependent(t *testing.T) { // takes instead. The // collapse lowers the surviving branch at the branch pointer but used to hand it // the *enclosing* schema's hint, while an outside $ref naming that same pointer -// derives variant_ through subSchemaHint — so whichever lowering arrived +// derived variant_ from the pointer — so whichever lowering arrived // first decided the name and the two declaration orders produced two different // documents, with no diagnostic on either side (GitHub #281). // @@ -1161,7 +1198,7 @@ func TestUnionCombinators_UnpreservableIsNotAnnounced(t *testing.T) { // resolves through the union to the shared primitive, so the outside $ref alone // hoists a node at the branch pointer and no hint ever has to agree with // another. Pinning the union's own variant to that node puts both lowerings on -// the pointer (branchHint, subSchemaHint, #181). +// the pointer (#181). func TestUnionCombinators_KeepingIsOrderIndependent(t *testing.T) { t.Parallel() host := ` S: diff --git a/compilers/openapi/internal/sourceindex/sourceindex.go b/compilers/openapi/internal/sourceindex/sourceindex.go index 6913bc076..da8f5aeae 100644 --- a/compilers/openapi/internal/sourceindex/sourceindex.go +++ b/compilers/openapi/internal/sourceindex/sourceindex.go @@ -10,6 +10,7 @@ package sourceindex import ( + "math" "slices" yaml "gopkg.in/yaml.v3" @@ -120,14 +121,6 @@ func (x Index) TaggedMappings() []*yaml.Node { return slices.Clone(x.taggedMappi // document. func (x Index) Truncated() bool { return x.truncated } -// frame is one entry of the walk's explicit stack: a node to visit at a known -// depth, or the exit of a node whose children have all been visited. -type frame struct { - n *yaml.Node - depth int - exit bool -} - // walker is one walk's working state: the ancestors of the node being visited, // the deepest nesting met so far under each node still open, and the measure // an alias's target is taken from. @@ -145,72 +138,44 @@ type walker struct { // The raw parse tree is a tree: aliasing adds only edges this walk never // follows, so no node is reached twice and the ancestor set is exactly the path // from the root. An alias nests as its target, which the measure takes once. -// The walk is iterative and bounded by maxNodes, and it never stops at a +// The walk is bounded by maxNodes (nodeview.Walk), and it never stops at a // finding, or one answer would hide another. func (x *Index) walk(maxNodes int64) { w := walker{x: x, ancestors: map[*yaml.Node]bool{}, measure: NewMeasure(maxNodes)} - stack := []frame{{n: x.root}} - - for len(stack) > 0 { - f := stack[len(stack)-1] - stack = stack[:len(stack)-1] - - if f.exit { - w.leave(f) - continue - } - if f.n == nil { - continue // never produced by a parse; not counted rather than dereferenced - } - - x.nodes++ - if x.nodes > maxNodes { - x.truncated = true - return - } - stack = w.visit(stack, f) - } - x.truncated = w.measure.Truncated() + out := nodeview.Walk{Nodes: int(min(maxNodes, math.MaxInt)), Visit: w.visit, Leave: w.leave}.Run(x.root) + x.nodes = int64(out.Visited) + x.truncated = out.Cut || w.measure.Truncated() } -// visit records what f's node answers and returns the stack with its children -// and its exit pushed, or unchanged for a node with none. An alias is a leaf: -// yaml.v3 gives it empty Content, and its target is not its child. -func (w *walker) visit(stack []frame, f frame) []frame { - n := f.n +// visit records what n answers, at depth below the root. An alias is a leaf: +// yaml.v3 gives it empty Content, and its target is not its child, so it is +// finished here as the nesting of what it names. Any other node is opened, for +// leave to close once its children are measured. +func (w *walker) visit(n *yaml.Node, depth int) nodeview.Step { w.x.recordTag(n) - tracked := f.depth <= maxTrackedDepth + tracked := depth <= maxTrackedDepth if n.Kind == yaml.AliasNode { if tracked { w.x.recordAlias(n, w.ancestors) } w.finish(w.measure.Of(n)) - return stack + return nodeview.Prune } - if len(n.Content) == 0 { - w.finish(Level(n)) - return stack - } - if tracked { w.ancestors[n] = true } w.tallest = append(w.tallest, 0) - stack = append(stack, frame{n: n, depth: f.depth, exit: true}) - for _, c := range slices.Backward(n.Content) { - stack = append(stack, frame{n: c, depth: f.depth + 1}) - } - return stack + return nodeview.Descend } -// leave closes the node f exits once its children are measured. -func (w *walker) leave(f frame) { - if f.depth <= maxTrackedDepth { - delete(w.ancestors, f.n) +// leave closes n once its children are measured. +func (w *walker) leave(n *yaml.Node, depth int) { + if depth <= maxTrackedDepth { + delete(w.ancestors, n) } tallest := w.tallest[len(w.tallest)-1] w.tallest = w.tallest[:len(w.tallest)-1] - w.finish(tallest + Level(f.n)) + w.finish(tallest + Level(n)) } // finish hands a finished node's nesting to the node holding it, or to the diff --git a/compilers/openapi/internal/value/scalar.go b/compilers/openapi/internal/value/scalar.go new file mode 100644 index 000000000..d68507510 --- /dev/null +++ b/compilers/openapi/internal/value/scalar.go @@ -0,0 +1,190 @@ +package value + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/ir" +) + +// ScalarKind is the kind of value a YAML scalar holds, read by its resolved +// tag. +type ScalarKind uint8 + +const ( + // ScalarNull is a null. + ScalarNull ScalarKind = iota + // ScalarBool is true or false. + ScalarBool + // ScalarNumber is a number, its exact decimal in Scalar.Num. + ScalarNumber + // ScalarString is a string, its text in Scalar.Text. + ScalarString + // ScalarTimestamp is a YAML timestamp. JSON has no such type, and the + // resolved time is derivable from the text while the text is not from the + // time, so the text is what a renderer keeps (GitHub #242). + ScalarTimestamp + // ScalarBinary is base64 text tagged !!binary, its bytes in Scalar.Bytes. + ScalarBinary + // ScalarTagged is a scalar under a tag YAML gives no type, such as a + // document's own `!acme`: text a renderer either keeps or refuses. + ScalarTagged +) + +// Scalar is one YAML scalar classified: its kind, its text as written, and +// the value its kind reads that text as. +type Scalar struct { + Kind ScalarKind + Text string + Bool bool + Num ir.BigVal + Bytes []byte +} + +// Classify reads the scalar node n by its resolved tag, never by the schema +// around it: the one classification a value lowered into the IR and a payload +// kept verbatim both render, so the two never read one scalar as two kinds +// (GitHub #783, #784). ShortTag resolves a node built by hand as a parse does. +// +// A scalar whose text its tag does not admit — `!!bool maybe`, `!!null abc`, +// a !!binary that is not base64, `.inf` and `.nan`, which no JSON number +// spells — is an error, as yaml.v3's own decoding refuses each (GitHub #245). +func Classify(n *yaml.Node) (Scalar, error) { + if n == nil || n.Kind != yaml.ScalarNode { + return Scalar{}, errors.New("openapi: not a scalar") + } + s := Scalar{Text: n.Value} + switch tag := n.ShortTag(); tag { + case "!!null": + return s, decodes(n, new(any), "null") + case "!!bool": + s.Kind = ScalarBool + return s, decodes(n, &s.Bool, "bool") + case "!!int", "!!float": + num, err := NumericLiteral(n) + if err != nil { + return Scalar{}, fmt.Errorf("openapi: numeric literal %q: %w", n.Value, err) + } + s.Kind, s.Num = ScalarNumber, num + return s, nil + case "!!str", "!!merge": + return text(n, s), nil + case "!!timestamp": + s.Kind = ScalarTimestamp + return s, decodes(n, new(time.Time), "timestamp") + case "!!binary": + return binary(n, s) + default: + s.Kind = ScalarTagged + return s, nil + } +} + +// text classifies a scalar YAML resolved to a string. A plain one that spells +// a finite decimal is a number all the same: yaml.v3 resolves a number past +// float64's range to a string, and the source wrote a number (GitHub #16). A +// quoted one, or one tagged !!str outright, is the string it says. +// +// `<<` written as a value, which yaml.v3 tags !!merge wherever it stands, is +// the text it spells: only a key merges. +func text(n *yaml.Node, s Scalar) Scalar { + s.Kind = ScalarString + if n.Style != 0 || n.ShortTag() != "!!str" { + return s + } + if num, err := ir.NewBigVal(n.Value); err == nil { + s.Kind, s.Num = ScalarNumber, num + } + return s +} + +// binary classifies a !!binary scalar, decoding its base64 to the bytes it +// holds. yaml.v3 base64-decodes a !!binary node into a string (it rejects a +// []byte target), so the decode goes through one. The error names no text, +// which may be any length of bytes. +func binary(n *yaml.Node, s Scalar) (Scalar, error) { + var raw string + if err := n.Decode(&raw); err != nil { + return Scalar{}, fmt.Errorf("openapi: binary literal: %w", err) + } + s.Kind, s.Bytes = ScalarBinary, []byte(raw) + return s, nil +} + +// decodes reports whether yaml.v3 decodes n into target, which is the check +// that n's text is one its tag admits. +func decodes(n *yaml.Node, target any, what string) error { + if err := n.Decode(target); err != nil { + return fmt.Errorf("openapi: %s literal %q: %w", what, n.Value, err) + } + return nil +} + +// NumericLiteral converts a numeric-tagged YAML scalar into the BigVal of the +// value YAML resolved it to, which is not what its source text means in base +// 10. YAML takes an !!int's base from its prefix and drops "_" separators, so +// 0644 is 420 and 0x1f is 31; passing that spelling to NewBigVal — a base-10 +// decimal literal — reads 0644 back as 644 instead. +// +// Only !!int is decoded, and every !!int yaml.v3 resolves fits int64 or uint64, +// so that decode is exact. A !!float keeps its text, because decoding one would +// round it through float64 and lose the arbitrary precision BigVal exists to +// preserve; only its separators are dropped. +func NumericLiteral(node *yaml.Node) (ir.BigVal, error) { + if node == nil { + return "", errors.New("openapi: nil numeric node") + } + switch node.ShortTag() { + case "!!int": + return decodeIntLiteral(node) + case "!!float": + return ir.NewBigVal(strings.ReplaceAll(node.Value, "_", "")) + default: + // A non-numeric tag — a quoted bound, say — carries no YAML-assigned + // base, so its text reads as the decimal literal it looks like. + return ir.NewBigVal(node.Value) + } +} + +// decodeIntLiteral renders an !!int node as its exact decimal literal. The +// signed range is tried first so a negative value is never read back as a large +// positive one; the unsigned range covers the int64..uint64 band above it. +// +// A magnitude past uint64 reaches an !!int node only when the source tags one +// explicitly (yaml.v3 resolves an untagged one as !!float), so nothing can +// decode it and its text is taken at face value. That is safe only for a +// spelling with no base prefix; every prefix starts with a zero, so a plain +// decimal run is the test. +func decodeIntLiteral(node *yaml.Node) (ir.BigVal, error) { + var signed int64 + if err := node.Decode(&signed); err == nil { + return ir.NewBigVal(strconv.FormatInt(signed, 10)) + } + var unsigned uint64 + if err := node.Decode(&unsigned); err == nil { + return ir.NewBigVal(strconv.FormatUint(unsigned, 10)) + } + plain := strings.ReplaceAll(node.Value, "_", "") + if !isPlainDecimal(plain) { + return "", fmt.Errorf("openapi: integer literal %q resolves to no base-10 value", node.Value) + } + return ir.NewBigVal(plain) +} + +// isPlainDecimal reports whether s is an optionally signed run of digits with no +// leading zero — the spellings whose value in base 10 is what they say, with no +// YAML base prefix to reinterpret. +func isPlainDecimal(s string) bool { + if s != "" && (s[0] == '+' || s[0] == '-') { + s = s[1:] + } + if s == "" || (s[0] == '0' && len(s) > 1) { + return false + } + return strings.TrimLeft(s, "0123456789") == "" +} diff --git a/compilers/openapi/internal/value/scalar_internal_test.go b/compilers/openapi/internal/value/scalar_internal_test.go new file mode 100644 index 000000000..471989463 --- /dev/null +++ b/compilers/openapi/internal/value/scalar_internal_test.go @@ -0,0 +1,106 @@ +package value + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" + + "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" + "github.com/dexpace/morphic/ir" +) + +// classifyRows is every way a scalar is written that classification turns on: +// tag × style × spelling. Each row is parsed from its YAML spelling, so the tag +// and style are the ones a parse gives, and names the kind and value it reads +// as, or that it is refused. annotation's renderer agreement test reads the +// same rows through both renderers. +var classifyRows = []struct { + name, src string + kind ScalarKind + num ir.BigVal + text string + refused bool +}{ + {name: "plain null", src: "null", kind: ScalarNull, text: "null"}, + {name: "tilde null", src: "~", kind: ScalarNull, text: "~"}, + {name: "empty null", src: "!!null ''", kind: ScalarNull}, + {name: "null tag over text", src: "!!null abc", refused: true}, + {name: "plain true", src: "true", kind: ScalarBool, text: "true"}, + {name: "bool tag over text", src: "!!bool maybe", refused: true}, + {name: "plain int", src: "42", kind: ScalarNumber, num: "42", text: "42"}, + {name: "octal int", src: "0644", kind: ScalarNumber, num: "420", text: "0644"}, + {name: "hex int", src: "0x1f", kind: ScalarNumber, num: "31", text: "0x1f"}, + {name: "separated int", src: "1_000", kind: ScalarNumber, num: "1000", text: "1_000"}, + {name: "plain float", src: "1.5", kind: ScalarNumber, num: "1.5", text: "1.5"}, + {name: "float past float64", src: "1.8e308", kind: ScalarNumber, num: "1.8e308", text: "1.8e308"}, + {name: "negative past float64", src: "-1.8e308", kind: ScalarNumber, num: "-1.8e308", text: "-1.8e308"}, + {name: "exponent past float64", src: "1e400", kind: ScalarNumber, num: "1e400", text: "1e400"}, + {name: "infinity", src: ".inf", refused: true}, + {name: "not a number", src: ".nan", refused: true}, + {name: "int tag over text", src: "!!int 1.5", refused: true}, + {name: "float tag over text", src: "!!float abc", refused: true}, + {name: "plain string", src: "hello", kind: ScalarString, text: "hello"}, + {name: "quoted number", src: `"1.8e308"`, kind: ScalarString, text: "1.8e308"}, + {name: "single-quoted number", src: "'42'", kind: ScalarString, text: "42"}, + {name: "string tag over a number", src: "!!str 42", kind: ScalarString, text: "42"}, + {name: "merge text as a value", src: "<<", kind: ScalarString, text: "<<"}, + {name: "plain timestamp", src: "2021-1-1", kind: ScalarTimestamp, text: "2021-1-1"}, + {name: "timestamp tag over text", src: "!!timestamp notadate", refused: true}, + {name: "binary", src: "!!binary aGVsbG8=", kind: ScalarBinary, text: "aGVsbG8="}, + {name: "binary tag over text", src: "!!binary '###'", refused: true}, + {name: "a document's own tag", src: "!acme thing", kind: ScalarTagged, text: "thing"}, +} + +// TestClassify_ReadsEachSpellingOnce holds Classify to classifyRows: the kind +// each spelling reads as, the number a number holds, and the text kept beside +// it, or that the text is refused for its tag. +func TestClassify_ReadsEachSpellingOnce(t *testing.T) { + t.Parallel() + for _, row := range classifyRows { + t.Run(row.name, func(t *testing.T) { + t.Parallel() + got, err := Classify(openapitest.YAMLNode(t, row.src)) + if row.refused { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, row.kind, got.Kind) + assert.Equal(t, row.num, got.Num) + assert.Equal(t, row.text, got.Text) + }) + } +} + +// TestClassify_ReadsTheValueOfItsKind pins the values a bool and a binary +// carry, and the refusal of a node that is no scalar. +func TestClassify_ReadsTheValueOfItsKind(t *testing.T) { + t.Parallel() + b, err := Classify(openapitest.YAMLNode(t, "false")) + require.NoError(t, err) + assert.False(t, b.Bool) + b, err = Classify(openapitest.YAMLNode(t, "true")) + require.NoError(t, err) + assert.True(t, b.Bool) + + bin, err := Classify(openapitest.YAMLNode(t, "!!binary aGVsbG8=")) + require.NoError(t, err) + assert.Equal(t, []byte("hello"), bin.Bytes) + + _, err = Classify(openapitest.YAMLNode(t, "[a]")) + require.Error(t, err) + _, err = Classify(nil) + require.Error(t, err) +} + +// TestClassify_ResolvesAnUntaggedNodeByItsText pins ShortTag: a node built by +// hand carries no tag, and is read as a parse would have tagged its text. +func TestClassify_ResolvesAnUntaggedNodeByItsText(t *testing.T) { + t.Parallel() + got, err := Classify(&yaml.Node{Kind: yaml.ScalarNode, Value: "7"}) + require.NoError(t, err) + assert.Equal(t, ScalarNumber, got.Kind) + assert.Equal(t, ir.BigVal("7"), got.Num) +} diff --git a/compilers/openapi/internal/value/value.go b/compilers/openapi/internal/value/value.go index 9577ee883..48004ee79 100644 --- a/compilers/openapi/internal/value/value.go +++ b/compilers/openapi/internal/value/value.go @@ -1,19 +1,19 @@ -// Package value lowers source scalars into ir.Value, keeping numeric literals as -// their exact source text so nothing rounds through float64. +// Package value classifies YAML scalars and lowers source values into +// ir.Value, keeping numeric literals as their exact source text so nothing +// rounds through float64. // // Classification is by wire spelling alone, independent of any surrounding // schema type — the Type-vs-Value split — which is why the package needs nothing -// from the compiler above it. +// from the compiler above it. A mapping is read as the parser reads one, through +// nodeview, so a value and a payload kept verbatim name the same members. package value import ( - "errors" "fmt" - "strconv" - "strings" yaml "gopkg.in/yaml.v3" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/ir" ) @@ -23,21 +23,28 @@ import ( const maxValueDepth = 128 // FromNode converts a YAML node into an ir.Value. Numeric literals are kept as -// their exact source text (the no-float64 escape), object member order is -// preserved, and alias nodes are followed. +// their exact source text (the no-float64 escape), object members are in the +// order the parser's model holds them, and aliases are followed. // -// The error reports a spec-level problem: a default nested past the depth cap, -// a !!binary whose payload is not base64, or a scalar with a tag this package -// does not lower (`!!python/object`). It is an error rather than a diagnostic -// because the caller decides what one means — a bad numeric bound is an error, -// an example or default that will not convert a warning — and holds the -// provenance (GitHub #169). +// The error reports a spec-level problem: a value nested past the depth cap, a +// scalar Classify refuses or no ir.Value holds (`!acme`), or a mapping that +// writes a key twice or merges past nodeview.MergeDepthLimit. The caller +// decides what one means — a bad numeric bound is an error, an example or +// default that will not convert a warning — and holds the provenance (GitHub +// #169). func FromNode(node *yaml.Node) (ir.Value, error) { - return fromNodeAt(node, 0) + c := converter{view: nodeview.New()} + return c.value(node, 0) } -// fromNodeAt is FromNode with an explicit recursion depth counter. -func fromNodeAt(node *yaml.Node, depth int) (ir.Value, error) { +// converter is one FromNode conversion: the view every mapping in the value is +// read through, shared so a merge chain is expanded once. +type converter struct { + view *nodeview.View +} + +// value converts node, depth levels below the value FromNode was handed. +func (c converter) value(node *yaml.Node, depth int) (ir.Value, error) { if depth > maxValueDepth { return ir.Value{}, fmt.Errorf("openapi: value nesting exceeds %d", maxValueDepth) } @@ -46,137 +53,50 @@ func fromNodeAt(node *yaml.Node, depth int) (ir.Value, error) { } switch node.Kind { case yaml.AliasNode: - return fromNodeAt(node.Alias, depth+1) + return c.value(node.Alias, depth+1) case yaml.ScalarNode: return scalarValue(node) case yaml.SequenceNode: - return sequenceValue(node, depth) + return c.sequence(node, depth) case yaml.MappingNode: - return mappingValue(node, depth) + return c.mapping(node, depth) default: return ir.Value{}, fmt.Errorf("openapi: unsupported yaml node kind %d", node.Kind) } } -// scalarValue converts a scalar YAML node into an ir.Value by its resolved tag. +// scalarValue converts a scalar YAML node into an ir.Value by its +// classification. A timestamp keeps its text: OpenAPI's data model is JSON, +// where a date is only expressible as a string, so `2021-1-1` stays +// `2021-1-1`. A scalar under a tag YAML gives no type has no ir.Value to hold +// it. func scalarValue(node *yaml.Node) (ir.Value, error) { - switch node.Tag { - case "!!null": + s, err := Classify(node) + if err != nil { + return ir.Value{}, err + } + switch s.Kind { + case ScalarNull: return ir.Value{Kind: ir.ValueNull}, nil - case "!!bool": - var b bool - if err := node.Decode(&b); err != nil { - return ir.Value{}, fmt.Errorf("openapi: bool literal %q: %w", node.Value, err) - } - return ir.Value{Kind: ir.ValueBool, Bool: b}, nil - case "!!str": - // A numeric literal beyond float64 range (e.g. 1.8e308) resolves to a - // plain, unquoted !!str node; capture it as a number, not a string. - // Quoted strings are never plain, so a genuine numeric string ("123") - // stays a string. This classification is by wire spelling alone, - // independent of any surrounding schema type (the Type-vs-Value split). - if node.Style == 0 { - if num, err := ir.NewBigVal(node.Value); err == nil { - return ir.Value{Kind: ir.ValueNumber, Num: num}, nil - } - } - return ir.Value{Kind: ir.ValueString, Str: node.Value}, nil - case "!!timestamp": - // YAML 1.1 resolves a plain date/time scalar (e.g. 2021-01-01) to - // !!timestamp, but OpenAPI's data model is JSON, where a date is only - // expressible as a string; the tag is a YAML parsing artifact, not a - // JSON Schema type. Keep the verbatim source spelling (2021-1-1 stays - // 2021-1-1), classified by wire spelling alone — same rule as !!str - // above, independent of any surrounding schema type. - return ir.Value{Kind: ir.ValueString, Str: node.Value}, nil - case "!!int", "!!float": - num, err := NumericLiteral(node) - if err != nil { - return ir.Value{}, fmt.Errorf("openapi: numeric literal %q: %w", node.Value, err) - } - return ir.Value{Kind: ir.ValueNumber, Num: num}, nil - case "!!binary": - // yaml.v3 base64-decodes a !!binary node into a string (it rejects a - // []byte target), so decode to string and carry the bytes from there. - var raw string - if err := node.Decode(&raw); err != nil { - return ir.Value{}, fmt.Errorf("openapi: binary literal: %w", err) - } - return ir.Value{Kind: ir.ValueBytes, Bytes: []byte(raw)}, nil + case ScalarBool: + return ir.Value{Kind: ir.ValueBool, Bool: s.Bool}, nil + case ScalarNumber: + return ir.Value{Kind: ir.ValueNumber, Num: s.Num}, nil + case ScalarString, ScalarTimestamp: + return ir.Value{Kind: ir.ValueString, Str: s.Text}, nil + case ScalarBinary: + return ir.Value{Kind: ir.ValueBytes, Bytes: s.Bytes}, nil default: + // ScalarTagged, and so any kind added later: refused, never guessed at. return ir.Value{}, fmt.Errorf("openapi: unsupported scalar tag %q", node.Tag) } } -// NumericLiteral converts a numeric-tagged YAML scalar into the BigVal of the -// value YAML resolved it to, which is not what its source text means in base -// 10. YAML takes an !!int's base from its prefix and drops "_" separators, so -// 0644 is 420 and 0x1f is 31; passing that spelling to NewBigVal — a base-10 -// decimal literal — reads 0644 back as 644 instead. -// -// Only !!int is decoded, and every !!int yaml.v3 resolves fits int64 or uint64, -// so that decode is exact. A !!float keeps its text, because decoding one would -// round it through float64 and lose the arbitrary precision BigVal exists to -// preserve; only its separators are dropped. -func NumericLiteral(node *yaml.Node) (ir.BigVal, error) { - if node == nil { - return "", errors.New("openapi: nil numeric node") - } - switch node.Tag { - case "!!int": - return decodeIntLiteral(node) - case "!!float": - return ir.NewBigVal(strings.ReplaceAll(node.Value, "_", "")) - default: - // A non-numeric tag — a quoted bound, say — carries no YAML-assigned - // base, so its text reads as the decimal literal it looks like. - return ir.NewBigVal(node.Value) - } -} - -// decodeIntLiteral renders an !!int node as its exact decimal literal. The -// signed range is tried first so a negative value is never read back as a large -// positive one; the unsigned range covers the int64..uint64 band above it. -// -// A magnitude past uint64 reaches an !!int node only when the source tags one -// explicitly (yaml.v3 resolves an untagged one as !!float), so nothing can -// decode it and its text is taken at face value. That is safe only for a -// spelling with no base prefix; every prefix starts with a zero, so a plain -// decimal run is the test. -func decodeIntLiteral(node *yaml.Node) (ir.BigVal, error) { - var signed int64 - if err := node.Decode(&signed); err == nil { - return ir.NewBigVal(strconv.FormatInt(signed, 10)) - } - var unsigned uint64 - if err := node.Decode(&unsigned); err == nil { - return ir.NewBigVal(strconv.FormatUint(unsigned, 10)) - } - plain := strings.ReplaceAll(node.Value, "_", "") - if !isPlainDecimal(plain) { - return "", fmt.Errorf("openapi: integer literal %q resolves to no base-10 value", node.Value) - } - return ir.NewBigVal(plain) -} - -// isPlainDecimal reports whether s is an optionally signed run of digits with no -// leading zero — the spellings whose value in base 10 is what they say, with no -// YAML base prefix to reinterpret. -func isPlainDecimal(s string) bool { - if s != "" && (s[0] == '+' || s[0] == '-') { - s = s[1:] - } - if s == "" || (s[0] == '0' && len(s) > 1) { - return false - } - return strings.TrimLeft(s, "0123456789") == "" -} - -// sequenceValue converts a YAML sequence into an ordered ValueList. -func sequenceValue(node *yaml.Node, depth int) (ir.Value, error) { +// sequence converts a YAML sequence into an ordered ValueList. +func (c converter) sequence(node *yaml.Node, depth int) (ir.Value, error) { list := make([]ir.Value, 0, len(node.Content)) for _, child := range node.Content { - v, err := fromNodeAt(child, depth+1) + v, err := c.value(child, depth+1) if err != nil { return ir.Value{}, err } @@ -185,17 +105,30 @@ func sequenceValue(node *yaml.Node, depth int) (ir.Value, error) { return ir.Value{Kind: ir.ValueList, List: list}, nil } -// mappingValue converts a YAML mapping into an ordered ValueObject. Mapping -// content is a flat [k0, v0, k1, v1, ...] slice, so member order is source order. -func mappingValue(node *yaml.Node, depth int) (ir.Value, error) { - fields := make([]ir.Field, 0, len(node.Content)/2) - for i := 0; i+1 < len(node.Content); i += 2 { - key := node.Content[i] - val, err := fromNodeAt(node.Content[i+1], depth+1) +// errMergedPastBound refuses a mapping whose `<<` merges run past what the +// view expands, so a member the source does write would be missing. +var errMergedPastBound = fmt.Errorf("openapi: mapping merges through `<<` deeper than %d levels", nodeview.MergeDepthLimit) + +// mapping converts a YAML mapping into a ValueObject holding the members the +// parser reads (nodeview.View.MappingPairs). A key written twice, by the +// mapping or a mapping it merges, is refused rather than read at either copy: +// an object's members are a set, yaml.v3 refuses such a mapping when it +// decodes one, and so does a payload kept verbatim (GitHub #1072). +func (c converter) mapping(node *yaml.Node, depth int) (ir.Value, error) { + if key, _, repeated := c.view.Repeat(node); repeated { + return ir.Value{}, fmt.Errorf("openapi: mapping key %q is written more than once", key) + } + pairs := c.view.MappingPairs(node) + if c.view.Exhausted() { + return ir.Value{}, errMergedPastBound + } + fields := make([]ir.Field, 0, len(pairs)) + for _, p := range pairs { + val, err := c.value(p.Val, depth+1) if err != nil { return ir.Value{}, err } - fields = append(fields, ir.Field{Name: key.Value, Value: val}) + fields = append(fields, ir.Field{Name: p.Key, Value: val}) } return ir.Value{Kind: ir.ValueObject, Object: fields}, nil } diff --git a/compilers/openapi/internal/value/value_internal_test.go b/compilers/openapi/internal/value/value_internal_test.go index 401a3f4f5..48941f516 100644 --- a/compilers/openapi/internal/value/value_internal_test.go +++ b/compilers/openapi/internal/value/value_internal_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v3" + "github.com/dexpace/morphic/compilers/openapi/internal/nodeview" "github.com/dexpace/morphic/compilers/openapi/internal/openapitest" + "github.com/dexpace/morphic/compilers/openapi/internal/ynode" "github.com/dexpace/morphic/ir" ) @@ -291,3 +293,53 @@ func TestNumericLiteral_NonNumericTagReadsAsDecimal(t *testing.T) { }) } } + +// TestValueFromNode_ReadsAMappingAsTheParserDoes pins the members of a mapping +// value to the pairs the parser reads (nodeview.View.MappingPairs): every `<<` +// merged in, first, a key written through an alias read as the key it names, +// and a key that is no scalar as the empty string (GitHub #395, #1089). +func TestValueFromNode_ReadsAMappingAsTheParserDoes(t *testing.T) { + t.Parallel() + cases := []struct { + name, src string + want []string + }{ + {"a merge comes first", "x: &x {a: 1}\nv: {b: 2, <<: *x}\n", []string{"a", "b"}}, + {"every merge merges", "x: &x {a: 1}\ny: &y {c: 3}\nv: {<<: *x, b: 2, <<: *y}\n", []string{"a", "c", "b"}}, + {"an alias key is its scalar", "k: &k rate\nv: {*k : 1, other: 2}\n", []string{"rate", "other"}}, + {"a key no scalar is empty", "v: {? [x] : 1}\n", []string{""}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := FromNode(nodeview.Child(openapitest.YAMLNode(t, tc.src), "v")) + require.NoError(t, err) + require.Equal(t, ir.ValueObject, got.Kind) + var names []string + for _, f := range got.Object { + names = append(names, f.Name) + } + assert.Equal(t, tc.want, names) + }) + } +} + +// TestValueFromNode_RefusesWhatNoObjectHolds pins the two mappings an object +// cannot hold faithfully: one writing a key twice, plainly, through an alias +// (GitHub #1072) or in a mapping it merges, and one merging past +// nodeview.MergeDepthLimit, whose deepest members would be missing. +func TestValueFromNode_RefusesWhatNoObjectHolds(t *testing.T) { + t.Parallel() + for name, src := range map[string]string{ + "a key written twice": "{a: 1, a: 2}", + "a key repeated as an alias": "{&k rate: 1, *k : 2}", + "a key a merge writes twice": "{<<: {a: 1, a: 2}}", + } { + _, err := FromNode(openapitest.YAMLNode(t, src)) + assert.ErrorContains(t, err, "written more than once", name) + } + _, err := FromNode(ynode.MergeChain(nodeview.MergeDepthLimit + 1)) + assert.ErrorIs(t, err, errMergedPastBound) + _, err = FromNode(ynode.MergeChain(nodeview.MergeDepthLimit)) + assert.NoError(t, err, "a chain at the bound is read whole") +} diff --git a/compilers/openapi/meta.go b/compilers/openapi/meta.go index 4dceff758..74c38ea56 100644 --- a/compilers/openapi/meta.go +++ b/compilers/openapi/meta.go @@ -101,7 +101,7 @@ func lowerMeta(c lowering.Ctx) (docMeta, []ir.Diagnostic) { // read in the OpenAPI dialect whatever it names. func keepJSONSchemaDialect(c lowering.Ctx, p *ir.Unmodeled) []ir.Diagnostic { at := ids.Ptr("jsonSchemaDialect") - node := nodeview.New().ChildByToken(nodeview.DocumentRoot(c.Doc.GetRootNode()), "jsonSchemaDialect") + node := nodeview.Child(nodeview.DocumentRoot(c.Doc.GetRootNode()), "jsonSchemaDialect") kept, diags := annotation.PreserveNodeInto(p, annotation.EntryKey("", "jsonSchemaDialect"), node, ir.ReasonOutOfScope, c.ProvenanceAt(at)) if !kept { @@ -119,11 +119,10 @@ func keepJSONSchemaDialect(c lowering.Ctx, p *ir.Unmodeled) []ir.Diagnostic { // into it yet (GitHub #856), so the map has no IR home. It is kept at every // version, as load reports it below 3.2 (diag.LaterVersionConstruct). func keepComponentMediaTypes(c lowering.Ctx, p *ir.Unmodeled) []ir.Diagnostic { - view := nodeview.New() - components := view.ChildByToken(nodeview.DocumentRoot(c.Doc.GetRootNode()), "components") + mediaTypes := nodeview.Child(nodeview.Child(c.Doc.GetRootNode(), "components"), "mediaTypes") at := ids.Ptr("components", "mediaTypes") kept, diags := annotation.PreserveNodeInto(p, annotation.EntryKey("components", "mediaTypes"), - view.ChildByToken(nodeview.Deref(components), "mediaTypes"), ir.ReasonNoIRHome, c.ProvenanceAt(at)) + mediaTypes, ir.ReasonNoIRHome, c.ProvenanceAt(at)) if !kept { return diags } diff --git a/compilers/openapi/meta_test.go b/compilers/openapi/meta_test.go index 809ab01eb..d0b5b884a 100644 --- a/compilers/openapi/meta_test.go +++ b/compilers/openapi/meta_test.go @@ -45,7 +45,7 @@ func TestMeta_UnserializableExtensionStillWarns(t *testing.T) { spec := `openapi: 3.1.0 info: {title: T, version: "1"} paths: {} -x-bad: {1: intkey} +x-bad: {n: .nan} ` doc, diags := parseFull(t, spec) assert.Positive(t, openapitest.CountDiagsAt(diags, diag.DegradedConstruct, ir.SeverityWarning), diff --git a/compilers/openapi/openapi.go b/compilers/openapi/openapi.go index 6193bb762..d3703f284 100644 --- a/compilers/openapi/openapi.go +++ b/compilers/openapi/openapi.go @@ -156,18 +156,16 @@ func (*Compiler) SourceTable(sources []compilers.Source, opts compilers.Options) // references types never interned or settled, but the diagnostics gathered // before the stop are returned. func run(ctx context.Context, c lowering.Ctx, ts *compile.Types) (*ir.Document, []ir.Diagnostic, error) { - // out and the anchor memo are this function's, not a struct's: a document - // being built and a memo (micro-compiler-design §4.1). Nothing below - // allocates them. + // out is this function's, not a struct's: a document being built + // (micro-compiler-design §4.1). Nothing below allocates it. out := &ir.Document{} - var anchors schema.AnchorIndex // acc is what makes the identity dedup still hold. Every lowering returns its // diagnostics now, so a shared declaration reported from N use sites returns N // copies; compile.Diags.Append is what collapses them, and it has to see the // whole stream to do it. var acc compile.Diags - acc.AppendAll(schema.LowerComponentSchemas(ctx, c, ts, &anchors)) + acc.AppendAll(schema.LowerComponentSchemas(ctx, c, ts)) // The checks in this function sit after the phases that honour ctx. Each // stops between items and returns what it had, so a non-nil ctx.Err() here // is the only way to tell a phase that finished from one that was cut short — @@ -183,7 +181,7 @@ func run(ctx context.Context, c lowering.Ctx, ts *compile.Types) (*ir.Document, // one, so no lowering above the schemes can read them as empty. svcCtx := c.WithAuth(schemes) - svc, tagDefs, svcDiags := operation.LowerService(ctx, svcCtx, ts, &anchors) + svc, tagDefs, svcDiags := operation.LowerService(ctx, svcCtx, ts) out.Services = []ir.Service{svc} out.TagDefs = tagDefs acc.AppendAll(svcDiags) diff --git a/compilers/openapi/resolve_internal_test.go b/compilers/openapi/resolve_internal_test.go index e747d873a..76c84e06c 100644 --- a/compilers/openapi/resolve_internal_test.go +++ b/compilers/openapi/resolve_internal_test.go @@ -90,7 +90,7 @@ components: $ref: '#/components/schemas/Target' minimum: 5 `) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) sc, ok := typeByName(l.document(t), "S").(*ir.Scalar) require.True(t, ok, "S aliases Target and must own a Scalar node") @@ -129,7 +129,7 @@ components: $ref: '#/components/schemas/Target' example: at-reference `) - l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) + l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) target := typeByName(l.document(t), "Target") require.NotNil(t, target) diff --git a/docs/architecture.md b/docs/architecture.md index 548bffe8d..9aaa82ad2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -96,9 +96,9 @@ Internal phases every compiler follows (each format implements them its own way) the IR's nullable bit. The IR never records which dialect a fact came from except in provenance. 3. **Hoist & identify** — every anonymous inline type is hoisted into the type registry exactly - once, keyed by its source pointer, with a naming *hint* (not a name) computed from context. - This is a single pass; no other code derives inline names (oagen's duplicated-naming failure - mode is designed out). + once, keyed by its source pointer, with a naming *hint* (not a name) that is a function of the + documents and that pointer alone. One function derives every inline name, whichever lowering + reaches the position first (oagen's duplicated-naming failure mode is designed out). 4. **Resolve & lower** — build IR nodes: type graph, services, operations, bindings, auth, channels. Declared semantics lower directly; inferred semantics (heuristic pagination detection, envelope unwrapping) run only if the corresponding policy is enabled and mark @@ -397,6 +397,7 @@ morphic/ │ │ ├── load/ # parse, validate, resolve │ │ ├── resolve/ # what a $ref names; reference-or-inline entries │ │ ├── merge/ # allOf property reconciliation +│ │ ├── naming/ # each node's name, from its coordinate alone │ │ ├── lowering/ # the immutable context every lowering reads │ │ ├── schema/ # the schema walk, composition, preservation │ │ ├── auth/ # security schemes and requirements diff --git a/docs/compilers/openapi.md b/docs/compilers/openapi.md index b0b8f5a0b..51db74ec9 100644 --- a/docs/compilers/openapi.md +++ b/docs/compilers/openapi.md @@ -161,17 +161,21 @@ absent, a value of the wrong type, a value outside the set the specification all exclude each other written together, a path template and its path parameters disagreeing, a list or map holding fewer entries than the specification requires or repeating one it requires to be unique, as JSON Schema requires of `allOf` and of `required`, or a value a grammar the specification cites -refuses, such as a token or an anchor's name. It is a warning where the document breaks a rule of -its version and still says what it means: a key outside the pattern the specification gives, a key a -mapping writes twice, a number or a boolean where a string belongs, a field written where the -specification gives it no effect (`validation-no-effect`), or a `null` written for an optional -field, which is read as the field left out, by the rules that bind it to other fields as well. A -Schema Object's keyword whose value is a schema is such a field, since the schema lowering reads its -`null` as left out too ([`null-schema`](#null-schema)). A `null` written as a schema entry, which is -another object's `schema`, a component, or an entry of a map or a list of schemas, is an error, -since that lowering reads it as the empty schema, a value, and a list counts it among its entries; -so is one for a Schema Object's other keywords. One written for a field whose requirement another -rule reports is left to that rule. +refuses, such as a token or an anchor's name. Every mapping, values and extensions included, is held +to the Format section's rule that a key is a string, so a key that is no scalar, which the parser +reads as the empty string ([`non-scalar-key`](../parser-library.md#non-scalar-key)), is an error, +and so is a `<<` merge key naming neither a mapping nor a list of mappings, from which the parser +merges nothing ([`merge-keys`](../parser-library.md#merge-keys)). It is a warning where the document +breaks a rule of its version and still says what it means: a key outside the pattern the +specification gives, a key a mapping writes twice, a number or a boolean where a string belongs, a +field written where the specification gives it no effect (`validation-no-effect`), or a `null` +written for an optional field, which is read as the field left out, by the rules that bind it to +other fields as well. A Schema Object's keyword whose value is a schema is such a field, since the +schema lowering reads its `null` as left out too ([`null-schema`](#null-schema)). A `null` written +as a schema entry, which is another object's `schema`, a component, or an entry of a map or a list +of schemas, is an error, since that lowering reads it as the empty schema, a value, and a list +counts it among its entries; so is one for a Schema Object's other keywords. One written for a field +whose requirement another rule reports is left to that rule. The source is read as it writes it, an applied overlay's patch included, each object as the kind its position gives it. An object a `$ref` reaches that the source's own reading did not take as @@ -204,7 +208,7 @@ gives it no effect is [`allow-reserved-locations`](#allow-reserved-locations)' a `oas.TestCheck_AFindingIsMadeOnceWhereItIsPlaced`, `oas.TestCheck_RulesAnotherPackageOwnsAreNotApplied`, `oas.TestCheck_AnAnchorIsANameJSONSchemaAdmits`, - `oas.TestCheck_KeywordsHoldWhatJSONSchemaRequires`, + `oas.TestCheck_KeywordsHoldWhatJSONSchemaRequires`, `oas.TestCheckFormat_AKeyIsAString`, `oas.TestTable_EveryRowCitesTheSpecification`, `load.TestJudgeSource_PlacesEachFindingAtItsNode`, `load.TestJudgeSource_RefusesTheAnchorsNoReferenceCanName`, `load.TestLoad_WhatAnAliasMountsIsReportedOnce`, @@ -519,25 +523,31 @@ A construct a rule keeps verbatim becomes an `Unmodeled` entry (§12) on the nod owns its position: keyed `openapi:` for a keyword or field, or by the scoped key the rule names where several objects reach one carrier ([`vendor-extensions`](#vendor-extensions)), under the reason the rule gives, and with a `Provenance` locating the construct itself rather than the node -that carries it. The entry holds the construct as JSON read off its YAML node: aliases followed and -merge keys merged, a number in its digits with its YAML base resolved, a YAML timestamp in the -spelling written, a `!!binary` as its source text, and a scalar under a tag of the document's own as -its text. A node no JSON can hold — a mapping key that is not a string, or repeats, `.nan` or -`.inf`, a scalar whose text its tag does not admit, invalid UTF-8, an alias that cycles or expands -past the converter's node budget — reaches the IR in no form. It is reported as an -`openapi/unpreservable-construct` error at the construct, and the rule that would have kept it -announces nothing as kept, so no diagnostic claims what the Document does not hold. A vendor -extension that converts to no JSON is [`vendor-extensions`](#vendor-extensions)' instead. - +that carries it. The entry holds the construct as JSON read off its YAML node as the parser reads +it: aliases followed, `<<` merge keys merged in the parser's order +([`merge-keys`](../parser-library.md#merge-keys)), a mapping key as the text it spells, since the +specification reads every key as a string, and one that is no scalar as the empty string +([`validation`](#validation) reports it), a number in its digits with its YAML base resolved, past +float64's range included, a YAML timestamp in the spelling written, a `!!binary` as its source text, +and a scalar under a tag of the document's own as its text. A field a reader finds is found the same +way, so one supplied through a merge key or an alias is kept as a written one is. A node no JSON can +hold — a key a mapping or a mapping merged into it writes twice, `.nan` or `.inf`, a scalar whose +text its tag does not admit, invalid UTF-8, an alias that cycles or expands past the converter's +node budget, a merge chain deeper than the reader expands — reaches the IR in no form. It is +reported as an `openapi/unpreservable-construct` error at the construct, and the rule that would +have kept it announces nothing as kept, so no diagnostic claims what the Document does not hold. A +vendor extension that converts to no JSON is [`vendor-extensions`](#vendor-extensions)' instead. + +- Cases: `yaml-merge-keys` - Codes: `openapi/unpreservable-construct` - Tests: `schema.TestAllOf_UnpreservableBranchIsNotAnnounced`, `schema.TestCoDeclaredFamily_UnpreservableIsNotAnnounced`, - `annotation.TestValidationOnlyAt_AnUnconvertibleKeywordIsReportedNotKept` -- Status: partly built: a number past float64's range is kept as a JSON string (#783), and a field - supplied through a merge key or an alias is found by no reader that keeps it, reaching the IR in - no form (#384). Open: whether a scalar its own tag does not admit is refused or kept as its text - (#245), and whether an extension mapping with non-string keys is kept with its keys stringified - (#617). + `annotation.TestValidationOnlyAt_AnUnconvertibleKeywordIsReportedNotKept`, + `annotation.TestRawFromNode_PreservesMergeAndOrderingSemantics`, + `annotation.TestRawFromNode_NamesTheRepeatedKey`, `annotation.TestScalarRenderers_AgreeOnKind`, + `annotation.TestRawPropertyNode_ReadsAKeywordAsTheParserDoes` +- Status: partly built: whether a scalar its own tag does not admit is refused or kept as its text + is open (#245). ### `unmodeled-dialect-fields` @@ -585,7 +595,8 @@ pointer, with no diagnostic. The sections are `responses`, `parameters`, `exampl `components/securitySchemes` are lowered unconditionally. 3.2's `components/mediaTypes`, into which no reference is followed, is kept whole as `openapi:components/mediaTypes`, with an info `openapi/degraded-construct` at the section. A section is read as the parser reads it, through `<<` -merge keys, so a merged-in entry is kept under its own name. +merge keys, so a merged-in entry is kept under its own name, and an entry whose key is no scalar +under the empty name the parser reads it by ([`validation`](#validation) reports the key). *Referenced* means a lowering followed a reference to the entry, not that some `$ref` string names it. The resolution every non-schema reference passes through records each position in the source its @@ -610,8 +621,7 @@ that decision lives (§12), which is what keeps a promotion pass from mistaking `openapi.TestUnreferencedComponents_ReadAsTheParserReadsThem`, `openapi.TestUnreferencedComponents_SelfReferenceByFileNameReaches` - Status: partly built: no reference into `components/mediaTypes` is followed, so the section is - kept whole however much of it a document uses (#856), and a section entry whose key is no scalar - is dropped without a diagnostic (#947). + kept whole however much of it a document uses (#856). ## 3. Names and values @@ -624,54 +634,66 @@ key, an enum member's value as text, a 3.2 server `name`, and under `grouping=pa path segment. A name with no letter or digit (`***`, `>=`) keeps its `Source` and has no `Canonical`; an emitter names it by its characters (emitter-design §4.12). A name spelled as the empty string is minted `Hint: "empty"` and no `Source` rather than passed through (an `operationId` -of `""` is read as absent instead, -[`operation-ids`](#operation-ids)), and a property's wire name stays `""`. A component schema keyed -`""` earns no named `TypeID`: it hoists anonymously at its own pointer -(`t/anon/components/schemas/`) with that hint. An entity the document gives no name — an inline -schema, a union variant, a response ([`response-conditions`](#response-conditions)), a server with -no `name` (its URL's words, else `server`), the synthesized `default` and `webhooks` groups — +of `""` is read as absent instead, [`operation-ids`](#operation-ids)), and a property's wire name +stays `""`. A component schema keyed `""` earns no named `TypeID`: it hoists anonymously at its own +pointer (`t/anon/components/schemas/`) with that hint. An entity the document gives no name — an +inline schema, a union variant, a response ([`response-conditions`](#response-conditions)), a server +with no `name` (its URL's words, else `server`), the synthesized `default` and `webhooks` groups — carries a `Hint` alone, in the same word grammar and composed from its position, each half neutralized first so an empty parent gives `empty_item`, never `_item`. A request body or header that references reach is named for its declaration, never for whichever reference lowered first: a component's name, a header's own key, the `operationId` of the operation declaring the body, in this document or another and whether or not that operation is lowered, or `request` for a body no -operation with an `operationId` declares. `Aliases` is never set. +operation with an `operationId` declares. A schema is named for the nearest object holding it, so +one a parameter, request body or response holds is named for that object even where the object is +kept under an extension and read through a `$ref`, and a document a `$ref` reads whole as a schema +is named for the document (`./Cat.yaml` names its root `cat_yaml`). Every such name is a function of +the documents and the position alone, read where the position is written, so no lowering order +changes it: a union variant, or a composition branch holding a `$ref`, is named as the position the +`$ref` lands on, in whichever document, and keeps its own name where that names nothing. `Aliases` +is never set. - Cases: `neutral-naming`, `empty-names` - Tests: `ids.TestComponentSchemaNamedEmpty_SeparatesAnEmptyNameFromNoEntry`, `schema.TestEnumMemberIDs_NamesThatCanonicalizeToNothingStillGetIDs`, `schema.TestInlinePosition_HintIsTheSameInBothOrders`, `operation.TestContent_ASharedBodyOrHeaderIsNamedForItsDeclaration`, - `schema.TestComponentBody_SubtreeIsNamedByItsDeclaration` -- Status: partly built: union variants and inline schemas ignore their `title` as a hint (#662); and - a union variant naming a position in a path's body takes the pointer's name (#729). + `schema.TestComponentBody_SubtreeIsNamedByItsDeclaration`, + `schema.TestUnionVariant_UnderPathsIsNamedAsTheNodeItHolds`, + `schema.TestUnionVariant_AWholeDocumentIsNamedForIt`, `naming.TestAt_NamesASchemaAfterItsOwner`, + `naming.TestAt_NamesABranchAfterItsTarget`, `naming.TestAt_NamesAStructureAReferenceReadsInsideRawYAML` +- Status: partly built: union variants and inline schemas ignore their `title` as a hint (#662). ### `values` A default, `const`, enum member or example lowers to an `ir.Value` read off the YAML node by its resolved tag alone, never by the type the schema declares (§6): `null`, `bool`, `string`, `list` and -`object` (members in source order, keyed by their text, an alias followed), a `!!binary` to `bytes`, -decoded, and a YAML timestamp to `string` in the spelling written (`2021-1-1` stays `2021-1-1`, -never RFC 3339). A number is the literal's exact decimal text in `Num`, never a `float64`: a -magnitude past float64 or int64 keeps every digit, `.5`, `+5` and `09` take their JSON spellings -(`0.5`, `5`, `9`), `_` separators drop, a YAML integer base resolves (`0644` is `420`, `0x1f` is -`31`, `0b1010` is `10`), and digits, exponent and trailing zeros otherwise stand as written (`10.0`, -`1.10e+3`). A plain scalar that YAML resolves to a string only because it is past float64's range is -still a number when it is a finite decimal literal; a quoted one stays a string. A value no `Value` -holds — `.inf` or `.nan`, a `!!binary` that is not base64, a tag the compiler does not lower -(`!acme`), nesting past the reader's depth bound — is not coerced: a default or example is dropped -with an `openapi/degraded-construct` warning at its position, and a `const` or an enum member is -[`enum-const`](#enum-const)'s. The number reader is the numeric bounds' as well -([`constraints`](#constraints)). A value kept verbatim instead is -[`verbatim-preservation`](#verbatim-preservation)'s. - -- Cases: `defaults`, `yaml-timestamp-scalars`, `numeric-precision` +`object` (members as the parser reads them, `<<` merge keys merged in its order +([`merge-keys`](../parser-library.md#merge-keys)), each keyed by its text and a key that is no +scalar by the empty string, an alias followed), a `!!binary` to `bytes`, decoded, and a YAML +timestamp to `string` in the spelling written (`2021-1-1` stays `2021-1-1`, never RFC 3339). A +number is the literal's exact decimal text in `Num`, never a `float64`: a magnitude past float64 or +int64 keeps every digit, `.5`, `+5` and `09` take their JSON spellings (`0.5`, `5`, `9`), `_` +separators drop, a YAML integer base resolves (`0644` is `420`, `0x1f` is `31`, `0b1010` is `10`), +and digits, exponent and trailing zeros otherwise stand as written (`10.0`, `1.10e+3`). A plain +scalar that YAML resolves to a string only because it is past float64's range is still a number when +it is a finite decimal literal; a quoted one stays a string. A value no `Value` holds — `.inf` or +`.nan`, a `!!binary` that is not base64, a scalar whose text its tag does not admit (`!!null abc`, +`!!timestamp soon`), a tag the compiler does not lower (`!acme`), a key a mapping or a mapping +merged into it writes twice, a merge chain deeper than the reader expands, nesting past the reader's +depth bound — is not coerced: a default or example is dropped with an `openapi/degraded-construct` +warning at its position, and a `const` or an enum member is [`enum-const`](#enum-const)'s. The +number reader is the numeric bounds' as well ([`constraints`](#constraints)). A value kept verbatim +instead is [`verbatim-preservation`](#verbatim-preservation)'s. + +- Cases: `defaults`, `yaml-timestamp-scalars`, `numeric-precision`, `yaml-merge-keys` - Codes: `openapi/degraded-construct` - Tests: `value.TestNumericLiteral_YAMLBases`, `value.TestValueFromNode_Timestamp`, + `value.TestValueFromNode_ReadsAMappingAsTheParserDoes`, + `value.TestValueFromNode_RefusesWhatNoObjectHolds`, `value.TestClassify_ReadsEachSpellingOnce`, `annotation.TestSchemaExamplesAt_UnconvertibleValueIsReportedNotDropped` -- Status: partly built: a `<<` merge key in a mapping value stays a member named `<<`, where a - verbatim payload merges it (#395), and an unconvertible default is reported at the schema of the - property or parameter that reads it rather than at its own position (#1101). +- Status: partly built: an unconvertible default is reported at the schema of the property or + parameter that reads it rather than at its own position (#1101). ## 4. Schemas: identity, references and annotations @@ -1848,8 +1870,8 @@ expansion. - Cases: `dynamic-ref` - Codes: `openapi/dynamic-ref-expanded`, `openapi/degraded-construct` - Tests: `schema.TestDynamicRef_IrreducibleIsKeptAndSaysWhy`, - `schema.TestDynamicAnchors_CountsWhatAnAliasBringsIn`, - `schema.TestDynamicAnchorIndex_ReportsATruncatedWalk` + `anchors.TestBuild_CountsWhatAnAliasBringsIn`, + `lowering.TestDynamicAnchorSites_ReportsATruncatedWalkOnce` - Status: partly built: a merge chain past the view's merge bound hides an `$id`, so a `$dynamicRef` can expand across a resource boundary (#401), and the truncation warning names the walk's own bounds when the merge bound stopped it (#454). A `$dynamicRef` written in another document is not @@ -2622,8 +2644,9 @@ rest reach the IR in no form, reported as one `openapi/unknown-key-budget` warni Two other outcomes also keep nothing and warn: `openapi/unknown-key-entry-taken`, where `openapi:` is already held by a construct written elsewhere on a carrier two objects share, a parameter's own key and its schema's keyword, whichever lowered first surviving; and -`openapi/unknown-key-unreachable`, where the key is merged in through `<<` and the raw mapping holds -no pair for it. +`openapi/unknown-key-unreachable`, where the key is merged in through a `<<` chain deeper than the +compiler's reader expands, so it reads no pair for it. A key merged in within that depth is read, +kept and reported as a written one is. - Cases: `path-item-anchored-key` - Codes: `openapi/unknown-object-key`, `openapi/unknown-schema-keyword`, @@ -2633,9 +2656,8 @@ no pair for it. `annotation.TestUnknownKeysIn_BudgetBoundsWhatOneObjectContributes`, `annotation.TestUndefinedKeywordsIn_KeepsWhatTheCallerNames` - Status: partly built: a parameter's key and its schema's keyword of one spelling share an entry - (#396), a key merged in through `<<` is reported and kept nowhere (#395), and a schema keyword at a - position that lowers to a shared node and carries no property or parameter is kept nowhere and - reported by nothing (#1053). + (#396), and a schema keyword at a position that lowers to a shared node and carries no property or + parameter is kept nowhere and reported by nothing (#1053). ### `extension-promotion` diff --git a/docs/ir-design.md b/docs/ir-design.md index 3cae1d0a1..97f17cede 100644 --- a/docs/ir-design.md +++ b/docs/ir-design.md @@ -857,10 +857,9 @@ when it carries no `Property` or `Parameter` of its own to hold them — `items` component — so nothing about composition is special here; a `$ref` entry writing nothing beside itself composes straight to the target and hoists no node. The alias's name hint is the hint the node the `$ref` resolves to carries — a component's name, or, for a position inside a schema, the -hint its own declaration gives it — which is also what an outside `$ref` naming the branch pointer -derives, since either may be the lowering that interns the node. A position under `/paths` whose -hint comes from its operation or response is the exception so far, since the pointer does not -spell that hint (#729). +hint its position gives it, an operation's or a response's included. A hint is a function of the +documents and the position alone, so an outside `$ref` naming the branch pointer gives the same one, +whichever lowering interns the node. `Base` and `Mixins` name whatever the source composed with, which in JSON Schema is any schema: `allOf: [{$ref: SomeScalar}]` is legal and lowers to a `Base` pointing at a `Scalar`, and the same diff --git a/docs/micro-compiler-design.md b/docs/micro-compiler-design.md index e4cdfd5cd..f4e2b3b94 100644 --- a/docs/micro-compiler-design.md +++ b/docs/micro-compiler-design.md @@ -279,13 +279,18 @@ overwhelming majority that never write `$dynamicRef`. And the walk is not free: ```bash # what deriving it at entry would cost a document that never asks -go test ./compilers/openapi -run XXX -bench BenchmarkAnchorWalk -benchtime=2000x +go test ./compilers/openapi/internal/anchors -run XXX -bench BenchmarkBuild -benchtime=2000x ``` It measured about 3.7µs and 75 allocations against a compile's 270µs and 2,767 — roughly 1.4% of the time and 2.7% of the allocations, spent on a keyword almost nothing uses. So it is a memo, built on the first `$dynamicRef` and not before. +*Later:* the memo moved into `Ctx`, which builds it on the first ask and shares it +with every copy, as it does the answers `TypePosition` reads (`Ctx.DynamicAnchorSites`, +GitHub #982). It is still built on the first `$dynamicRef` and not before; what +changed is that no lowering carries it as a parameter. + That makes it the one counterexample to §4's claim that interning is "irreducibly shared and stateful; nothing else is". It is a weaker exception than that phrasing suggests — a memo caches a pure function of the document, where the @@ -536,6 +541,11 @@ builder of what it owns. It replaced the provisional naming that rebuilt a node before its declaration (#372, #529): the rebuild replaced the node, but not what the reference had reported (#750) nor where it met the depth bound (#748). +*Later:* names did move to the table (#981). The drift this section feared needs two readings, and +the declaration walk now gives no name at all: every lowering asks `lowering.NameAt`, which reads +the grammar's kinds (`bind.Table.Kinds`) and the document, so the table's is the only reading. The +home a declaration lowers in is still the declaration walk's. + ## 7. Enforcement 1. **`internal/archtest`'s prefix matching is a live hole and is fixed first.** `hasAllowedPrefix` @@ -736,7 +746,7 @@ sorts, which `maps.Keys` + `slices.Sorted` already cover. answers what the cycle and amplification refusals each used to walk it to ask, over the single decode the loader now performs. What is still out of scope is the other half — a pointer → node map that would make reference resolution a lookup, a per-node key index that would make - `annotation.RawChildNode` one, and the `--explain` substrate both would give. Each of those is + `nodeview.Child` one, and the `--explain` substrate both would give. Each of those is read during lowering rather than before it, so handing them down means widening the lowering context and the signatures beneath it: its own change. - **Rebasing the GraphQL and Protobuf drafts.** They are evidence here, not work items. diff --git a/docs/parser-library.md b/docs/parser-library.md index 1d8583d79..9f4c8aede 100644 --- a/docs/parser-library.md +++ b/docs/parser-library.md @@ -425,10 +425,55 @@ components: - Verified at: v1.25.5 - Pinned by: `TestLibraryBehaviour/duplicate-key` - Code: - - `nodeview.dedupeLastWins` reads a mapping's repeated key as the model does. - - `annotation.RawChildNode` does too. + - `nodeview.flatten` reads a mapping's repeated key as the model does. + - `nodeview.Child` reads one at the copy the model does, a merge source's at its first. - `load.mappingKeys` records each key's first pair, as a raw read finds it. +### `merge-keys` + +The decoder reads a mapping through every `<<` merge key it writes (`yml.ResolveMergeKeys`). The +pairs merged in come first, in the order their sources are written, an earlier source winning a key +two share and none overriding a key the mapping writes itself; the mapping's own pairs follow. A +source that writes a key twice gives its first copy, where the mapping itself gives its last +([`duplicate-key`](#duplicate-key)). A `<<` naming anything but a mapping or a list of mappings +merges nothing, and nothing reports it. + +```yaml +x-base: &base {k: {type: string}, z: {type: string}} +components: + schemas: + S: + properties: {a: {type: integer}, <<: *base, k: {type: boolean}} # z, a, then k + T: + properties: {<<: 5, q: {type: string}} # q alone +``` + +- Verified at: v1.25.5 +- Pinned by: `TestLibraryBehaviour/merge-keys` +- Code: + - `nodeview.flatten` orders and settles a mapping's pairs as the decoder does. + - `nodeview.View.mergeSource` reads each source's pairs as a merge reads them. + - `nodeview.Child` looks one key up through the merges by the same order and copies. + - `oas.CheckFormat` reports a `<<` naming no mapping. + +### `non-scalar-key` + +A mapping key that is no scalar, a mapping or a list written as a key, is read as the empty string, +and nothing reports it. The Format section of the OpenAPI Specification limits a key to a string. + +```yaml +components: + responses: + ? {k: v} + : {description: x} # read as the response keyed "" +``` + +- Verified at: v1.25.5 +- Pinned by: `TestLibraryBehaviour/non-scalar-key` +- Code: + - `nodeview.KeyOf` reads such a key as the empty string. + - `oas.CheckFormat` reports it. + ### `unmodeled-keywords` The schema model has no field for `$dynamicRef`, `$dynamicAnchor`, `$comment`, `$vocabulary` or @@ -444,7 +489,7 @@ components: - Verified at: v1.25.5 - Pinned by: `TestLibraryBehaviour/unmodeled-keywords` - Code: - - `schema.dynamicAnchors` reads every `$dynamicAnchor` from the raw tree. + - `anchors.Build` reads every `$dynamicAnchor` from the raw tree. - `annotation.DecidedKeywords` names `$comment`, `$dynamicAnchor` and `$dynamicRef`, which the unknown-key census leaves to other readers. - `annotation.UnknownKeywordsIn` reads every other unknown keyword's value from the raw node. diff --git a/internal/archtest/arch_test.go b/internal/archtest/arch_test.go index 301a3ea2c..5fb31e392 100644 --- a/internal/archtest/arch_test.go +++ b/internal/archtest/arch_test.go @@ -56,11 +56,14 @@ var rules = map[string][]string{ // the compiler would be able to derive one from something other than a // source coordinate. "compilers/openapi/internal/ids": {module + "/ir", module + "/compilers/compile"}, - // Scalar and BigVal lowering. It reaches ir for the value types and yaml for - // the node it reads, and nothing else: a value is decided by its own wire - // spelling, so a package that could reach the compiler would be able to let a - // surrounding schema type change what a literal means. - "compilers/openapi/internal/value": {module + "/ir", "gopkg.in/yaml.v3"}, + // The one classification of a YAML scalar, and value lowering built on it. + // It reaches ir for the value types, yaml for the node it reads and nodeview + // for a mapping's members as the parser reads them, and nothing else: a + // value is decided by its own wire spelling, so a package that could reach + // the compiler would be able to let a surrounding schema type change what a + // literal means. + "compilers/openapi/internal/value": {module + "/ir", + module + "/compilers/openapi/internal/nodeview", "gopkg.in/yaml.v3"}, // The yaml.v3 nodes tests build trees from: the constructors for the node // kinds a parse produces, the `<<` merge key among them. It reaches yaml and // nothing else, so any package's internal tests can import it, nodeview's @@ -79,11 +82,11 @@ var rules = map[string][]string{ // resolver's own reading and nothing of the compiler, so the scans, the loader // and the lowering can all reach it. "compilers/openapi/internal/fragment": {"github.com/speakeasy-api/openapi/references"}, - // A view over the raw source: mappings read the way the resolver reads them, - // through aliases and `<<` merge keys. It reaches the library's yml package - // for the merge-key predicate the unmarshaller itself uses, and is below the - // scan, the schema bindings and the schema lowering, which read alike through - // it. + // The one reading of the raw source: mappings read the way the parser reads + // them, through aliases and `<<` merge keys, and the one bounded walk of a + // tree. It reaches the library's yml package for the merge-key predicate the + // unmarshaller itself uses, and is below every package that reads raw YAML, + // which reads through it rather than by a reading of its own. "compilers/openapi/internal/nodeview": {"github.com/speakeasy-api/openapi/yml", "gopkg.in/yaml.v3"}, // How the library's walk reads a pointer through the parsed model, a token at // a time, and where it leaves the model for raw YAML. It reaches the library's @@ -106,6 +109,21 @@ var rules = map[string][]string{ module + "/compilers/openapi/internal/nodeview", "github.com/speakeasy-api/openapi/jsonschema/oas3", "github.com/speakeasy-api/openapi/marshaller", "github.com/speakeasy-api/openapi/walk", "gopkg.in/yaml.v3"}, + // The name of every node the lowering builds, as a function of the documents + // and the node's coordinate (GitHub #981). It reads the grammar through a + // bind table, the raw tree through nodeview and a $ref's fragment through + // fragment, and composes names in compile's grammar. It reaches nothing that + // lowers, so no lowering can make a name depend on the order it runs in. + "compilers/openapi/internal/naming": {module + "/compilers/compile", + module + "/compilers/openapi/internal/bind", + module + "/compilers/openapi/internal/fragment", + module + "/compilers/openapi/internal/ids", + module + "/compilers/openapi/internal/nodeview", "gopkg.in/yaml.v3"}, + // The $dynamicAnchor declarations of a source tree, which a $dynamicRef + // expands against. It reads the tree through nodeview, as the parser reads + // it, spells pointers through ids, and reaches nothing that lowers. + "compilers/openapi/internal/anchors": {module + "/compilers/openapi/internal/ids", + module + "/compilers/openapi/internal/nodeview", "gopkg.in/yaml.v3"}, // One walk over the decoded source tree, answering what the pre-lowering // refusals would otherwise each walk it to ask. It reaches nodeview for the // document root and nothing else: an index of what the source says is not @@ -122,14 +140,15 @@ var rules = map[string][]string{ // What a schema or a carrier says about itself rather than about its shape, // plus how a construct is kept verbatim under Unmodeled, the validation-only // keywords among them. It reads the parsed model and the raw nodes behind - // it, and holds no opinion about lowering — so it sits below the schema - // lowering that asks it questions and above nothing that asks it any. It - // reaches the library's yml package for the tags its decoder accepts as a - // string, so a field read off a raw node decodes as the parser decodes the - // fields it models. + // it, those through nodeview and their scalars through value, and holds no + // opinion about lowering — so it sits below the schema lowering that asks it + // questions and above nothing that asks it any. It reaches the library's yml + // package for the tags its decoder accepts as a string, so a field read off + // a raw node decodes as the parser decodes the fields it models. "compilers/openapi/internal/annotation": {module + "/ir", module + "/compilers/openapi/internal/diag", module + "/compilers/openapi/internal/ids", + module + "/compilers/openapi/internal/nodeview", module + "/compilers/openapi/internal/value", "github.com/speakeasy-api/openapi/extensions", "github.com/speakeasy-api/openapi/jsonschema/oas3", @@ -210,25 +229,31 @@ var rules = map[string][]string{ // and so must reach neither, which is why it sits here rather than with either // one. It reaches load for the version grammar — the dialect question is asked // of the document, and the grammar that answers it is the loader's — and for - // the schema bindings the load phase built, which its scope reads. + // the schema bindings the load phase built, which its scope reads. It reaches + // naming for what each position is called and anchors for the + // $dynamicAnchor index it builds on first ask. "compilers/openapi/internal/lowering": {module + "/ir", + module + "/compilers/openapi/internal/anchors", module + "/compilers/openapi/internal/bind", module + "/compilers/openapi/internal/bind/typed", module + "/compilers/openapi/internal/diag", module + "/compilers/openapi/internal/ids", module + "/compilers/openapi/internal/load", + module + "/compilers/openapi/internal/naming", module + "/compilers/openapi/internal/overlay", module + "/compilers/openapi/internal/resolve", "github.com/speakeasy-api/openapi/openapi"}, // The security schemes a document declares and the requirements that name // them. It is its own package because both sides of the compiler reach it and // neither may reach the other: the document walk lowers the schemes, and the - // service and operation walks lower the requirements against them. + // service and operation walks lower the requirements against them. It reads + // a field the model names no member for off the raw node, through nodeview. "compilers/openapi/internal/auth": {module + "/ir", module + "/compilers/compile", module + "/compilers/openapi/internal/annotation", module + "/compilers/openapi/internal/diag", module + "/compilers/openapi/internal/ids", module + "/compilers/openapi/internal/lowering", + module + "/compilers/openapi/internal/nodeview", module + "/compilers/openapi/internal/resolve", "github.com/speakeasy-api/openapi/openapi"}, // The schema walk, and everything that recurses with it: composition, @@ -252,9 +277,10 @@ var rules = map[string][]string{ "github.com/speakeasy-api/openapi/values", "gopkg.in/yaml.v3"}, // The operation walk: path items, webhooks and callbacks, the parameters // merged onto them, and the content of every body, response and header. It - // reaches the schema walk for the positions it hoists and auth for the - // requirements it reads, and nothing above itself — the compiler assembles - // the document from what this returns rather than the other way round. + // reaches the schema walk for the positions it hoists, auth for the + // requirements it reads and nodeview for the raw nodes behind the model, and + // nothing above itself — the compiler assembles the document from what this + // returns rather than the other way round. "compilers/openapi/internal/operation": {module + "/ir", module + "/compilers/compile", module + "/compilers/openapi/internal/annotation", module + "/compilers/openapi/internal/auth", @@ -262,6 +288,7 @@ var rules = map[string][]string{ module + "/compilers/openapi/internal/diag", module + "/compilers/openapi/internal/ids", module + "/compilers/openapi/internal/lowering", + module + "/compilers/openapi/internal/nodeview", module + "/compilers/openapi/internal/resolve", module + "/compilers/openapi/internal/schema", "github.com/speakeasy-api/openapi" + subtreeSuffix, "gopkg.in/yaml.v3"}, diff --git a/internal/archtest/recursion_test.go b/internal/archtest/recursion_test.go index 684e913de..c879fde8f 100644 --- a/internal/archtest/recursion_test.go +++ b/internal/archtest/recursion_test.go @@ -52,11 +52,6 @@ var loweringRecursions = [][]string{ // an object's key is built from its elements', each a value keyed the same // way. Bounded by maxValueKeyDepth. {"listKey", "objectKey", "valueKey"}, - // The $dynamicAnchor index. walk hands a mapping to walkMapping, which walks - // each of that mapping's values back through walk. Bounded by charge, which - // refuses past maxDynamicAnchorDepth or a spent node budget and records the - // refusal, so a caller learns the index is partial. - {"anchorWalk.walk", "anchorWalk.walkMapping"}, // Property lookup through a composition. Finding a property by wire name // descends into a model's base and mixins, each of which is a model whose // properties are looked up the same way. @@ -446,21 +441,22 @@ func TestLoweringCallGraph_RecordsAHandoffAsAnEdge(t *testing.T) { } // TestLoweringCallGraph_RecordsAMethodEdge holds the graph to the fix for GitHub -// #224 on the tree rather than on a planted source. anchorWalk's two methods -// call each other and the walk descends into itself, and a graph built from free -// functions alone described every one of those edges as absent. +// #224 on the tree rather than on a planted source. The nullability reader's +// methods call each other through their receiver, round the cycle pinned +// above, and a graph built from free functions alone described every one of +// those edges as absent. // -// It names charge as well, which closes no cycle: an edge the graph cannot see +// It names widen as well, which closes no cycle: an edge the graph cannot see // is unheld whether or not it currently changes an answer. func TestLoweringCallGraph_RecordsAMethodEdge(t *testing.T) { t.Parallel() graph := loweringCallGraph(t) edges := [][2]string{ - {"anchorWalk.walk", "anchorWalk.walkMapping"}, - {"anchorWalk.walkMapping", "anchorWalk.walk"}, - {"anchorWalk.walk", "anchorWalk.walk"}, - {"anchorWalk.walk", "anchorWalk.charge"}, + {"nulls.schema", "nulls.keywords"}, + {"nulls.keywords", "nulls.allOf"}, + {"nulls.allOf", "nulls.conjunct"}, + {"nulls.schema", "nulls.widen"}, } for _, e := range edges { // Membership is read with a comma-ok rather than require.Contains, which @@ -479,20 +475,18 @@ func TestLoweringCallGraph_RecordsAMethodEdge(t *testing.T) { // write: a constructor's local, a pointer parameter, a composite literal; the // multi-result bind they no longer write is planted beside them. // -// Not every selector reaches a node: anchorWalk.walkMapping reads its pairs -// through w.view.MappingPairs, a method of a type nodeview declares, and no -// package this pin reads holds a node for it. It adds no edge. +// Not every selector reaches a node: soleAnchorSite reads its sites through +// c.DynamicAnchorSites, a method of a type lowering declares, and no package +// this pin reads holds a node for it. It adds no edge. func TestLoweringCallGraph_ResolvesAMethodOnAValue(t *testing.T) { t.Parallel() graph := loweringCallGraph(t) edges := [][2]string{ - {"dynamicAnchors", "anchorWalk.walk"}, // w := newAnchorWalk(…) + {"composedVariantRef", "nulls.schema"}, // n := newNulls(c) {"LowerService", "serviceGroups.finalize"}, // groups := newServiceGroups() {"lowerPathItem", "serviceGroups.place"}, // a *serviceGroups parameter {"lowerWebhooks", "serviceGroups.group"}, // the same, in the other caller - {"soleAnchorSite", "AnchorIndex.sites"}, // an *AnchorIndex parameter - {"dynamicHop", "AnchorIndex.sites"}, // the same, in the other caller {"optionsFrom", "Options.withDefaults"}, // Options{}.withDefaults() } for _, e := range edges { diff --git a/internal/lintrules/naming.go b/internal/lintrules/naming.go index 9e2094014..8e9750df1 100644 --- a/internal/lintrules/naming.go +++ b/internal/lintrules/naming.go @@ -103,7 +103,8 @@ func (c nameCheck) node(n ast.Node) { // literal reports each name channel a struct literal fills, keyed or by // position, with anything but a framework call, an element of a slice or map -// of them, its type elided, among them. +// of them, its type elided, among them. A key may name a promoted field, so +// it is judged in the struct declaring it, not the literal's. func (c nameCheck) literal(lit *ast.CompositeLit) { t := c.pass.TypesInfo.TypeOf(lit) if ptr, ok := t.(*types.Pointer); ok { @@ -114,12 +115,15 @@ func (c nameCheck) literal(lit *ast.CompositeLit) { return } for i, elt := range lit.Elts { - name, value := fields.Field(i).Name(), elt + owner, name, value := fields, "", elt if kv, keyed := elt.(*ast.KeyValueExpr); keyed { key, _ := kv.Key.(*ast.Ident) // a struct literal's keys are its fields' names - name, value = key.Name, kv.Value + _, path, _ := types.LookupFieldOrMethod(t, false, c.pass.Pkg, key.Name) + owner, name, value = declaringAlong(t, path), key.Name, kv.Value + } else { + name = fields.Field(i).Name() } - if c.isChannel(fields, name) && !frameworkCall(c.pass.TypesInfo, value) { + if c.isChannel(owner, name) && !frameworkCall(c.pass.TypesInfo, value) { c.pass.Reportf(value.Pos(), notFramework, name) } } @@ -182,13 +186,17 @@ func (c nameCheck) channel(e ast.Expr) (string, bool) { return sel.Sel.Name, c.isChannel(declaring(selection), sel.Sel.Name) } -// declaring returns the struct that declares the field selection selects, -// stepping through the embedded fields on its path. Only a field is assigned -// or has its address taken, so every step is a struct or a pointer to one. +// declaring returns the struct that declares the field selection selects. +// Only a field is assigned or has its address taken. func declaring(selection *types.Selection) *types.Struct { - t := selection.Recv() - path := selection.Index() - for _, i := range path[:len(path)-1] { + return declaringAlong(selection.Recv(), selection.Index()) +} + +// declaringAlong returns the struct that declares the field path reaches from +// t, stepping through the embedded fields before its last index, each a +// struct or a pointer to one. +func declaringAlong(t types.Type, path []int) *types.Struct { + for _, i := range path[:max(len(path)-1, 0)] { t = structOf(t).Field(i).Type() } return structOf(t) diff --git a/internal/lintrules/testdata/mod/compilers/graphql/naming.go b/internal/lintrules/testdata/mod/compilers/graphql/naming.go index 12ea4dbe1..be11abcd5 100644 --- a/internal/lintrules/testdata/mod/compilers/graphql/naming.go +++ b/internal/lintrules/testdata/mod/compilers/graphql/naming.go @@ -99,11 +99,13 @@ func plantNaming(name string, words []string, n *named, c *common, big ir.BigVal field := ir.Naming{Canonical: compile.NamingFor(name).Canonical} // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" positional := irx.Naming{name, localWords(name), compile.SubHint(name, "x"), nil} // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" var late ir.Naming - late.Canonical = strings.ToLower(name) // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" - late.Hint = localWords(name) // want "namechannels: ir.Naming.Hint is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" - n.Hint = name // want "namechannels: ir.Naming.Hint is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" - c.Name.Canonical = name // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" - late.Canonical += "_x" // want "namechannels: ir.Naming.Canonical is extended in place; fill it whole, from compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + late.Canonical = strings.ToLower(name) // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + late.Hint = localWords(name) // want "namechannels: ir.Naming.Hint is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + n.Hint = name // want "namechannels: ir.Naming.Hint is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + promoted := named{Hint: localWords(name)} // want "namechannels: ir.Naming.Hint is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + promotedAll := named{Source: name, Canonical: ir.CanonicalWords(name), Hint: compile.SubHint(name, "x")} + c.Name.Canonical = name // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" + late.Canonical += "_x" // want "namechannels: ir.Naming.Canonical is extended in place; fill it whole, from compile.NamingFor, compile.NamingHint or ir.CanonicalWords" var other string other, late.Hint = twoWords(name) // want "namechannels: ir.Naming.Hint is filled from a call returning several values; fill it alone, from compile.NamingFor, compile.NamingHint or ir.CanonicalWords" for _, late.Hint = range words { // want "namechannels: ir.Naming.Hint is filled by a range clause; fill it from compile.NamingFor, compile.NamingHint or ir.CanonicalWords" @@ -123,6 +125,7 @@ func plantNaming(name string, words []string, n *named, c *common, big ir.BigVal kept := ir.Naming{Source: name, Aliases: []string{name}} _, _, _, _, _, _ = byKey, pointers, other, converted, method, parenthesized _, _ = spelled, spelledHint + _, _ = promoted, promotedAll return []ir.Naming{declared, hinted, carried, wrapped, field, positional, late, canonical, sub, made, kept, {Canonical: localWords(name)}} // want "namechannels: ir.Naming.Canonical is filled by something other than a call into ir or compilers/compile; use compile.NamingFor, compile.NamingHint or ir.CanonicalWords" } diff --git a/testdata/conformance/openapi/yaml-merge-keys.golden.json b/testdata/conformance/openapi/yaml-merge-keys.golden.json new file mode 100644 index 000000000..8cf580265 --- /dev/null +++ b/testdata/conformance/openapi/yaml-merge-keys.golden.json @@ -0,0 +1,674 @@ +{ + "idSpaces": { + "auth": [ + "openapi" + ], + "e": [ + "anon", + "openapi" + ], + "g": [ + "default", + "path-prefix", + "tags", + "webhooks" + ], + "op": [ + "openapi" + ], + "p": [ + "openapi" + ], + "param": [ + "openapi" + ], + "s": [ + "openapi" + ], + "t": [ + "anon", + "composed", + "openapi" + ] + }, + "name": "YAMLMergeKeys", + "version": "1", + "docs": {}, + "services": [ + { + "id": "s/openapi/0", + "name": { + "source": "YAMLMergeKeys", + "canonical": "yaml_merge_keys" + }, + "docs": {}, + "groups": [ + { + "id": "g/default", + "name": { + "hint": "default" + }, + "docs": {}, + "operations": [ + { + "id": "op/openapi/paths/~1merged/get", + "name": { + "source": "getMerged", + "canonical": "get_merged" + }, + "docs": {}, + "params": [ + { + "id": "param/openapi/paths/~1merged/get/parameters/q/query", + "name": { + "source": "q", + "canonical": "q" + }, + "type": { + "target": "t/prim/string", + "nullable": false + }, + "required": false, + "docs": {}, + "unmodeled": { + "openapi:allowEmptyValue": { + "reason": "no_ir_home", + "value": true, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/parameters/0/allowEmptyValue" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/parameters/0" + } + } + ], + "responses": [ + { + "name": { + "hint": "200" + }, + "conditions": { + "statusCodes": [ + { + "from": 200, + "to": 200 + } + ] + }, + "payload": { + "contents": [ + { + "mediaType": "application/json", + "type": { + "target": "t/anon/paths/~1merged/get/responses/200/content/application~1json/schema", + "nullable": false + } + } + ] + }, + "headers": [ + { + "id": "p/openapi/paths/~1merged/get/responses/200/headers/X-Rate", + "name": { + "source": "X-Rate", + "canonical": "x_rate" + }, + "wireName": "X-Rate", + "type": { + "target": "t/prim/integer", + "nullable": false + }, + "required": false, + "clientOptional": false, + "defaultAdded": false, + "visibility": { + "none": false + }, + "flatten": false, + "eventHeader": false, + "eventPayload": false, + "secret": false, + "docs": {}, + "unmodeled": { + "openapi:style": { + "reason": "no_ir_home", + "value": "simple", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/headers/X-Rate/style" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/headers/X-Rate" + } + } + ], + "docs": { + "description": "ok" + }, + "unmodeled": { + "openapi:links": { + "reason": "no_ir_home", + "value": { + "self": { + "operationId": "getMerged" + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/links" + } + } + } + } + ], + "oneWay": false, + "idempotency": {}, + "bindings": { + "http": [ + { + "method": "GET", + "uriTemplate": "/merged", + "sharedRoute": false, + "paramBindings": [ + { + "param": "param/openapi/paths/~1merged/get/parameters/q/query", + "location": "query", + "wireName": "q", + "style": "form", + "explode": true, + "allowReserved": false + } + ], + "checksumRequired": false, + "isWebhook": false + } + ] + }, + "unmodeled": { + "openapi:operationServers": { + "reason": "no_ir_home", + "value": [ + { + "url": "https://operation.example" + } + ], + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/servers" + } + }, + "openapi:pathItem/mergedBogus": { + "reason": "out_of_scope", + "value": { + "responses": { + "200": { + "description": "MERGED" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/mergedBogus" + } + }, + "openapi:pathItemDescription": { + "reason": "no_ir_home", + "value": "Shared description", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/description" + } + }, + "openapi:pathItemSummary": { + "reason": "no_ir_home", + "value": "Shared summary", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/summary" + } + }, + "openapi:servers": { + "reason": "no_ir_home", + "value": [ + { + "url": "https://path.example" + } + ], + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/servers" + } + }, + "openapi:x-big": { + "reason": "vendor_extension", + "value": 1e400, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/x-big" + } + }, + "openapi:x-codes": { + "reason": "vendor_extension", + "value": { + "100": "Continue", + "200": "OK", + "true": "T" + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/x-codes" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get" + } + }, + { + "id": "op/openapi/paths/~1merged/post", + "name": { + "source": "postMerged", + "canonical": "post_merged" + }, + "docs": {}, + "request": { + "contents": [ + { + "mediaType": "multipart/form-data", + "type": { + "target": "t/anon/paths/~1merged/post/requestBody/content/multipart~1form-data/schema", + "nullable": false + }, + "encoding": { + "p/openapi/paths/~1merged/post/requestBody/content/multipart~1form-data/schema/properties/f": { + "contentTypes": [ + "text/plain" + ], + "multi": false, + "filename": false + } + }, + "unmodeled": { + "openapi:encoding/f/allowReserved": { + "reason": "no_ir_home", + "value": true, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post/requestBody/content/multipart~1form-data/encoding/f/allowReserved" + } + } + } + } + ], + "required": false + }, + "responses": [ + { + "name": { + "hint": "204" + }, + "conditions": { + "statusCodes": [ + { + "from": 204, + "to": 204 + } + ] + }, + "docs": { + "description": "done" + } + } + ], + "oneWay": false, + "idempotency": {}, + "bindings": { + "http": [ + { + "method": "POST", + "uriTemplate": "/merged", + "sharedRoute": false, + "requestContentTypes": [ + "multipart/form-data" + ], + "checksumRequired": false, + "isWebhook": false + } + ] + }, + "unmodeled": { + "openapi:pathItem/mergedBogus": { + "reason": "out_of_scope", + "value": { + "responses": { + "200": { + "description": "MERGED" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/mergedBogus" + } + }, + "openapi:pathItemDescription": { + "reason": "no_ir_home", + "value": "Shared description", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/description" + } + }, + "openapi:pathItemSummary": { + "reason": "no_ir_home", + "value": "Shared summary", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/summary" + } + }, + "openapi:servers": { + "reason": "no_ir_home", + "value": [ + { + "url": "https://path.example" + } + ], + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/servers" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post" + } + } + ] + } + ], + "provenance": { + "source": 0 + } + } + ], + "types": { + "t/anon/paths/~1merged/get/responses/200/content/application~1json/schema": { + "kind": "model", + "id": "t/anon/paths/~1merged/get/responses/200/content/application~1json/schema", + "name": { + "hint": "response" + }, + "anonymous": true, + "docs": {}, + "sensitive": false, + "examples": [ + { + "value": { + "kind": "object", + "object": [ + { + "name": "a", + "value": { + "kind": "number", + "num": "1" + } + }, + { + "name": "b", + "value": { + "kind": "number", + "num": "2" + } + } + ] + } + } + ], + "unmodeled": { + "openapi:not": { + "reason": "validation_only", + "value": { + "required": [ + "forbidden" + ] + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/content/application~1json/schema/not" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/content/application~1json/schema" + }, + "abstract": false, + "positional": false, + "inputOnly": false + }, + "t/anon/paths/~1merged/post/requestBody/content/multipart~1form-data/schema": { + "kind": "model", + "id": "t/anon/paths/~1merged/post/requestBody/content/multipart~1form-data/schema", + "name": { + "hint": "post_merged_request" + }, + "anonymous": true, + "docs": {}, + "sensitive": false, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post/requestBody/content/multipart~1form-data/schema" + }, + "properties": [ + { + "id": "p/openapi/paths/~1merged/post/requestBody/content/multipart~1form-data/schema/properties/f", + "name": { + "source": "f", + "canonical": "f" + }, + "wireName": "f", + "type": { + "target": "t/prim/string", + "nullable": false + }, + "required": false, + "clientOptional": false, + "defaultAdded": false, + "visibility": { + "none": false + }, + "flatten": false, + "eventHeader": false, + "eventPayload": false, + "secret": false, + "docs": {}, + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post/requestBody/content/multipart~1form-data/schema/properties/f" + } + } + ], + "abstract": false, + "positional": false, + "inputOnly": false + }, + "t/prim/integer": { + "kind": "primitive", + "id": "t/prim/integer", + "name": {}, + "anonymous": false, + "docs": {}, + "sensitive": false, + "provenance": { + "source": -1 + }, + "prim": "integer" + }, + "t/prim/string": { + "kind": "primitive", + "id": "t/prim/string", + "name": {}, + "anonymous": false, + "docs": {}, + "sensitive": false, + "provenance": { + "source": -1 + }, + "prim": "string" + } + }, + "auth": { + "auth/openapi/components/securitySchemes/key": { + "id": "auth/openapi/components/securitySchemes/key", + "name": { + "source": "key", + "canonical": "key" + }, + "kind": "apiKey", + "docs": {}, + "in": "header", + "keyName": "X-Key", + "unmodeled": { + "openapi:bearerFormat": { + "reason": "degraded_lowering", + "value": "JWT", + "provenance": { + "source": 0, + "pointer": "/components/securitySchemes/key/bearerFormat" + } + } + }, + "provenance": { + "source": 0, + "pointer": "/components/securitySchemes/key" + } + } + }, + "servers": [ + { + "name": { + "hint": "server" + }, + "urlTemplate": "/", + "description": {} + } + ], + "diagnostics": [ + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "security scheme bearerFormat is not defined by type \"apiKey\"; kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/components/securitySchemes/key/bearerFormat" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "parameter allowEmptyValue has no ir.HTTPParamBinding home; kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/parameters/0/allowEmptyValue" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "header style has no ir.Property home; kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/headers/X-Rate/style" + } + }, + { + "severity": "info", + "code": "openapi/validation-only-keyword", + "message": "validation-only keyword \"not\" kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get/responses/200/content/application~1json/schema" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "operation servers kept under Unmodeled; an operation has no server-scope list to bind them to, and these override any path-item servers kept beside them", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "path-item servers kept under Unmodeled; an operation has no server-scope list to bind them to", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "path-item documentation kept under Unmodeled; it documents the path rather than this operation, and ir.Docs holds the operation's own summary and description", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/get" + } + }, + { + "severity": "warning", + "code": "openapi/unknown-object-key", + "message": "key \"mergedBogus\" is not defined by the OpenAPI object it is written on and is not an x- extension; kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/mergedBogus" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "encoding allowReserved has no ir.PartEncoding home; kept verbatim under Unmodeled", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post/requestBody/content/multipart~1form-data/encoding/f/allowReserved" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "path-item servers kept under Unmodeled; an operation has no server-scope list to bind them to", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post" + } + }, + { + "severity": "info", + "code": "openapi/degraded-construct", + "message": "path-item documentation kept under Unmodeled; it documents the path rather than this operation, and ir.Docs holds the operation's own summary and description", + "provenance": { + "source": 0, + "pointer": "/paths/~1merged/post" + } + } + ], + "sources": [ + { + "format": "openapi@3.1", + "path": "yaml-merge-keys.yaml", + "hash": "f2f7c894f515e8d8bf47ce396f7c8fa0f28b007b8f965706d349f9662eeee959" + } + ] +} diff --git a/testdata/conformance/openapi/yaml-merge-keys.yaml b/testdata/conformance/openapi/yaml-merge-keys.yaml new file mode 100644 index 000000000..36bdf2208 --- /dev/null +++ b/testdata/conformance/openapi/yaml-merge-keys.yaml @@ -0,0 +1,41 @@ +openapi: 3.1.0 +info: {title: YAMLMergeKeys, version: "1"} +# Each construct below reaches the compiler only through a `<<` merge key or a +# key YAML types as no string, and is read as the parser reads it: merged in, +# the merged members first, and every key as the text it spells. +paths: + /merged: + <<: + - {summary: Shared summary, description: Shared description, servers: [{url: "https://path.example"}]} + - {mergedBogus: {responses: {"200": {description: MERGED}}}} + get: + operationId: getMerged + <<: {servers: [{url: "https://operation.example"}]} + x-codes: {<<: {100: Continue}, 200: OK, true: T} + x-big: 1e400 + parameters: + - {<<: {allowEmptyValue: true}, name: q, in: query, schema: {type: string}} + responses: + "200": + description: ok + <<: {links: {self: {operationId: getMerged}}} + headers: + X-Rate: {<<: {style: simple}, schema: {type: integer}} + content: + application/json: + schema: + type: object + <<: {not: {required: [forbidden]}} + example: {<<: {a: 1}, b: 2} + post: + operationId: postMerged + requestBody: + content: + multipart/form-data: + schema: {type: object, properties: {f: {type: string}}} + encoding: + f: {<<: {allowReserved: true}, contentType: text/plain} + responses: {"204": {description: done}} +components: + securitySchemes: + key: {<<: {bearerFormat: JWT}, type: apiKey, name: X-Key, in: header} diff --git a/testdata/openapi/pointer_hint_pure_ref_under_paths.yaml b/testdata/openapi/pointer_hint_pure_ref_under_paths.yaml index 7473d8a19..295fccca5 100644 --- a/testdata/openapi/pointer_hint_pure_ref_under_paths.yaml +++ b/testdata/openapi/pointer_hint_pure_ref_under_paths.yaml @@ -1,6 +1,6 @@ # GitHub #519 under /paths: a pure $ref position's own declaration interns -# nothing there either, and positionHint cannot replay the enclosing hint -# under /paths — it comes from an operationId or a response, which the +# nothing there either, and the pointer alone cannot name the position under +# /paths — its name comes from an operationId or a response, which the # pointer does not record. Declaring /b before /a is the order that exposed # it: /b's declaration reaches the position and interns nothing, so there was # nothing for it to rename once /a's reference hoisted a node under its own