refactor(compilers/openapi): unify YAML reading and node naming

compilers/compile/naming.go

@@ -1,6 +1,10 @@
 package compile
 
-import "github.com/dexpace/morphic/ir"
+import (
+	"strings"
+
+	"github.com/dexpace/morphic/ir"
+)
 
 // emptyNameHint is the hint minted for an entity the source left unnamed: an
 // enum member whose value is "", a property or component keyed by it, a tag
@@ -59,6 +63,24 @@ func SubHint(parent, suffix string) string {
 	return neutralHint(parent) + "_" + neutralHint(suffix)
 }
 
+// SubHints is SubHint along a chain, each suffix composed onto the hint
+// before it: SubHints(p, a, b) is SubHint(SubHint(p, a), b), and SubHints(p)
+// is p. Since a composed hint is neutral already, the chain is each part
+// neutralized once and joined, so a hint composed of n parts costs n parts,
+// not the n² a fold of SubHint spends re-reading its growing parent.
+func SubHints(parent string, suffixes ...string) string {
+	if len(suffixes) == 0 {
+		return parent
+	}
+	var b strings.Builder
+	b.WriteString(neutralHint(parent))
+	for _, s := range suffixes {
+		b.WriteByte('_')
+		b.WriteString(neutralHint(s))
+	}
+	return b.String()
+}
+
 // neutralHint returns the neutral word sequence of hint, or the minted name when
 // the position it was derived from names nothing a word can be read out of.
 func neutralHint(hint string) string {

compilers/compile/naming_test.go

@@ -118,3 +118,34 @@ func TestSubHint_IsItselfANeutralHint(t *testing.T) {
 	assert.Equal(t, "combo_a_2_item", nested)
 	assert.Equal(t, nested, ir.CanonicalWords(nested), "the grammar leaves a composed hint alone")
 }
+
+// TestSubHints_IsAFoldOfSubHint pins SubHints to the SubHint chain it
+// shortens, over parents and suffixes that neutralize to nothing, carry case,
+// digits and separators, and are already composed, and to the parent alone
+// for an empty chain.
+func TestSubHints_IsAFoldOfSubHint(t *testing.T) {
+	t.Parallel()
+	parts := []string{"", "***", "Combo_A", "userID", "2", "item", "x-ext", "empty_item", "HTTPServer2Go"}
+	for _, parent := range parts {
+		assert.Equal(t, parent, compile.SubHints(parent), "an empty chain leaves the parent as it is")
+		for _, a := range parts {
+			for _, b := range parts {
+				assert.Equal(t, compile.SubHint(compile.SubHint(parent, a), b), compile.SubHints(parent, a, b),
+					"%q, %q, %q", parent, a, b)
+			}
+		}
+	}
+}
+
+// FuzzSubHints_IsAFoldOfSubHint holds SubHints to the SubHint chain on any
+// three parts.
+func FuzzSubHints_IsAFoldOfSubHint(f *testing.F) {
+	f.Add("Combo_A", "2", "item")
+	f.Add("", "***", "éA")
+	f.Fuzz(func(t *testing.T, parent, a, b string) {
+		want := compile.SubHint(compile.SubHint(compile.SubHint(parent, a), b), a)
+		if got := compile.SubHints(parent, a, b, a); got != want {
+			t.Fatalf("SubHints(%q, %q, %q, %q) = %q, the chain %q", parent, a, b, a, got, want)
+		}
+	})
+}

compilers/openapi/conformance_test.go

@@ -237,6 +237,7 @@ func conformanceCases() []conformanceCase {
 		{"path-item-docs", assertPathItemDocs},
 		{"path-item-operations", assertPathItemOperations},
 		{"path-item-anchored-key", assertPathItemAnchoredKey},
+		{"yaml-merge-keys", assertYAMLMergeKeys},
 		{"path-item-ref-siblings", assertPathItemRefSiblings},
 		{"deprecation", assertDeprecation},
 		{"extension-promotion", assertExtensionPromotion},

compilers/openapi/conformance_unmodeled_test.go

@@ -1072,3 +1072,91 @@ func indexByParam(bindings []ir.HTTPParamBinding) map[string]ir.HTTPParamBinding
 	}
 	return out
 }
+
+// assertYAMLMergeKeys pins that every reader of the source reads a mapping as
+// the parser does (GitHub #1089): a field, a stray key or a keyword merged in
+// through `<<` is kept and announced as a written one is, by each kind of
+// reader that keeps one (GitHub #384, #395), an extension keeps a key YAML
+// types as a number or a boolean as its text (GitHub #617), a plain number
+// past float64's range stays a number (GitHub #783), and an example's members
+// come in the parser's order. The spec writes no alias, so the order oracle
+// can permute it; alias keys are the readers' own tests'.
+func assertYAMLMergeKeys(t *testing.T, doc *ir.Document, diags []ir.Diagnostic) {
+	t.Helper()
+	op, ok := opByName(doc, "getMerged")
+	require.True(t, ok)
+	assertKept(t, op.Unmodeled, []keptEntry{
+		{"openapi:pathItemSummary", `"Shared summary"`, "/paths/~1merged/summary"},
+		{"openapi:pathItemDescription", `"Shared description"`, "/paths/~1merged/description"},
+		{"openapi:servers", `[{"url":"https://path.example"}]`, "/paths/~1merged/servers"},
+		{"openapi:pathItem/mergedBogus", `{"responses":{"200":{"description":"MERGED"}}}`, "/paths/~1merged/mergedBogus"},
+		{"openapi:operationServers", `[{"url":"https://operation.example"}]`, "/paths/~1merged/get/servers"},
+		{"openapi:x-codes", `{"100":"Continue","200":"OK","true":"T"}`, "/paths/~1merged/get/x-codes"},
+		{"openapi:x-big", `1e400`, "/paths/~1merged/get/x-big"},
+	})
+	assert.Equal(t, []ir.Severity{ir.SeverityWarning},
+		diagsAt(diags, "openapi/unknown-object-key", "/paths/~1merged/mergedBogus"), "the merged stray key is announced")
+	for _, d := range diags {
+		assert.NotContains(t, d.Message, "could not be serialized", "every extension converts")
+		assert.NotEqual(t, "openapi/unknown-key-unreachable", d.Code, "every merged key is reached")
+	}
+	assertMergedReaders(t, doc, op)
+
+	body, ok := doc.Types["t/anon/paths/~1merged/get/responses/200/content/application~1json/schema"]
+	require.True(t, ok)
+	assertKept(t, body.Common().Unmodeled, []keptEntry{
+		{"openapi:not", `{"required":["forbidden"]}`, "/paths/~1merged/get/responses/200/content/application~1json/schema/not"},
+	})
+	examples := body.Common().Examples
+	require.Len(t, examples, 1)
+	require.NotNil(t, examples[0].Value)
+	members := make([]string, 0, len(examples[0].Value.Object))
+	for _, f := range examples[0].Value.Object {
+		members = append(members, f.Name)
+	}
+	assert.Equal(t, []string{"a", "b"}, members, "the merged member first, as the parser reads it")
+}
+
+// assertMergedReaders pins the merged constructs kept beside the operation:
+// on its parameter, its response and that response's header, the other
+// operation's encoding, and the security scheme.
+func assertMergedReaders(t *testing.T, doc *ir.Document, op ir.Operation) {
+	t.Helper()
+	require.Len(t, op.Params, 1)
+	assertKept(t, op.Params[0].Unmodeled, []keptEntry{
+		{"openapi:allowEmptyValue", `true`, "/paths/~1merged/get/parameters/0/allowEmptyValue"},
+	})
+	require.Len(t, op.Responses, 1)
+	assertKept(t, op.Responses[0].Unmodeled, []keptEntry{
+		{"openapi:links", `{"self":{"operationId":"getMerged"}}`, "/paths/~1merged/get/responses/200/links"},
+	})
+	require.Len(t, op.Responses[0].Headers, 1)
+	assertKept(t, op.Responses[0].Headers[0].Unmodeled, []keptEntry{
+		{"openapi:style", `"simple"`, "/paths/~1merged/get/responses/200/headers/X-Rate/style"},
+	})
+	post, ok := opByName(doc, "postMerged")
+	require.True(t, ok)
+	require.NotNil(t, post.Request)
+	require.Len(t, post.Request.Contents, 1)
+	assertKept(t, post.Request.Contents[0].Unmodeled, []keptEntry{{"openapi:encoding/f/allowReserved", `true`,
+		"/paths/~1merged/post/requestBody/content/multipart~1form-data/encoding/f/allowReserved"}})
+	scheme, ok := doc.Auth["auth/openapi/components/securitySchemes/key"]
+	require.True(t, ok)
+	assertKept(t, scheme.Unmodeled, []keptEntry{
+		{"openapi:bearerFormat", `"JWT"`, "/components/securitySchemes/key/bearerFormat"},
+	})
+}
+
+// keptEntry is an Unmodeled entry expected under key, holding value as JSON,
+// located at the pointer at.
+type keptEntry struct{ key, value, at string }
+
+// assertKept requires each of want in got, with its value and location.
+func assertKept(t *testing.T, got ir.Unmodeled, want []keptEntry) {
+	t.Helper()
+	for _, w := range want {
+		entry := unmodeledEntry(t, got, w.key)
+		assert.Equal(t, w.value, string(entry.Value), w.key)
+		assert.Equal(t, jsontext.Pointer(w.at), entry.Provenance.Pointer, w.key)
+	}
+}

compilers/openapi/helpers_test.go

@@ -51,7 +51,7 @@ func loweredFor(t *testing.T, src string) (*lowerer, []ir.Diagnostic) {
 func lowerSpec(t *testing.T, src string) (*ir.Document, []ir.Diagnostic) {
 	t.Helper()
 	l, diags := loweredFor(t, src)
-	l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors)) // named components; the entry Compile's run() calls first
+	l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types)) // named components; the entry Compile's run() calls first
 	doc := l.document(t)
 	return doc, append(diags, l.diags.List()...)
 }
@@ -65,11 +65,11 @@ func lowerServiceSpec(t *testing.T, src string) (*ir.Document, ir.Service, []ir.
 		require.NoError(t, err)
 		require.NotNil(t, loadedDoc)
 		l := newLowerer(loadedDoc, Options{}.withDefaults())
-		l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types, &l.anchors))
+		l.diags.AppendAll(schema.LowerComponentSchemas(t.Context(), l.ctx, l.types))
 		auth, authDiags := auth.LowerSecuritySchemes(l.ctx)
 		l.out.Auth = auth
 		l.diags.AppendAll(authDiags)
-		svc, tagDefs, svcDiags := operation.LowerService(t.Context(), l.ctx.WithAuth(l.out.Auth), l.types, &l.anchors)
+		svc, tagDefs, svcDiags := operation.LowerService(t.Context(), l.ctx.WithAuth(l.out.Auth), l.types)
 		l.out.TagDefs = tagDefs
 		l.diags.AppendAll(svcDiags)
 		l.out.Services = []ir.Service{svc}
@@ -81,16 +81,15 @@ func lowerServiceSpec(t *testing.T, src string) (*ir.Document, ir.Service, []ir.
 }
 
 // lowerer is test scaffolding, not a compiler type. The compiler has no such
-// struct: every lowering is a function of the context, the registry and the
-// memos, and run owns the document being built (#177). Tests that drive one
-// lowering in isolation still want those five things in one place, and holding
-// them here keeps that convenience out of the production call graph.
+// struct: every lowering is a function of the context and the registry, and
+// run owns the document being built (#177). Tests that drive one lowering in
+// isolation still want those four things in one place, and holding them here
+// keeps that convenience out of the production call graph.
 type lowerer struct {
-	ctx     lowering.Ctx
-	out     *ir.Document
-	types   *compile.Types
-	diags   compile.Diags
-	anchors schema.AnchorIndex
+	ctx   lowering.Ctx
+	out   *ir.Document
+	types *compile.Types
+	diags compile.Diags
 }
 
 // lowererOver is the only place the fixture's fields are initialised. Both

compilers/openapi/internal/anchors/anchors.go

@@ -0,0 +1,127 @@
+// Package anchors indexes the $dynamicAnchor declarations of a source
+// document, which a $dynamicRef expands against. It reads the tree as the
+// parser does, through aliases and `<<` merges: an anchor a merge or an alias
+// pulls into a position is declared there as well, so "declared exactly once"
+// counts what the document declares rather than what it spells out.
+package anchors
+
+import (
+	"encoding/json/jsontext"
+	"slices"
+	"strconv"
+
+	yaml "gopkg.in/yaml.v3"
+
+	"github.com/dexpace/morphic/compilers/openapi/internal/ids"
+	"github.com/dexpace/morphic/compilers/openapi/internal/nodeview"
+)
+
+// MaxDepth bounds how deep the index walk reads. Document trees nest far
+// shallower — a schema under a media type under a response is about ten
+// levels — so it only ever stops a pathological structure.
+const MaxDepth = 512
+
+// MaxNodes bounds how many nodes the walk reads in all. Depth alone does not
+// bound it: the walk follows aliases, and an alias graph is a DAG, so a shallow
+// document can present far more paths than it has nodes.
+const MaxNodes = 1 << 20
+
+// Index is a document's $dynamicAnchor declarations: for each name, the
+// pointers of the mappings declaring it, in document order.
+type Index struct {
+	byName   map[string][]jsontext.Pointer
+	complete bool
+}
+
+// Sites returns the pointers declaring name.
+func (x Index) Sites(name string) []jsontext.Pointer { return slices.Clone(x.byName[name]) }
+
+// Complete reports whether the walk read the whole tree. A partial index can
+// only undercount, and an undercount reads as "declared exactly once", the one
+// count that expands a $dynamicRef, so a reader of a partial one says so.
+func (x Index) Complete() bool { return x.complete }
+
+// frame is one entry of the walk's stack: a node to read at a pointer and
+// depth, or, where anchor is set, a mapping's $dynamicAnchor to record.
+type frame struct {
+	node    *yaml.Node
+	pointer jsontext.Pointer
+	depth   int
+	anchor  bool
+}
+
+// Build indexes the document under root. It reads the raw tree, since the
+// parser's schema model has no field for the keyword
+// (docs/parser-library.md#unmodeled-keywords), position by position as the
+// parser presents each mapping's pairs (nodeview.View.MappingPairs): a walk
+// of Content (nodeview.Walk) would miss what a merge brings in. It is bounded
+// by MaxDepth and MaxNodes, and by the view's merge bound, each reported by
+// Complete.
+func Build(root *yaml.Node) Index {
+	return build(root, MaxDepth, MaxNodes)
+}
+
+// build is Build within maxDepth levels and maxNodes nodes.
+func build(root *yaml.Node, maxDepth, maxNodes int) Index {
+	view := nodeview.New()
+	x := Index{byName: map[string][]jsontext.Pointer{}, complete: true}
+	stack := []frame{{node: root}}
+	for budget := maxNodes; len(stack) > 0; {
+		f := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if f.anchor {
+			x.record(f.node, f.pointer)
+			continue
+		}
+		n := nodeview.Deref(f.node)
+		if n == nil {
+			continue
+		}
+		if f.depth > maxDepth || budget == 0 {
+			x.complete = false
+			continue
+		}
+		budget--
+		stack = append(stack, children(view, n, f)...)
+	}
+	x.complete = x.complete && !view.Exhausted()
+	return x
+}
+
+// children returns what the walk reads below n, read at f, last first so the
+// stack yields them in order: a document's content, a sequence's items, a
+// mapping's values, and a mapping's $dynamicAnchor where it writes one. A
+// scalar holds nothing.
+func children(view *nodeview.View, n *yaml.Node, f frame) []frame {
+	var out []frame
+	switch n.Kind {
+	case yaml.DocumentNode:
+		for _, c := range n.Content {
+			out = append(out, frame{node: c, pointer: f.pointer, depth: f.depth + 1})
+		}
+	case yaml.SequenceNode:
+		for i, c := range n.Content {
+			out = append(out, frame{node: c, pointer: f.pointer + ids.Ptr(strconv.Itoa(i)), depth: f.depth + 1})
+		}
+	case yaml.MappingNode:
+		for _, p := range view.MappingPairs(n) {
+			if p.Key == "$dynamicAnchor" {
+				out = append(out, frame{node: p.Val, pointer: f.pointer, anchor: true})
+				continue
+			}
+			out = append(out, frame{node: p.Val, pointer: f.pointer + ids.Ptr(p.Key), depth: f.depth + 1})
+		}
+	default:
+	}
+	slices.Reverse(out)
+	return out
+}
+
+// record indexes the anchor val names at pointer. A value that is not a
+// non-empty scalar names nothing a $dynamicRef could spell.
+func (x Index) record(val *yaml.Node, pointer jsontext.Pointer) {
+	if val == nil || val.Kind != yaml.ScalarNode || val.Value == "" {
+		return
+	}
+	x.byName[val.Value] = append(x.byName[val.Value], pointer)
+}

compilers/openapi/internal/anchors/anchors_bench_test.go

@@ -0,0 +1,42 @@
+package anchors_test
+
+import (
+	"os"
+	"testing"
+
+	yaml "gopkg.in/yaml.v3"
+
+	"github.com/dexpace/morphic/compilers/openapi/internal/anchors"
+)
+
+// BenchmarkBuild measures what deriving the $dynamicAnchor index at entry
+// would cost a document that never asks for it.
+//
+// The index stays a lazy memo (lowering.Ctx.DynamicAnchorSites) rather than
+// being built with the context: the walk is not free, almost no document
+// writes $dynamicRef, and building it emits a diagnostic that at entry would
+// reach documents that never use the keyword.
+//
+// Read the number against BenchmarkCompile_Petstore in compilers/openapi; the
+// design's claim is a ratio.
+func BenchmarkBuild(b *testing.B) {
+	// Four levels up: this package sits at compilers/openapi/internal/anchors,
+	// and the corpus is at the repo root. A missing or unparseable fixture
+	// stops the benchmark rather than skipping it, since a skip is silent
+	// without -v and exits 0.
+	data, err := os.ReadFile("../../../../testdata/conformance/openapi/allof-inline-merge.yaml")
+	if err != nil {
+		b.Fatalf("corpus fixture unavailable: %v", err)
+	}
+	var root yaml.Node
+	if err := yaml.Unmarshal(data, &root); err != nil {
+		b.Fatalf("fixture does not parse: %v", err)
+	}
+
+	b.ReportAllocs()
+	for b.Loop() {
+		if !anchors.Build(&root).Complete() {
+			b.Fatal("the fixture should not exhaust the walk bounds")
+		}
+	}
+}