From b2b8a3108f63ce52d7660eb5f0191221b60a93fb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Sep 2026 11:00:57 -0400 Subject: [PATCH 1/2] refactor(config): remove component base metadata --- lib/docs-renderer/src/main.rs | 9 +- lib/vector-config-common/src/constants.rs | 1 - lib/vector-config-common/src/schema/mod.rs | 1 + .../src/schema/navigation.rs | 93 +++++++++++++++++++ lib/vector-config/src/schema/parser/query.rs | 52 +++++++++++ .../src/schema/visitors/inline_single.rs | 54 +++++++++-- src/config/sink.rs | 1 - src/config/source.rs | 1 - src/config/transform.rs | 1 - .../commands/build/component_docs/runner.rs | 58 +++++++++--- 10 files changed, 242 insertions(+), 29 deletions(-) create mode 100644 lib/vector-config-common/src/schema/navigation.rs diff --git a/lib/docs-renderer/src/main.rs b/lib/docs-renderer/src/main.rs index 230d3f7527398..3e7cef44c7c3d 100644 --- a/lib/docs-renderer/src/main.rs +++ b/lib/docs-renderer/src/main.rs @@ -22,13 +22,8 @@ fn main() -> Result<()> { // Find the base component schema for the component type itself, which is analogous to // `SourceOuter`, `SinkOuter`, etc. We render the schema for that separately as it's meant // to be common across components of the same type, etc. - let base_component_schema = querier - .query() - .with_custom_attribute_kv( - constants::DOCS_META_COMPONENT_BASE_TYPE, - base_component_type, - ) - .run_single()?; + let base_component_schema = + querier.root_map_value_schema(&format!("{}s", base_component_type.as_str()))?; debug!( "Got base component schema for component type '{}'.", diff --git a/lib/vector-config-common/src/constants.rs b/lib/vector-config-common/src/constants.rs index 04b5b8284fdec..9d0a688b7942f 100644 --- a/lib/vector-config-common/src/constants.rs +++ b/lib/vector-config-common/src/constants.rs @@ -10,7 +10,6 @@ pub const COMPONENT_TYPE_SOURCE: &str = "source"; pub const COMPONENT_TYPE_TRANSFORM: &str = "transform"; pub const COMPONENT_TYPE_GLOBAL_OPTION: &str = "global_option"; pub const DOCS_META_ADDITIONAL_PROPS_DESC: &str = "docs::additional_props_description"; -pub const DOCS_META_COMPONENT_BASE_TYPE: &str = "docs::component_base_type"; pub const DOCS_META_COMPONENT_NAME: &str = "docs::component_name"; pub const DOCS_META_COMPONENT_TYPE: &str = "docs::component_type"; pub const DOCS_META_ENUM_CONTENT_FIELD: &str = "docs::enum_content_field"; diff --git a/lib/vector-config-common/src/schema/mod.rs b/lib/vector-config-common/src/schema/mod.rs index e5b59447cc488..ea957368bdca6 100644 --- a/lib/vector-config-common/src/schema/mod.rs +++ b/lib/vector-config-common/src/schema/mod.rs @@ -4,6 +4,7 @@ mod generator; mod json_schema; +mod navigation; pub mod visit; pub(crate) const DEFINITIONS_PREFIX: &str = "#/definitions/"; diff --git a/lib/vector-config-common/src/schema/navigation.rs b/lib/vector-config-common/src/schema/navigation.rs new file mode 100644 index 0000000000000..90bf162616201 --- /dev/null +++ b/lib/vector-config-common/src/schema/navigation.rs @@ -0,0 +1,93 @@ +use super::{DEFINITIONS_PREFIX, RootSchema, Schema, SchemaObject, Set}; + +impl RootSchema { + /// Finds the value schema of a named map at the configuration root. + /// + /// Follows local references and `allOf` (flattened fields), but does not + /// descend into nested properties or choose between union alternatives. + pub fn root_map_value_schema(&self, property: &str) -> Option<&Schema> { + for schema in self.object_schemas(&self.schema) { + let Some(map) = schema + .object + .as_ref() + .and_then(|object| object.properties.get(property)) + .and_then(Schema::as_object) + else { + continue; + }; + if let Some(value) = self + .object_schemas(map) + .find_map(|schema| schema.object.as_ref()?.additional_properties.as_deref()) + { + return Some(value); + } + } + None + } + + fn object_schemas<'a>( + &'a self, + schema: &'a SchemaObject, + ) -> impl Iterator { + let mut pending = vec![schema]; + let mut seen = Set::new(); + std::iter::from_fn(move || { + let schema = pending.pop()?; + if let Some(all_of) = schema.subschemas.as_ref().and_then(|s| s.all_of.as_ref()) { + pending.extend(all_of.iter().rev().filter_map(Schema::as_object)); + } + if let Some(name) = schema + .reference + .as_deref() + .and_then(|r| r.strip_prefix(DEFINITIONS_PREFIX)) + && seen.insert(name) + && let Some(target) = self.definitions.get(name).and_then(Schema::as_object) + { + pending.push(target); + } + Some(schema) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn root_map_lookup_follows_flattening_and_references() { + let root: RootSchema = serde_json::from_value(json!({ + "allOf": [{"$ref": "#/definitions/config"}], + "definitions": { + "config": {"properties": {"sources": {"$ref": "#/definitions/map"}}}, + "map": {"type": "object", "additionalProperties": {"$ref": "#/definitions/outer"}}, + "outer": {"type": "object"} + } + })) + .unwrap(); + assert_eq!( + root.root_map_value_schema("sources") + .unwrap() + .as_object() + .unwrap() + .reference + .as_deref(), + Some("#/definitions/outer") + ); + assert!(root.root_map_value_schema("sinks").is_none()); + } + + #[test] + fn root_map_lookup_does_not_search_nested_properties_or_loop_on_refs() { + let root: RootSchema = serde_json::from_value(json!({ + "allOf": [{"$ref": "#/definitions/loop"}], + "properties": {"nested": {"properties": { + "sources": {"additionalProperties": {"type": "object"}} + }}}, + "definitions": {"loop": {"$ref": "#/definitions/loop"}} + })) + .unwrap(); + assert!(root.root_map_value_schema("sources").is_none()); + } +} diff --git a/lib/vector-config/src/schema/parser/query.rs b/lib/vector-config/src/schema/parser/query.rs index 45b1fa8cabcfa..a5c49e5fd31f4 100644 --- a/lib/vector-config/src/schema/parser/query.rs +++ b/lib/vector-config/src/schema/parser/query.rs @@ -56,6 +56,24 @@ impl SchemaQuerier { pub fn query(&self) -> SchemaQueryBuilder<'_> { SchemaQueryBuilder::from_schema(&self.schema) } + + /// Gets the schema for values in a map at the configuration root. + pub fn root_map_value_schema(&self, property: &str) -> Result, QueryError> { + let value = self + .schema + .root_map_value_schema(property) + .ok_or(QueryError::NoMatches)?; + let schema = value.as_object().ok_or(QueryError::NoMatches)?; + let schema = match schema.reference.as_deref() { + Some(reference) => reference + .strip_prefix("#/definitions/") + .and_then(|name| self.schema.definitions.get(name)) + .and_then(Schema::as_object) + .ok_or(QueryError::NoMatches)?, + None => schema, + }; + Ok(schema.into()) + } } /// A query builder for querying against a root schema. @@ -430,3 +448,37 @@ fn schema_to_simple_schema(schema: &Schema) -> SimpleSchema<'_> { schema: schema_object, } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn queries_component_base_through_the_root_map() { + let querier = SchemaQuerier { + schema: serde_json::from_value(json!({ + "allOf": [{"properties": {"sources": { + "additionalProperties": {"$ref": "#/definitions/outer"} + }}}], + "definitions": {"outer": {"type": "object", "properties": { + "shared": {"type": "boolean"} + }}} + })) + .unwrap(), + }; + let base = querier.root_map_value_schema("sources").unwrap(); + assert!( + base.into_inner() + .object + .as_ref() + .unwrap() + .properties + .contains_key("shared") + ); + assert!(matches!( + querier.root_map_value_schema("sinks"), + Err(QueryError::NoMatches) + )); + } +} diff --git a/lib/vector-config/src/schema/visitors/inline_single.rs b/lib/vector-config/src/schema/visitors/inline_single.rs index a7010f8c6ca4a..dfd1d24bf2022 100644 --- a/lib/vector-config/src/schema/visitors/inline_single.rs +++ b/lib/vector-config/src/schema/visitors/inline_single.rs @@ -42,6 +42,16 @@ impl Visitor for InlineSingleUseReferencesVisitor { occurrence_visitor.visit_root_schema(root); let occurrence_map = occurrence_visitor.occurrence_map; + // The root component maps identify their outer schemas. Keep those + // definitions available to docs consumers even when referenced once. + let component_bases: HashSet<&str> = ["sources", "transforms", "sinks"] + .into_iter() + .filter_map(|property| root.root_map_value_schema(property)) + .filter_map(Schema::as_object) + .filter_map(|schema| schema.reference.as_deref()) + .map(get_cleaned_schema_reference) + .collect(); + self.eligible_to_inline = occurrence_map .into_iter() // Filter out any schemas which have more than one occurrence, as naturally, we're @@ -60,7 +70,8 @@ impl Visitor for InlineSingleUseReferencesVisitor { .and_then(Schema::as_object) .expect("schema definition must exist"); - is_inlineable_schema(def_name.as_ref(), schema) + !component_bases.contains(def_name.as_ref()) + && is_inlineable_schema(def_name.as_ref(), schema) }) .map(|s| s.as_ref().to_string()) .collect::>(); @@ -122,19 +133,18 @@ fn is_inlineable_schema(definition_name: &str, schema: &SchemaObject) -> bool { "vector::sinks::Sinks", ]; - // We want to avoid inlining all of the relevant top-level types used for defining components: - // the "outer" types (i.e. `SinkOuter`), the enum/collection types (i.e. the big `Sources` - // enum), and the component configuration types themselves (i.e. `AmqpSinkConfig`). + // Outer types are protected through the root component maps above. Also keep + // enum/collection types (the big `Sources` enum) and individual component + // configuration types (such as `AmqpSinkConfig`). // // There's nothing _technically_ wrong with doing so, but it would break downstream consumers of // the schema that parse it in order to extract the individual components and other // component-specific metadata. - let is_component_base = get_schema_metadata_attr(schema, "docs::component_base_type").is_some(); let is_component = get_schema_metadata_attr(schema, "docs::component_type").is_some(); let is_allowed_schema = !DISALLOWED_SCHEMAS.contains(&definition_name); - !is_component_base && !is_component && is_allowed_schema + !is_component && is_allowed_schema } #[derive(Debug, Default)] @@ -207,6 +217,38 @@ mod tests { assert_schemas_eq(expected_schema, actual_schema); } + #[test] + fn retains_root_component_map_values_without_metadata() { + for property in ["sources", "sinks", "transforms"] { + let mut schema = as_schema(json!({ + "allOf": [{"$ref": "#/definitions/config"}], + "definitions": { + "config": {"properties": {property: {"$ref": "#/definitions/map"}}}, + "map": {"type": "object", "additionalProperties": {"$ref": "#/definitions/renamed_outer"}}, + "renamed_outer": {"type": "object", "properties": { + "shared": {"$ref": "#/definitions/inline_me"} + }}, + "inline_me": {"type": "string"} + } + })); + InlineSingleUseReferencesVisitor::default().visit_root_schema(&mut schema); + assert!(schema.definitions.contains_key("renamed_outer")); + assert!(!schema.definitions.contains_key("map")); + assert!(!schema.definitions.contains_key("inline_me")); + assert_eq!(schema.definitions.len(), 1); + assert_eq!( + schema + .root_map_value_schema(property) + .unwrap() + .as_object() + .unwrap() + .reference + .as_deref(), + Some("#/definitions/renamed_outer") + ); + } + } + #[test] fn single_ref_single_usage() { let mut actual_schema = as_schema(json!({ diff --git a/src/config/sink.rs b/src/config/sink.rs index 5010f42fc6a40..e19f7fb67fbd6 100644 --- a/src/config/sink.rs +++ b/src/config/sink.rs @@ -56,7 +56,6 @@ impl From for BoxedSink { /// Fully resolved sink component. #[configurable_component] -#[configurable(metadata(docs::component_base_type = "sink"))] #[derive(Clone, derive_more::Debug)] pub struct SinkOuter where diff --git a/src/config/source.rs b/src/config/source.rs index df4cd8dd24b6c..2073692ee741c 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -50,7 +50,6 @@ impl From for BoxedSource { /// Fully resolved source component. #[configurable_component] -#[configurable(metadata(docs::component_base_type = "source"))] #[derive(Clone, Debug)] pub struct SourceOuter { #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] diff --git a/src/config/transform.rs b/src/config/transform.rs index 03cc0d4f8b954..f94a87462c853 100644 --- a/src/config/transform.rs +++ b/src/config/transform.rs @@ -55,7 +55,6 @@ impl From for BoxedTransform { /// Fully resolved transform component. #[configurable_component] -#[configurable(metadata(docs::component_base_type = "transform"))] #[derive(Clone, Debug)] pub struct TransformOuter where diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 4469215837079..13a4892f47918 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -364,18 +364,13 @@ pub fn run(schema_path: &Path) -> Result<()> { let component_types = ["source", "transform", "sink"]; // 1. Process Component Bases (sorted by component type for deterministic output) - let mut component_bases: IndexMap = IndexMap::new(); - if let Some(definitions) = root_schema.get("definitions").and_then(|d| d.as_object()) { - for (key, definition) in definitions { - if let Some(base_type) = - super::schema::get_schema_metadata(definition, "docs::component_base_type") - .and_then(|v| v.as_str()) - && component_types.contains(&base_type) - { - component_bases.insert(base_type.to_string(), key.clone()); - } - } - } + let mut component_bases: IndexMap = component_types + .iter() + .map(|kind| { + let name = component_base_schema_name(&context, kind)?; + Ok(((*kind).to_owned(), name.to_owned())) + }) + .collect::>()?; component_bases.sort_keys(); for (comp_type, schema_name) in &component_bases { @@ -554,6 +549,18 @@ fn render_schema( } } +// Component maps already point at their outer configuration schemas. Use that +// relationship rather than requiring a second identifier on each definition. +fn component_base_schema_name<'a>(context: &'a SchemaContext, kind: &str) -> Result<&'a str> { + let field = format!("{kind}s"); + context + .find_nested_object_property_schema(&context.root_schema, &field) + .and_then(|schema| schema.get("additionalProperties")) + .and_then(super::schema::get_schema_ref) + .and_then(|reference| reference.strip_prefix("#/definitions/")) + .with_context(|| format!("Could not find component base reference for '{field}'")) +} + fn render_generated_component_schema( context: &mut SchemaContext, schema_name: &str, @@ -820,6 +827,33 @@ fn hide_cue_definitions_field(cue_output: &str) -> Result { mod tests { use super::*; + #[test] + fn component_bases_come_from_root_map_references() { + let context = SchemaContext { + root_schema: json!({"allOf": [{"properties": { + "sources": {"additionalProperties": {"$ref": "#/definitions/renamed_source"}}, + "sinks": {"additionalProperties": {"$ref": "#/definitions/generic_sink"}}, + "transforms": {"additionalProperties": {"$ref": "#/definitions/transform"}} + }}]}), + cue_binary_path: String::new(), + resolved_schema_cache: IndexMap::new(), + expanded_schema_cache: IndexMap::new(), + }; + assert_eq!( + component_base_schema_name(&context, "source").unwrap(), + "renamed_source" + ); + assert_eq!( + component_base_schema_name(&context, "sink").unwrap(), + "generic_sink" + ); + assert_eq!( + component_base_schema_name(&context, "transform").unwrap(), + "transform" + ); + assert!(component_base_schema_name(&context, "missing").is_err()); + } + #[test] fn replaces_repeated_values_with_cue_references() { let shared_name = "shared_options".to_string(); From 904c47ae2bf3f3d65abbbce91cbec250cd4033d9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Sep 2026 11:22:47 -0400 Subject: [PATCH 2/2] chore(vdev): bump version to 0.3.21 --- Cargo.lock | 2 +- vdev/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5907f64428dea..8d69d689e6cd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13296,7 +13296,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.20" +version = "0.3.21" dependencies = [ "anyhow", "cfg-if", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 906189570a983..64c1d5894618e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.20" +version = "0.3.21" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0"